From a798e30db66fb87c33b1ca85a7972863bb0c3eb8 Mon Sep 17 00:00:00 2001 From: Thomas Freudenberg Date: Wed, 18 Mar 2026 11:30:53 +0100 Subject: [PATCH 1/3] Fix AG-UI multimodal user message handling (#3729) Fixes #3729 Add typed AG-UI user input content models and converters so user messages can round-trip either plain text or multimodal content arrays. Update AG-UI chat message conversion, ASP.NET Core request binding, and AGUIChatClient serialization to preserve text, binary data, URLs, and hosted file references. Add unit and integration coverage for serializer behavior, endpoint binding, and end-to-end multimodal requests. --- .../Shared/AGUIBinaryInputContent.cs | 32 ++++ .../Shared/AGUIChatMessageExtensions.cs | 137 +++++++++++++++++- .../Shared/AGUIInputContent.cs | 16 ++ .../Shared/AGUIInputContentJsonConverter.cs | 79 ++++++++++ .../Shared/AGUIJsonSerializerContext.cs | 4 + .../Shared/AGUITextInputContent.cs | 20 +++ .../Shared/AGUIUserMessage.cs | 4 + .../Shared/AGUIUserMessageJsonConverter.cs | 86 +++++++++++ .../AGUIChatClientTests.cs | 105 +++++++++++++- .../AGUIChatMessageExtensionsTests.cs | 130 +++++++++++++++++ .../AGUIJsonSerializerContextTests.cs | 33 +++++ .../ForwardedPropertiesTests.cs | 53 +++++++ ...AGUIEndpointRouteBuilderExtensionsTests.cs | 57 ++++++++ 13 files changed, 744 insertions(+), 12 deletions(-) create mode 100644 dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIBinaryInputContent.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIInputContent.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIInputContentJsonConverter.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUITextInputContent.cs create mode 100644 dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIUserMessageJsonConverter.cs diff --git a/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIBinaryInputContent.cs b/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIBinaryInputContent.cs new file mode 100644 index 0000000000..4c42b3f4dd --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIBinaryInputContent.cs @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Text.Json.Serialization; + +#if ASPNETCORE +namespace Microsoft.Agents.AI.Hosting.AGUI.AspNetCore.Shared; +#else +namespace Microsoft.Agents.AI.AGUI.Shared; +#endif + +internal sealed class AGUIBinaryInputContent : AGUIInputContent +{ + public AGUIBinaryInputContent() + { + this.Type = "binary"; + } + + [JsonPropertyName("mimeType")] + public string MimeType { get; set; } = string.Empty; + + [JsonPropertyName("id")] + public string? Id { get; set; } + + [JsonPropertyName("url")] + public string? Url { get; set; } + + [JsonPropertyName("data")] + public string? Data { get; set; } + + [JsonPropertyName("filename")] + public string? Filename { get; set; } +} diff --git a/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIChatMessageExtensions.cs b/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIChatMessageExtensions.cs index 506956cac8..659bcf1c01 100644 --- a/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIChatMessageExtensions.cs +++ b/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIChatMessageExtensions.cs @@ -94,15 +94,26 @@ public static IEnumerable AsChatMessages( { AGUIDeveloperMessage dev => dev.Content, AGUISystemMessage sys => sys.Content, - AGUIUserMessage user => user.Content, + AGUIUserMessage => string.Empty, AGUIAssistantMessage asst => asst.Content, _ => string.Empty }; - yield return new ChatMessage(role, content) + if (message is AGUIUserMessage userMessage) { - MessageId = message.Id - }; + yield return new ChatMessage(role, MapUserContents(userMessage)) + { + MessageId = message.Id + }; + } + else + { + yield return new ChatMessage(role, content) + { + MessageId = message.Id + }; + } + break; } } @@ -137,7 +148,7 @@ public static IEnumerable AsAGUIMessages( { AGUIRoles.Developer => new AGUIDeveloperMessage { Id = message.MessageId, Content = message.Text ?? string.Empty }, AGUIRoles.System => new AGUISystemMessage { Id = message.MessageId, Content = message.Text ?? string.Empty }, - AGUIRoles.User => new AGUIUserMessage { Id = message.MessageId, Content = message.Text ?? string.Empty }, + AGUIRoles.User => MapUserMessage(message), _ => throw new InvalidOperationException($"Unknown role: {message.Role.Value}") }; } @@ -213,4 +224,120 @@ public static ChatRole MapChatRole(string role) => string.Equals(role, AGUIRoles.Developer, StringComparison.OrdinalIgnoreCase) ? s_developerChatRole : string.Equals(role, AGUIRoles.Tool, StringComparison.OrdinalIgnoreCase) ? ChatRole.Tool : throw new InvalidOperationException($"Unknown chat role: {role}"); + + private static List MapUserContents(AGUIUserMessage userMessage) + { + if (userMessage.InputContents is not { Length: > 0 }) + { + return [new TextContent(userMessage.Content)]; + } + + List contents = []; + foreach (AGUIInputContent inputContent in userMessage.InputContents) + { + switch (inputContent) + { + case AGUITextInputContent textInput: + contents.Add(new TextContent(textInput.Text)); + break; + case AGUIBinaryInputContent binaryInput: + contents.Add(MapBinaryInput(binaryInput)); + break; + default: + throw new InvalidOperationException($"Unsupported AG-UI input content type '{inputContent.GetType().Name}'."); + } + } + + return contents; + } + + private static AIContent MapBinaryInput(AGUIBinaryInputContent binaryInput) + { + if (!string.IsNullOrEmpty(binaryInput.Data)) + { + try + { + return new DataContent(Convert.FromBase64String(binaryInput.Data), binaryInput.MimeType) + { + Name = binaryInput.Filename + }; + } + catch (FormatException ex) + { + throw new InvalidOperationException("AG-UI binary input content contains invalid base64 data.", ex); + } + } + + if (!string.IsNullOrEmpty(binaryInput.Url)) + { + return new UriContent(binaryInput.Url, binaryInput.MimeType); + } + + if (!string.IsNullOrEmpty(binaryInput.Id)) + { + HostedFileContent hostedFileContent = new(binaryInput.Id) + { + Name = binaryInput.Filename + }; + + if (!string.IsNullOrEmpty(binaryInput.MimeType)) + { + hostedFileContent.MediaType = binaryInput.MimeType; + } + + return hostedFileContent; + } + + throw new InvalidOperationException("AG-UI binary input content must include id, url, or data."); + } + + private static AGUIUserMessage MapUserMessage(ChatMessage message) + { + List inputContents = []; + foreach (AIContent content in message.Contents) + { + switch (content) + { + case TextContent textContent: + inputContents.Add(new AGUITextInputContent { Text = textContent.Text }); + break; + case DataContent dataContent: + inputContents.Add(new AGUIBinaryInputContent + { + MimeType = dataContent.MediaType, + Data = dataContent.Base64Data.ToString(), + Filename = dataContent.Name + }); + break; + case UriContent uriContent: + inputContents.Add(new AGUIBinaryInputContent + { + MimeType = uriContent.MediaType, + Url = uriContent.Uri.ToString() + }); + break; + case HostedFileContent hostedFileContent: + inputContents.Add(new AGUIBinaryInputContent + { + MimeType = hostedFileContent.MediaType ?? string.Empty, + Id = hostedFileContent.FileId, + Filename = hostedFileContent.Name + }); + break; + } + } + + if (inputContents.Count == 1 && + inputContents[0] is AGUITextInputContent textInputContent) + { + return new AGUIUserMessage { Id = message.MessageId, Content = textInputContent.Text }; + } + + if (inputContents.Count > 0) + { + return new AGUIUserMessage { Id = message.MessageId, InputContents = [.. inputContents] }; + } + + return new AGUIUserMessage { Id = message.MessageId, Content = message.Text ?? string.Empty }; + } } diff --git a/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIInputContent.cs b/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIInputContent.cs new file mode 100644 index 0000000000..baeff3e056 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIInputContent.cs @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Text.Json.Serialization; + +#if ASPNETCORE +namespace Microsoft.Agents.AI.Hosting.AGUI.AspNetCore.Shared; +#else +namespace Microsoft.Agents.AI.AGUI.Shared; +#endif + +[JsonConverter(typeof(AGUIInputContentJsonConverter))] +internal abstract class AGUIInputContent +{ + [JsonPropertyName("type")] + public string Type { get; set; } = string.Empty; +} diff --git a/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIInputContentJsonConverter.cs b/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIInputContentJsonConverter.cs new file mode 100644 index 0000000000..ea52fff9c7 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIInputContentJsonConverter.cs @@ -0,0 +1,79 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Text.Json; +using System.Text.Json.Serialization; + +#if ASPNETCORE +namespace Microsoft.Agents.AI.Hosting.AGUI.AspNetCore.Shared; +#else +namespace Microsoft.Agents.AI.AGUI.Shared; +#endif + +internal sealed class AGUIInputContentJsonConverter : JsonConverter +{ + private const string TypeDiscriminatorPropertyName = "type"; + + public override bool CanConvert(Type typeToConvert) => + typeof(AGUIInputContent).IsAssignableFrom(typeToConvert); + + public override AGUIInputContent Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + var jsonElementTypeInfo = options.GetTypeInfo(typeof(JsonElement)); + JsonElement jsonElement = (JsonElement)JsonSerializer.Deserialize(ref reader, jsonElementTypeInfo)!; + + if (!jsonElement.TryGetProperty(TypeDiscriminatorPropertyName, out JsonElement discriminatorElement)) + { + throw new JsonException($"Missing required property '{TypeDiscriminatorPropertyName}' for AGUIInputContent deserialization"); + } + + string? discriminator = discriminatorElement.GetString(); + + AGUIInputContent? result = discriminator switch + { + "text" => jsonElement.Deserialize(options.GetTypeInfo(typeof(AGUITextInputContent))) as AGUITextInputContent, + "binary" => DeserializeBinaryInputContent(jsonElement, options), + _ => throw new JsonException($"Unknown AGUIInputContent type discriminator: '{discriminator}'") + }; + + if (result is null) + { + throw new JsonException($"Failed to deserialize AGUIInputContent with type discriminator: '{discriminator}'"); + } + + return result; + } + + public override void Write(Utf8JsonWriter writer, AGUIInputContent value, JsonSerializerOptions options) + { + switch (value) + { + case AGUITextInputContent text: + JsonSerializer.Serialize(writer, text, options.GetTypeInfo(typeof(AGUITextInputContent))); + break; + case AGUIBinaryInputContent binary: + JsonSerializer.Serialize(writer, binary, options.GetTypeInfo(typeof(AGUIBinaryInputContent))); + break; + default: + throw new JsonException($"Unknown AGUIInputContent type: {value.GetType().Name}"); + } + } + + private static AGUIBinaryInputContent? DeserializeBinaryInputContent(JsonElement jsonElement, JsonSerializerOptions options) + { + AGUIBinaryInputContent? binaryContent = jsonElement.Deserialize(options.GetTypeInfo(typeof(AGUIBinaryInputContent))) as AGUIBinaryInputContent; + if (binaryContent is null) + { + return null; + } + + if (string.IsNullOrEmpty(binaryContent.Id) && + string.IsNullOrEmpty(binaryContent.Url) && + string.IsNullOrEmpty(binaryContent.Data)) + { + throw new JsonException("Binary input content must provide at least one of 'id', 'url', or 'data'."); + } + + return binaryContent; + } +} diff --git a/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIJsonSerializerContext.cs b/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIJsonSerializerContext.cs index b13a803625..a599215047 100644 --- a/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIJsonSerializerContext.cs +++ b/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIJsonSerializerContext.cs @@ -26,6 +26,10 @@ namespace Microsoft.Agents.AI.AGUI; [JsonSerializable(typeof(AGUIDeveloperMessage))] [JsonSerializable(typeof(AGUISystemMessage))] [JsonSerializable(typeof(AGUIUserMessage))] +[JsonSerializable(typeof(AGUIInputContent))] +[JsonSerializable(typeof(AGUIInputContent[]))] +[JsonSerializable(typeof(AGUITextInputContent))] +[JsonSerializable(typeof(AGUIBinaryInputContent))] [JsonSerializable(typeof(AGUIAssistantMessage))] [JsonSerializable(typeof(AGUIToolMessage))] [JsonSerializable(typeof(AGUITool))] diff --git a/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUITextInputContent.cs b/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUITextInputContent.cs new file mode 100644 index 0000000000..de6f96430b --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUITextInputContent.cs @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Text.Json.Serialization; + +#if ASPNETCORE +namespace Microsoft.Agents.AI.Hosting.AGUI.AspNetCore.Shared; +#else +namespace Microsoft.Agents.AI.AGUI.Shared; +#endif + +internal sealed class AGUITextInputContent : AGUIInputContent +{ + public AGUITextInputContent() + { + this.Type = "text"; + } + + [JsonPropertyName("text")] + public string Text { get; set; } = string.Empty; +} diff --git a/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIUserMessage.cs b/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIUserMessage.cs index e8e9f2ed57..880500df95 100644 --- a/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIUserMessage.cs +++ b/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIUserMessage.cs @@ -8,6 +8,7 @@ namespace Microsoft.Agents.AI.Hosting.AGUI.AspNetCore.Shared; namespace Microsoft.Agents.AI.AGUI.Shared; #endif +[JsonConverter(typeof(AGUIUserMessageJsonConverter))] internal sealed class AGUIUserMessage : AGUIMessage { public AGUIUserMessage() @@ -17,4 +18,7 @@ public AGUIUserMessage() [JsonPropertyName("name")] public string? Name { get; set; } + + [JsonIgnore] + public AGUIInputContent[]? InputContents { get; set; } } diff --git a/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIUserMessageJsonConverter.cs b/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIUserMessageJsonConverter.cs new file mode 100644 index 0000000000..6ab8adbaa3 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIUserMessageJsonConverter.cs @@ -0,0 +1,86 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Text.Json; +using System.Text.Json.Serialization; + +#if ASPNETCORE +namespace Microsoft.Agents.AI.Hosting.AGUI.AspNetCore.Shared; +#else +namespace Microsoft.Agents.AI.AGUI.Shared; +#endif + +internal sealed class AGUIUserMessageJsonConverter : JsonConverter +{ + public override AGUIUserMessage? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + var jsonElementTypeInfo = options.GetTypeInfo(typeof(JsonElement)); + JsonElement jsonElement = (JsonElement)JsonSerializer.Deserialize(ref reader, jsonElementTypeInfo)!; + + var message = new AGUIUserMessage(); + + if (jsonElement.TryGetProperty("id", out JsonElement idElement)) + { + message.Id = idElement.GetString(); + } + + if (jsonElement.TryGetProperty("role", out JsonElement roleElement)) + { + message.Role = roleElement.GetString() ?? AGUIRoles.User; + } + + if (jsonElement.TryGetProperty("name", out JsonElement nameElement)) + { + message.Name = nameElement.GetString(); + } + + if (!jsonElement.TryGetProperty("content", out JsonElement contentElement)) + { + throw new JsonException("Missing required property 'content' for AGUIUserMessage deserialization"); + } + + switch (contentElement.ValueKind) + { + case JsonValueKind.String: + message.Content = contentElement.GetString() ?? string.Empty; + break; + case JsonValueKind.Array: + message.InputContents = contentElement.Deserialize(options.GetTypeInfo(typeof(AGUIInputContent[]))) as AGUIInputContent[]; + message.Content = string.Empty; + break; + default: + throw new JsonException("AGUI user message content must be a string or an array."); + } + + return message; + } + + public override void Write(Utf8JsonWriter writer, AGUIUserMessage value, JsonSerializerOptions options) + { + writer.WriteStartObject(); + + if (value.Id is not null) + { + writer.WriteString("id", value.Id); + } + + writer.WriteString("role", value.Role); + + if (value.InputContents is { Length: > 0 }) + { + writer.WritePropertyName("content"); + JsonSerializer.Serialize(writer, value.InputContents, options.GetTypeInfo(typeof(AGUIInputContent[]))); + } + else + { + writer.WriteString("content", value.Content); + } + + if (value.Name is not null) + { + writer.WriteString("name", value.Name); + } + + writer.WriteEndObject(); + } +} diff --git a/dotnet/tests/Microsoft.Agents.AI.AGUI.UnitTests/AGUIChatClientTests.cs b/dotnet/tests/Microsoft.Agents.AI.AGUI.UnitTests/AGUIChatClientTests.cs index ede2c07d37..92cf9575a1 100644 --- a/dotnet/tests/Microsoft.Agents.AI.AGUI.UnitTests/AGUIChatClientTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.AGUI.UnitTests/AGUIChatClientTests.cs @@ -5,6 +5,7 @@ using System.Linq; using System.Net; using System.Net.Http; +using System.Text; using System.Text.Json; using System.Threading; using System.Threading.Tasks; @@ -1289,7 +1290,7 @@ public async Task GetStreamingResponseAsync_ExtractsStateFromDataContent_AndRemo // Arrange var stateData = new { counter = 42, status = "active" }; string stateJson = JsonSerializer.Serialize(stateData); - byte[] stateBytes = System.Text.Encoding.UTF8.GetBytes(stateJson); + byte[] stateBytes = Encoding.UTF8.GetBytes(stateJson); var dataContent = new DataContent(stateBytes, "application/json"); var captureHandler = new StateCapturingTestDelegatingHandler(); @@ -1359,7 +1360,7 @@ public async Task GetStreamingResponseAsync_WithNoStateDataContent_SendsEmptySta public async Task GetStreamingResponseAsync_WithMalformedStateJson_ThrowsInvalidOperationExceptionAsync() { // Arrange - byte[] invalidJson = System.Text.Encoding.UTF8.GetBytes("{invalid json"); + byte[] invalidJson = Encoding.UTF8.GetBytes("{invalid json"); var dataContent = new DataContent(invalidJson, "application/json"); using HttpClient httpClient = this.CreateMockHttpClient([]); @@ -1389,7 +1390,7 @@ public async Task GetStreamingResponseAsync_WithEmptyStateObject_SendsEmptyObjec // Arrange var emptyState = new { }; string stateJson = JsonSerializer.Serialize(emptyState); - byte[] stateBytes = System.Text.Encoding.UTF8.GetBytes(stateJson); + byte[] stateBytes = Encoding.UTF8.GetBytes(stateJson); var dataContent = new DataContent(stateBytes, "application/json"); var captureHandler = new StateCapturingTestDelegatingHandler(); @@ -1425,12 +1426,12 @@ public async Task GetStreamingResponseAsync_OnlyProcessesDataContentFromLastMess // Arrange var oldState = new { counter = 10 }; string oldStateJson = JsonSerializer.Serialize(oldState); - byte[] oldStateBytes = System.Text.Encoding.UTF8.GetBytes(oldStateJson); + byte[] oldStateBytes = Encoding.UTF8.GetBytes(oldStateJson); var oldDataContent = new DataContent(oldStateBytes, "application/json"); var newState = new { counter = 20 }; string newStateJson = JsonSerializer.Serialize(newState); - byte[] newStateBytes = System.Text.Encoding.UTF8.GetBytes(newStateJson); + byte[] newStateBytes = Encoding.UTF8.GetBytes(newStateJson); var newDataContent = new DataContent(newStateBytes, "application/json"); var captureHandler = new StateCapturingTestDelegatingHandler(); @@ -1470,7 +1471,7 @@ public async Task GetStreamingResponseAsync_OnlyProcessesDataContentFromLastMess public async Task GetStreamingResponseAsync_WithNonJsonMediaType_IgnoresDataContentAsync() { // Arrange - byte[] imageData = System.Text.Encoding.UTF8.GetBytes("fake image data"); + byte[] imageData = Encoding.UTF8.GetBytes("fake image data"); var dataContent = new DataContent(imageData, "image/png"); var captureHandler = new StateCapturingTestDelegatingHandler(); @@ -1500,6 +1501,94 @@ public async Task GetStreamingResponseAsync_WithNonJsonMediaType_IgnoresDataCont Assert.Equal(1, captureHandler.CapturedMessageCount); } + [Fact] + public async Task GetStreamingResponseAsync_WithMultimodalUserMessage_SerializesInputContentArrayAsync() + { + // Arrange + var captureHandler = new StateCapturingTestDelegatingHandler(); + captureHandler.AddResponse( + [ + new RunStartedEvent { ThreadId = "thread1", RunId = "run1" }, + new RunFinishedEvent { ThreadId = "thread1", RunId = "run1" } + ]); + using HttpClient httpClient = new(captureHandler); + + var chatClient = new AGUIChatClient(httpClient, "http://localhost/agent", null, AGUIJsonSerializerContext.Default.Options); + List messages = + [ + new(ChatRole.User, + [ + new TextContent("What is in this image?"), + new DataContent(Encoding.UTF8.GetBytes("png-bytes"), "image/png") { Name = "pixel.png" } + ]) + ]; + + // Act + await foreach (var _ in chatClient.GetStreamingResponseAsync(messages, null)) + { + // Just consume the stream + } + + // Assert + Assert.NotNull(captureHandler.CapturedInput); + AGUIUserMessage userMessage = Assert.IsType(captureHandler.CapturedInput.Messages.Single()); + Assert.NotNull(userMessage.InputContents); + Assert.Equal(2, userMessage.InputContents.Length); + + AGUITextInputContent textInput = Assert.IsType(userMessage.InputContents[0]); + Assert.Equal("What is in this image?", textInput.Text); + + AGUIBinaryInputContent binaryInput = Assert.IsType(userMessage.InputContents[1]); + Assert.Equal("image/png", binaryInput.MimeType); + Assert.Equal("pixel.png", binaryInput.Filename); + Assert.Equal(Convert.ToBase64String(Encoding.UTF8.GetBytes("png-bytes")), binaryInput.Data); + } + + [Fact] + public async Task GetStreamingResponseAsync_WithUriAndHostedFileUserContent_SerializesInputContentArrayAsync() + { + // Arrange + var captureHandler = new StateCapturingTestDelegatingHandler(); + captureHandler.AddResponse( + [ + new RunStartedEvent { ThreadId = "thread1", RunId = "run1" }, + new RunFinishedEvent { ThreadId = "thread1", RunId = "run1" } + ]); + using HttpClient httpClient = new(captureHandler); + + var chatClient = new AGUIChatClient(httpClient, "http://localhost/agent", null, AGUIJsonSerializerContext.Default.Options); + List messages = + [ + new(ChatRole.User, + [ + new TextContent("Inspect these assets"), + new UriContent("https://example.com/image.png", "image/png"), + new HostedFileContent("file_123") { MediaType = "application/pdf", Name = "doc.pdf" } + ]) + ]; + + // Act + await foreach (var _ in chatClient.GetStreamingResponseAsync(messages, null)) + { + // Just consume the stream + } + + // Assert + Assert.NotNull(captureHandler.CapturedInput); + AGUIUserMessage userMessage = Assert.IsType(captureHandler.CapturedInput.Messages.Single()); + Assert.NotNull(userMessage.InputContents); + Assert.Equal(3, userMessage.InputContents.Length); + + AGUIBinaryInputContent uriInput = Assert.IsType(userMessage.InputContents[1]); + Assert.Equal("https://example.com/image.png", uriInput.Url); + Assert.Equal("image/png", uriInput.MimeType); + + AGUIBinaryInputContent hostedFileInput = Assert.IsType(userMessage.InputContents[2]); + Assert.Equal("file_123", hostedFileInput.Id); + Assert.Equal("application/pdf", hostedFileInput.MimeType); + Assert.Equal("doc.pdf", hostedFileInput.Filename); + } + [Fact] public async Task GetStreamingResponseAsync_RoundTripState_PreservesJsonStructureAsync() { @@ -1583,7 +1672,7 @@ public async Task GetStreamingResponseAsync_ReceivesStateSnapshot_AsDataContentW DataContent dataContent = (DataContent)stateUpdate.Contents[0]; Assert.Equal("application/json", dataContent.MediaType); - string jsonText = System.Text.Encoding.UTF8.GetString(dataContent.Data.ToArray()); + string jsonText = Encoding.UTF8.GetString(dataContent.Data.ToArray()); JsonElement deserializedState = JsonElement.Parse(jsonText); Assert.Equal("abc123", deserializedState.GetProperty("sessionId").GetString()); Assert.Equal(5, deserializedState.GetProperty("step").GetInt32()); @@ -1690,6 +1779,7 @@ internal sealed class StateCapturingTestDelegatingHandler : DelegatingHandler public bool RequestWasMade { get; private set; } public JsonElement? CapturedState { get; private set; } public int CapturedMessageCount { get; private set; } + public RunAgentInput? CapturedInput { get; private set; } public void AddResponse(BaseEvent[] events) { @@ -1709,6 +1799,7 @@ protected override async Task SendAsync(HttpRequestMessage RunAgentInput? input = JsonSerializer.Deserialize(requestBody, AGUIJsonSerializerContext.Default.RunAgentInput); if (input != null) { + this.CapturedInput = input; if (input.State.ValueKind is not JsonValueKind.Undefined and not JsonValueKind.Null) { this.CapturedState = input.State; diff --git a/dotnet/tests/Microsoft.Agents.AI.AGUI.UnitTests/AGUIChatMessageExtensionsTests.cs b/dotnet/tests/Microsoft.Agents.AI.AGUI.UnitTests/AGUIChatMessageExtensionsTests.cs index bc3a73fb4c..8fc35bf5f7 100644 --- a/dotnet/tests/Microsoft.Agents.AI.AGUI.UnitTests/AGUIChatMessageExtensionsTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.AGUI.UnitTests/AGUIChatMessageExtensionsTests.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Text; using System.Text.Json.Serialization; using Microsoft.Agents.AI.AGUI.Shared; using Microsoft.Extensions.AI; @@ -170,6 +171,135 @@ public void AsAGUIMessages_WithMultipleMessages_PreservesOrder() Assert.Equal("Third", ((AGUIUserMessage)aguiMessages[2]).Content); } + [Fact] + public void AsChatMessages_WithMultimodalUserMessage_MapsTextAndBinaryDataInOrder() + { + // Arrange + List aguiMessages = + [ + new AGUIUserMessage + { + Id = "msg1", + InputContents = + [ + new AGUITextInputContent { Text = "Describe this image" }, + new AGUIBinaryInputContent + { + MimeType = "image/png", + Filename = "pixel.png", + Data = Convert.ToBase64String(Encoding.UTF8.GetBytes("png-bytes")) + } + ] + } + ]; + + // Act + ChatMessage message = aguiMessages.AsChatMessages(AGUIJsonSerializerContext.Default.Options).Single(); + + // Assert + Assert.Equal(ChatRole.User, message.Role); + Assert.Equal(2, message.Contents.Count); + TextContent textContent = Assert.IsType(message.Contents[0]); + Assert.Equal("Describe this image", textContent.Text); + + DataContent dataContent = Assert.IsType(message.Contents[1]); + Assert.Equal("image/png", dataContent.MediaType); + Assert.Equal("pixel.png", dataContent.Name); + Assert.Equal("png-bytes", Encoding.UTF8.GetString(dataContent.Data.ToArray())); + } + + [Fact] + public void AsChatMessages_WithBinaryUrl_MapsToUriContent() + { + // Arrange + List aguiMessages = + [ + new AGUIUserMessage + { + Id = "msg1", + InputContents = + [ + new AGUIBinaryInputContent + { + MimeType = "image/png", + Url = "https://example.com/image.png" + } + ] + } + ]; + + // Act + ChatMessage message = aguiMessages.AsChatMessages(AGUIJsonSerializerContext.Default.Options).Single(); + + // Assert + UriContent uriContent = Assert.IsType(message.Contents.Single()); + Assert.Equal("image/png", uriContent.MediaType); + Assert.Equal("https://example.com/image.png", uriContent.Uri.ToString()); + } + + [Fact] + public void AsChatMessages_WithBinaryId_MapsToHostedFileContent() + { + // Arrange + List aguiMessages = + [ + new AGUIUserMessage + { + Id = "msg1", + InputContents = + [ + new AGUIBinaryInputContent + { + MimeType = "image/png", + Id = "file_123", + Filename = "hosted.png" + } + ] + } + ]; + + // Act + ChatMessage message = aguiMessages.AsChatMessages(AGUIJsonSerializerContext.Default.Options).Single(); + + // Assert + HostedFileContent hostedFileContent = Assert.IsType(message.Contents.Single()); + Assert.Equal("file_123", hostedFileContent.FileId); + Assert.Equal("image/png", hostedFileContent.MediaType); + Assert.Equal("hosted.png", hostedFileContent.Name); + } + + [Fact] + public void AsAGUIMessages_WithMultimodalUserMessage_SerializesAsInputContentArray() + { + // Arrange + List chatMessages = + [ + new(ChatRole.User, + [ + new TextContent("What is in this image?"), + new DataContent(Encoding.UTF8.GetBytes("png-bytes"), "image/png") { Name = "pixel.png" } + ]) + { + MessageId = "msg1" + } + ]; + + // Act + AGUIUserMessage message = Assert.IsType(chatMessages.AsAGUIMessages(AGUIJsonSerializerContext.Default.Options).Single()); + + // Assert + Assert.NotNull(message.InputContents); + Assert.Equal(2, message.InputContents.Length); + + AGUITextInputContent textInput = Assert.IsType(message.InputContents[0]); + Assert.Equal("What is in this image?", textInput.Text); + + AGUIBinaryInputContent binaryInput = Assert.IsType(message.InputContents[1]); + Assert.Equal("image/png", binaryInput.MimeType); + Assert.Equal("pixel.png", binaryInput.Filename); + Assert.Equal(Convert.ToBase64String(Encoding.UTF8.GetBytes("png-bytes")), binaryInput.Data); + } + [Fact] public void AsAGUIMessages_PreservesMessageId_WhenPresent() { diff --git a/dotnet/tests/Microsoft.Agents.AI.AGUI.UnitTests/AGUIJsonSerializerContextTests.cs b/dotnet/tests/Microsoft.Agents.AI.AGUI.UnitTests/AGUIJsonSerializerContextTests.cs index 33f259a681..4127238c58 100644 --- a/dotnet/tests/Microsoft.Agents.AI.AGUI.UnitTests/AGUIJsonSerializerContextTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.AGUI.UnitTests/AGUIJsonSerializerContextTests.cs @@ -804,6 +804,39 @@ public void AGUIUserMessage_SerializesAndDeserializes_Correctly() Assert.Equal("Hello, assistant!", deserialized.Content); } + [Fact] + public void AGUIUserMessage_WithInputContentArray_SerializesAndDeserializes_Correctly() + { + // Arrange + var originalMessage = new AGUIUserMessage + { + Id = "user2", + InputContents = + [ + new AGUITextInputContent { Text = "What is in this image?" }, + new AGUIBinaryInputContent + { + MimeType = "image/png", + Data = "aGVsbG8=", + Filename = "sample.png" + } + ] + }; + + // Act + string json = JsonSerializer.Serialize(originalMessage, AGUIJsonSerializerContext.Default.AGUIUserMessage); + JsonElement jsonElement = JsonElement.Parse(json); + var deserialized = JsonSerializer.Deserialize(json, AGUIJsonSerializerContext.Default.AGUIUserMessage); + + // Assert + Assert.Equal(JsonValueKind.Array, jsonElement.GetProperty("content").ValueKind); + Assert.NotNull(deserialized); + Assert.NotNull(deserialized.InputContents); + Assert.Equal(2, deserialized.InputContents.Length); + Assert.IsType(deserialized.InputContents[0]); + Assert.IsType(deserialized.InputContents[1]); + } + [Fact] public void AGUISystemMessage_SerializesAndDeserializes_Correctly() { diff --git a/dotnet/tests/Microsoft.Agents.AI.Hosting.AGUI.AspNetCore.IntegrationTests/ForwardedPropertiesTests.cs b/dotnet/tests/Microsoft.Agents.AI.Hosting.AGUI.AspNetCore.IntegrationTests/ForwardedPropertiesTests.cs index 60d430d23c..cf802d079d 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Hosting.AGUI.AspNetCore.IntegrationTests/ForwardedPropertiesTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Hosting.AGUI.AspNetCore.IntegrationTests/ForwardedPropertiesTests.cs @@ -265,6 +265,55 @@ public async Task ForwardedProps_WithMixedTypes_AreCorrectlyParsedAsync() fakeAgent.ReceivedForwardedProperties.GetProperty("objectProp").GetProperty("nested").GetString().Should().Be("value"); } + [Fact] + public async Task MultimodalUserMessage_IsAcceptedAndPassedToAgentAsync() + { + // Arrange + FakeForwardedPropsAgent fakeAgent = new(); + await this.SetupTestServerAsync(fakeAgent); + + const string RequestJson = """ + { + "threadId": "thread-1", + "runId": "run-1", + "messages": [ + { + "id": "m1", + "role": "user", + "content": [ + { "type": "text", "text": "What is in this image?" }, + { + "type": "binary", + "mimeType": "image/png", + "filename": "pixel.png", + "data": "aGVsbG8=" + } + ] + } + ], + "context": [] + } + """; + + using StringContent content = new(RequestJson, Encoding.UTF8, "application/json"); + + // Act + HttpResponseMessage response = await this._client!.PostAsync(new Uri("/agent", UriKind.Relative), content); + + // Assert + response.IsSuccessStatusCode.Should().BeTrue(); + fakeAgent.ReceivedMessages.Should().HaveCount(1); + ChatMessage message = fakeAgent.ReceivedMessages[0]; + message.Role.Should().Be(ChatRole.User); + message.Contents.Should().HaveCount(2); + message.Contents[0].Should().BeOfType(); + message.Contents[1].Should().BeOfType(); + + DataContent dataContent = (DataContent)message.Contents[1]; + dataContent.MediaType.Should().Be("image/png"); + dataContent.Name.Should().Be("pixel.png"); + } + private async Task SetupTestServerAsync(FakeForwardedPropsAgent fakeAgent) { WebApplicationBuilder builder = WebApplication.CreateBuilder(); @@ -303,6 +352,7 @@ public FakeForwardedPropsAgent() public override string? Description => "Agent for forwarded properties testing"; public JsonElement ReceivedForwardedProperties { get; private set; } + public List ReceivedMessages { get; } = []; protected override Task RunCoreAsync(IEnumerable messages, AgentSession? session = null, AgentRunOptions? options = null, CancellationToken cancellationToken = default) { @@ -315,6 +365,9 @@ protected override async IAsyncEnumerable RunCoreStreamingA AgentRunOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { + this.ReceivedMessages.Clear(); + this.ReceivedMessages.AddRange(messages); + // Extract forwarded properties from ChatOptions.AdditionalProperties (set by AG-UI hosting layer) if (options is ChatClientAgentRunOptions { ChatOptions.AdditionalProperties: { } properties } && properties.TryGetValue("ag_ui_forwarded_properties", out object? propsObj) && diff --git a/dotnet/tests/Microsoft.Agents.AI.Hosting.AGUI.AspNetCore.UnitTests/AGUIEndpointRouteBuilderExtensionsTests.cs b/dotnet/tests/Microsoft.Agents.AI.Hosting.AGUI.AspNetCore.UnitTests/AGUIEndpointRouteBuilderExtensionsTests.cs index 84a20e1938..33e16cb94c 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Hosting.AGUI.AspNetCore.UnitTests/AGUIEndpointRouteBuilderExtensionsTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Hosting.AGUI.AspNetCore.UnitTests/AGUIEndpointRouteBuilderExtensionsTests.cs @@ -191,6 +191,63 @@ AIAgent factory(IEnumerable messages, IEnumerable tools, IE Assert.Equal("Second", capturedMessages[1].Text); } + [Fact] + public async Task MapAGUIAgent_WithMultimodalUserMessage_ConvertsInputContentArrayAsync() + { + // Arrange + List? capturedMessages = null; + + AIAgent factory(IEnumerable messages, IEnumerable tools, IEnumerable> context, JsonElement props) + { + capturedMessages = messages.ToList(); + return new TestAgent(); + } + + const string Json = """ + { + "threadId": "thread1", + "runId": "run1", + "messages": [ + { + "id": "m1", + "role": "user", + "content": [ + { "type": "text", "text": "What is in this image?" }, + { + "type": "binary", + "mimeType": "image/png", + "filename": "pixel.png", + "data": "aGVsbG8=" + } + ] + } + ] + } + """; + + DefaultHttpContext httpContext = new(); + httpContext.Request.Body = new MemoryStream(Encoding.UTF8.GetBytes(Json)); + httpContext.Response.Body = new MemoryStream(); + + RequestDelegate handler = this.CreateRequestDelegate(factory); + + // Act + await handler(httpContext); + + // Assert + Assert.NotNull(capturedMessages); + ChatMessage message = Assert.Single(capturedMessages); + Assert.Equal(ChatRole.User, message.Role); + Assert.Equal(2, message.Contents.Count); + + TextContent textContent = Assert.IsType(message.Contents[0]); + Assert.Equal("What is in this image?", textContent.Text); + + DataContent dataContent = Assert.IsType(message.Contents[1]); + Assert.Equal("image/png", dataContent.MediaType); + Assert.Equal("pixel.png", dataContent.Name); + } + [Fact] public async Task MapAGUIAgent_ProducesValidAGUIEventStream_WithRunStartAndFinishAsync() { From 0dfaf3c8f7666c1ecae46f0ea3ee2f8c221de76e Mon Sep 17 00:00:00 2001 From: Thomas Freudenberg Date: Wed, 18 Mar 2026 14:35:29 +0100 Subject: [PATCH 2/3] Address PR feedback on AG-UI user message metadata Respond to review comments on PR #4761 by enforcing role="user" for AGUIUserMessage serialization and deserialization, and by preserving user message name metadata across AG-UI and ChatMessage conversions. Add regression tests for invalid roles, forced user-role serialization, and Name/AuthorName round-tripping for both text-only and multimodal user messages. --- .../Shared/AGUIChatMessageExtensions.cs | 24 +++++-- .../Shared/AGUIUserMessageJsonConverter.cs | 9 ++- .../AGUIChatMessageExtensionsTests.cs | 67 +++++++++++++++++++ .../AGUIJsonSerializerContextTests.cs | 35 ++++++++++ 4 files changed, 129 insertions(+), 6 deletions(-) diff --git a/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIChatMessageExtensions.cs b/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIChatMessageExtensions.cs index 659bcf1c01..9ca299f418 100644 --- a/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIChatMessageExtensions.cs +++ b/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIChatMessageExtensions.cs @@ -103,7 +103,8 @@ public static IEnumerable AsChatMessages( { yield return new ChatMessage(role, MapUserContents(userMessage)) { - MessageId = message.Id + MessageId = message.Id, + AuthorName = userMessage.Name }; } else @@ -330,14 +331,29 @@ private static AGUIUserMessage MapUserMessage(ChatMessage message) if (inputContents.Count == 1 && inputContents[0] is AGUITextInputContent textInputContent) { - return new AGUIUserMessage { Id = message.MessageId, Content = textInputContent.Text }; + return new AGUIUserMessage + { + Id = message.MessageId, + Name = message.AuthorName, + Content = textInputContent.Text + }; } if (inputContents.Count > 0) { - return new AGUIUserMessage { Id = message.MessageId, InputContents = [.. inputContents] }; + return new AGUIUserMessage + { + Id = message.MessageId, + Name = message.AuthorName, + InputContents = [.. inputContents] + }; } - return new AGUIUserMessage { Id = message.MessageId, Content = message.Text ?? string.Empty }; + return new AGUIUserMessage + { + Id = message.MessageId, + Name = message.AuthorName, + Content = message.Text ?? string.Empty + }; } } diff --git a/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIUserMessageJsonConverter.cs b/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIUserMessageJsonConverter.cs index 6ab8adbaa3..ab93fccf58 100644 --- a/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIUserMessageJsonConverter.cs +++ b/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIUserMessageJsonConverter.cs @@ -26,7 +26,12 @@ internal sealed class AGUIUserMessageJsonConverter : JsonConverter 0 }) { diff --git a/dotnet/tests/Microsoft.Agents.AI.AGUI.UnitTests/AGUIChatMessageExtensionsTests.cs b/dotnet/tests/Microsoft.Agents.AI.AGUI.UnitTests/AGUIChatMessageExtensionsTests.cs index 8fc35bf5f7..12bc2b0c25 100644 --- a/dotnet/tests/Microsoft.Agents.AI.AGUI.UnitTests/AGUIChatMessageExtensionsTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.AGUI.UnitTests/AGUIChatMessageExtensionsTests.cs @@ -268,6 +268,27 @@ public void AsChatMessages_WithBinaryId_MapsToHostedFileContent() Assert.Equal("hosted.png", hostedFileContent.Name); } + [Fact] + public void AsChatMessages_WithNamedUserMessage_MapsNameToAuthorName() + { + // Arrange + List aguiMessages = + [ + new AGUIUserMessage + { + Id = "msg1", + Name = "alice", + Content = "Hello" + } + ]; + + // Act + ChatMessage message = aguiMessages.AsChatMessages(AGUIJsonSerializerContext.Default.Options).Single(); + + // Assert + Assert.Equal("alice", message.AuthorName); + } + [Fact] public void AsAGUIMessages_WithMultimodalUserMessage_SerializesAsInputContentArray() { @@ -300,6 +321,52 @@ public void AsAGUIMessages_WithMultimodalUserMessage_SerializesAsInputContentArr Assert.Equal(Convert.ToBase64String(Encoding.UTF8.GetBytes("png-bytes")), binaryInput.Data); } + [Fact] + public void AsAGUIMessages_WithNamedUserMessage_PreservesNameForTextContent() + { + // Arrange + List chatMessages = + [ + new ChatMessage(ChatRole.User, "Hello") + { + MessageId = "msg1", + AuthorName = "alice" + } + ]; + + // Act + AGUIUserMessage message = Assert.IsType(chatMessages.AsAGUIMessages(AGUIJsonSerializerContext.Default.Options).Single()); + + // Assert + Assert.Equal("alice", message.Name); + Assert.Equal("Hello", message.Content); + } + + [Fact] + public void AsAGUIMessages_WithNamedMultimodalUserMessage_PreservesName() + { + // Arrange + List chatMessages = + [ + new(ChatRole.User, + [ + new TextContent("What is in this image?"), + new DataContent(Encoding.UTF8.GetBytes("png-bytes"), "image/png") + ]) + { + MessageId = "msg1", + AuthorName = "alice" + } + ]; + + // Act + AGUIUserMessage message = Assert.IsType(chatMessages.AsAGUIMessages(AGUIJsonSerializerContext.Default.Options).Single()); + + // Assert + Assert.Equal("alice", message.Name); + Assert.NotNull(message.InputContents); + } + [Fact] public void AsAGUIMessages_PreservesMessageId_WhenPresent() { diff --git a/dotnet/tests/Microsoft.Agents.AI.AGUI.UnitTests/AGUIJsonSerializerContextTests.cs b/dotnet/tests/Microsoft.Agents.AI.AGUI.UnitTests/AGUIJsonSerializerContextTests.cs index 4127238c58..838c50a024 100644 --- a/dotnet/tests/Microsoft.Agents.AI.AGUI.UnitTests/AGUIJsonSerializerContextTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.AGUI.UnitTests/AGUIJsonSerializerContextTests.cs @@ -837,6 +837,41 @@ public void AGUIUserMessage_WithInputContentArray_SerializesAndDeserializes_Corr Assert.IsType(deserialized.InputContents[1]); } + [Fact] + public void AGUIUserMessage_Deserialize_WithNonUserRole_ThrowsJsonException() + { + // Arrange + const string Json = """ + { + "id": "user3", + "role": "assistant", + "content": "Hello" + } + """; + + // Act & Assert + Assert.Throws(() => JsonSerializer.Deserialize(Json, AGUIJsonSerializerContext.Default.AGUIUserMessage)); + } + + [Fact] + public void AGUIUserMessage_Serialize_AlwaysWritesUserRole() + { + // Arrange + AGUIUserMessage message = new() + { + Id = "user4", + Role = AGUIRoles.Assistant, + Content = "Hello" + }; + + // Act + string json = JsonSerializer.Serialize(message, AGUIJsonSerializerContext.Default.AGUIUserMessage); + JsonElement jsonElement = JsonElement.Parse(json); + + // Assert + Assert.Equal(AGUIRoles.User, jsonElement.GetProperty("role").GetString()); + } + [Fact] public void AGUISystemMessage_SerializesAndDeserializes_Correctly() { From da8ab66f461669762d1bbdfcf3e96007ea914a3a Mon Sep 17 00:00:00 2001 From: Thomas Freudenberg Date: Wed, 18 Mar 2026 17:16:08 +0100 Subject: [PATCH 3/3] Address remaining PR feedback on AG-UI user content mapping Respond to the remaining review comments on PR #4761 by throwing for unsupported user AI content, omitting absent mime types from outbound binary input content, and making the integration test fake agent capture received messages in both RunCoreAsync and RunCoreStreamingAsync. Also standardize the AGUIUserMessageJsonConverter missing-content JsonException punctuation and add regression coverage for unsupported user content and hosted files without media types. --- .../Shared/AGUIBinaryInputContent.cs | 3 +- .../Shared/AGUIChatMessageExtensions.cs | 8 +-- .../Shared/AGUIUserMessageJsonConverter.cs | 2 +- .../AGUIChatMessageExtensionsTests.cs | 49 +++++++++++++++++++ .../ForwardedPropertiesTests.cs | 10 +++- 5 files changed, 65 insertions(+), 7 deletions(-) diff --git a/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIBinaryInputContent.cs b/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIBinaryInputContent.cs index 4c42b3f4dd..bcf33f906f 100644 --- a/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIBinaryInputContent.cs +++ b/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIBinaryInputContent.cs @@ -16,7 +16,8 @@ public AGUIBinaryInputContent() } [JsonPropertyName("mimeType")] - public string MimeType { get; set; } = string.Empty; + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? MimeType { get; set; } [JsonPropertyName("id")] public string? Id { get; set; } diff --git a/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIChatMessageExtensions.cs b/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIChatMessageExtensions.cs index 9ca299f418..e4cf5225ec 100644 --- a/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIChatMessageExtensions.cs +++ b/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIChatMessageExtensions.cs @@ -258,7 +258,7 @@ private static AIContent MapBinaryInput(AGUIBinaryInputContent binaryInput) { try { - return new DataContent(Convert.FromBase64String(binaryInput.Data), binaryInput.MimeType) + return new DataContent(Convert.FromBase64String(binaryInput.Data), binaryInput.MimeType ?? string.Empty) { Name = binaryInput.Filename }; @@ -271,7 +271,7 @@ private static AIContent MapBinaryInput(AGUIBinaryInputContent binaryInput) if (!string.IsNullOrEmpty(binaryInput.Url)) { - return new UriContent(binaryInput.Url, binaryInput.MimeType); + return new UriContent(binaryInput.Url, binaryInput.MimeType ?? string.Empty); } if (!string.IsNullOrEmpty(binaryInput.Id)) @@ -320,11 +320,13 @@ private static AGUIUserMessage MapUserMessage(ChatMessage message) case HostedFileContent hostedFileContent: inputContents.Add(new AGUIBinaryInputContent { - MimeType = hostedFileContent.MediaType ?? string.Empty, + MimeType = hostedFileContent.MediaType, Id = hostedFileContent.FileId, Filename = hostedFileContent.Name }); break; + default: + throw new InvalidOperationException($"Unsupported user AI content type '{content.GetType().Name}'."); } } diff --git a/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIUserMessageJsonConverter.cs b/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIUserMessageJsonConverter.cs index ab93fccf58..99f0ed4e02 100644 --- a/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIUserMessageJsonConverter.cs +++ b/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIUserMessageJsonConverter.cs @@ -41,7 +41,7 @@ internal sealed class AGUIUserMessageJsonConverter : JsonConverter))] internal sealed partial class CustomTypesContext : JsonSerializerContext; +internal sealed class UnsupportedUserContent : AIContent +{ +} + /// /// Unit tests for the class. /// @@ -367,6 +371,51 @@ public void AsAGUIMessages_WithNamedMultimodalUserMessage_PreservesName() Assert.NotNull(message.InputContents); } + [Fact] + public void AsAGUIMessages_WithUnsupportedUserContent_ThrowsInvalidOperationException() + { + // Arrange + List chatMessages = + [ + new(ChatRole.User, [new UnsupportedUserContent()]) + { + MessageId = "msg1" + } + ]; + + // Act & Assert + InvalidOperationException ex = Assert.Throws(() => chatMessages.AsAGUIMessages(AGUIJsonSerializerContext.Default.Options).ToList()); + Assert.Contains("Unsupported user AI content type", ex.Message); + } + + [Fact] + public void AsAGUIMessages_WithHostedFileWithoutMediaType_DoesNotSerializeEmptyMimeType() + { + // Arrange + List chatMessages = + [ + new(ChatRole.User, + [ + new HostedFileContent("file_123") + { + Name = "hosted.png" + } + ]) + { + MessageId = "msg1" + } + ]; + + // Act + AGUIUserMessage message = Assert.IsType(chatMessages.AsAGUIMessages(AGUIJsonSerializerContext.Default.Options).Single()); + string json = System.Text.Json.JsonSerializer.Serialize(message, AGUIJsonSerializerContext.Default.AGUIUserMessage); + System.Text.Json.JsonElement jsonElement = System.Text.Json.JsonElement.Parse(json); + System.Text.Json.JsonElement binaryContent = jsonElement.GetProperty("content")[0]; + + // Assert + Assert.False(binaryContent.TryGetProperty("mimeType", out _)); + } + [Fact] public void AsAGUIMessages_PreservesMessageId_WhenPresent() { diff --git a/dotnet/tests/Microsoft.Agents.AI.Hosting.AGUI.AspNetCore.IntegrationTests/ForwardedPropertiesTests.cs b/dotnet/tests/Microsoft.Agents.AI.Hosting.AGUI.AspNetCore.IntegrationTests/ForwardedPropertiesTests.cs index cf802d079d..1f0ab0a7a8 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Hosting.AGUI.AspNetCore.IntegrationTests/ForwardedPropertiesTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Hosting.AGUI.AspNetCore.IntegrationTests/ForwardedPropertiesTests.cs @@ -356,6 +356,7 @@ public FakeForwardedPropsAgent() protected override Task RunCoreAsync(IEnumerable messages, AgentSession? session = null, AgentRunOptions? options = null, CancellationToken cancellationToken = default) { + this.CaptureMessages(messages); return this.RunCoreStreamingAsync(messages, session, options, cancellationToken).ToAgentResponseAsync(cancellationToken); } @@ -365,8 +366,7 @@ protected override async IAsyncEnumerable RunCoreStreamingA AgentRunOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { - this.ReceivedMessages.Clear(); - this.ReceivedMessages.AddRange(messages); + this.CaptureMessages(messages); // Extract forwarded properties from ChatOptions.AdditionalProperties (set by AG-UI hosting layer) if (options is ChatClientAgentRunOptions { ChatOptions.AdditionalProperties: { } properties } && @@ -417,4 +417,10 @@ public FakeAgentSession(AgentSessionStateBag stateBag) : base(stateBag) } public override object? GetService(Type serviceType, object? serviceKey = null) => null; + + private void CaptureMessages(IEnumerable messages) + { + this.ReceivedMessages.Clear(); + this.ReceivedMessages.AddRange(messages); + } }