diff --git a/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIBinaryInputContent.cs b/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIBinaryInputContent.cs new file mode 100644 index 0000000000..bcf33f906f --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIBinaryInputContent.cs @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Text.Json.Serialization; + +#if ASPNETCORE +namespace Microsoft.Agents.AI.Hosting.AGUI.AspNetCore.Shared; +#else +namespace Microsoft.Agents.AI.AGUI.Shared; +#endif + +internal sealed class AGUIBinaryInputContent : AGUIInputContent +{ + public AGUIBinaryInputContent() + { + this.Type = "binary"; + } + + [JsonPropertyName("mimeType")] + [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? MimeType { get; set; } + + [JsonPropertyName("id")] + public string? Id { get; set; } + + [JsonPropertyName("url")] + public string? Url { get; set; } + + [JsonPropertyName("data")] + public string? Data { get; set; } + + [JsonPropertyName("filename")] + public string? Filename { get; set; } +} diff --git a/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIChatMessageExtensions.cs b/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIChatMessageExtensions.cs index 506956cac8..e4cf5225ec 100644 --- a/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIChatMessageExtensions.cs +++ b/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIChatMessageExtensions.cs @@ -94,15 +94,27 @@ public static IEnumerable AsChatMessages( { AGUIDeveloperMessage dev => dev.Content, AGUISystemMessage sys => sys.Content, - AGUIUserMessage user => user.Content, + AGUIUserMessage => string.Empty, AGUIAssistantMessage asst => asst.Content, _ => string.Empty }; - yield return new ChatMessage(role, content) + if (message is AGUIUserMessage userMessage) { - MessageId = message.Id - }; + yield return new ChatMessage(role, MapUserContents(userMessage)) + { + MessageId = message.Id, + AuthorName = userMessage.Name + }; + } + else + { + yield return new ChatMessage(role, content) + { + MessageId = message.Id + }; + } + break; } } @@ -137,7 +149,7 @@ public static IEnumerable AsAGUIMessages( { AGUIRoles.Developer => new AGUIDeveloperMessage { Id = message.MessageId, Content = message.Text ?? string.Empty }, AGUIRoles.System => new AGUISystemMessage { Id = message.MessageId, Content = message.Text ?? string.Empty }, - AGUIRoles.User => new AGUIUserMessage { Id = message.MessageId, Content = message.Text ?? string.Empty }, + AGUIRoles.User => MapUserMessage(message), _ => throw new InvalidOperationException($"Unknown role: {message.Role.Value}") }; } @@ -213,4 +225,137 @@ public static ChatRole MapChatRole(string role) => string.Equals(role, AGUIRoles.Developer, StringComparison.OrdinalIgnoreCase) ? s_developerChatRole : string.Equals(role, AGUIRoles.Tool, StringComparison.OrdinalIgnoreCase) ? ChatRole.Tool : throw new InvalidOperationException($"Unknown chat role: {role}"); + + private static List MapUserContents(AGUIUserMessage userMessage) + { + if (userMessage.InputContents is not { Length: > 0 }) + { + return [new TextContent(userMessage.Content)]; + } + + List contents = []; + foreach (AGUIInputContent inputContent in userMessage.InputContents) + { + switch (inputContent) + { + case AGUITextInputContent textInput: + contents.Add(new TextContent(textInput.Text)); + break; + case AGUIBinaryInputContent binaryInput: + contents.Add(MapBinaryInput(binaryInput)); + break; + default: + throw new InvalidOperationException($"Unsupported AG-UI input content type '{inputContent.GetType().Name}'."); + } + } + + return contents; + } + + private static AIContent MapBinaryInput(AGUIBinaryInputContent binaryInput) + { + if (!string.IsNullOrEmpty(binaryInput.Data)) + { + try + { + return new DataContent(Convert.FromBase64String(binaryInput.Data), binaryInput.MimeType ?? string.Empty) + { + Name = binaryInput.Filename + }; + } + catch (FormatException ex) + { + throw new InvalidOperationException("AG-UI binary input content contains invalid base64 data.", ex); + } + } + + if (!string.IsNullOrEmpty(binaryInput.Url)) + { + return new UriContent(binaryInput.Url, binaryInput.MimeType ?? string.Empty); + } + + if (!string.IsNullOrEmpty(binaryInput.Id)) + { + HostedFileContent hostedFileContent = new(binaryInput.Id) + { + Name = binaryInput.Filename + }; + + if (!string.IsNullOrEmpty(binaryInput.MimeType)) + { + hostedFileContent.MediaType = binaryInput.MimeType; + } + + return hostedFileContent; + } + + throw new InvalidOperationException("AG-UI binary input content must include id, url, or data."); + } + + private static AGUIUserMessage MapUserMessage(ChatMessage message) + { + List inputContents = []; + foreach (AIContent content in message.Contents) + { + switch (content) + { + case TextContent textContent: + inputContents.Add(new AGUITextInputContent { Text = textContent.Text }); + break; + case DataContent dataContent: + inputContents.Add(new AGUIBinaryInputContent + { + MimeType = dataContent.MediaType, + Data = dataContent.Base64Data.ToString(), + Filename = dataContent.Name + }); + break; + case UriContent uriContent: + inputContents.Add(new AGUIBinaryInputContent + { + MimeType = uriContent.MediaType, + Url = uriContent.Uri.ToString() + }); + break; + case HostedFileContent hostedFileContent: + inputContents.Add(new AGUIBinaryInputContent + { + MimeType = hostedFileContent.MediaType, + Id = hostedFileContent.FileId, + Filename = hostedFileContent.Name + }); + break; + default: + throw new InvalidOperationException($"Unsupported user AI content type '{content.GetType().Name}'."); + } + } + + if (inputContents.Count == 1 && + inputContents[0] is AGUITextInputContent textInputContent) + { + return new AGUIUserMessage + { + Id = message.MessageId, + Name = message.AuthorName, + Content = textInputContent.Text + }; + } + + if (inputContents.Count > 0) + { + return new AGUIUserMessage + { + Id = message.MessageId, + Name = message.AuthorName, + InputContents = [.. inputContents] + }; + } + + return new AGUIUserMessage + { + Id = message.MessageId, + Name = message.AuthorName, + Content = message.Text ?? string.Empty + }; + } } diff --git a/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIInputContent.cs b/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIInputContent.cs new file mode 100644 index 0000000000..baeff3e056 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIInputContent.cs @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Text.Json.Serialization; + +#if ASPNETCORE +namespace Microsoft.Agents.AI.Hosting.AGUI.AspNetCore.Shared; +#else +namespace Microsoft.Agents.AI.AGUI.Shared; +#endif + +[JsonConverter(typeof(AGUIInputContentJsonConverter))] +internal abstract class AGUIInputContent +{ + [JsonPropertyName("type")] + public string Type { get; set; } = string.Empty; +} diff --git a/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIInputContentJsonConverter.cs b/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIInputContentJsonConverter.cs new file mode 100644 index 0000000000..ea52fff9c7 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIInputContentJsonConverter.cs @@ -0,0 +1,79 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Text.Json; +using System.Text.Json.Serialization; + +#if ASPNETCORE +namespace Microsoft.Agents.AI.Hosting.AGUI.AspNetCore.Shared; +#else +namespace Microsoft.Agents.AI.AGUI.Shared; +#endif + +internal sealed class AGUIInputContentJsonConverter : JsonConverter +{ + private const string TypeDiscriminatorPropertyName = "type"; + + public override bool CanConvert(Type typeToConvert) => + typeof(AGUIInputContent).IsAssignableFrom(typeToConvert); + + public override AGUIInputContent Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + var jsonElementTypeInfo = options.GetTypeInfo(typeof(JsonElement)); + JsonElement jsonElement = (JsonElement)JsonSerializer.Deserialize(ref reader, jsonElementTypeInfo)!; + + if (!jsonElement.TryGetProperty(TypeDiscriminatorPropertyName, out JsonElement discriminatorElement)) + { + throw new JsonException($"Missing required property '{TypeDiscriminatorPropertyName}' for AGUIInputContent deserialization"); + } + + string? discriminator = discriminatorElement.GetString(); + + AGUIInputContent? result = discriminator switch + { + "text" => jsonElement.Deserialize(options.GetTypeInfo(typeof(AGUITextInputContent))) as AGUITextInputContent, + "binary" => DeserializeBinaryInputContent(jsonElement, options), + _ => throw new JsonException($"Unknown AGUIInputContent type discriminator: '{discriminator}'") + }; + + if (result is null) + { + throw new JsonException($"Failed to deserialize AGUIInputContent with type discriminator: '{discriminator}'"); + } + + return result; + } + + public override void Write(Utf8JsonWriter writer, AGUIInputContent value, JsonSerializerOptions options) + { + switch (value) + { + case AGUITextInputContent text: + JsonSerializer.Serialize(writer, text, options.GetTypeInfo(typeof(AGUITextInputContent))); + break; + case AGUIBinaryInputContent binary: + JsonSerializer.Serialize(writer, binary, options.GetTypeInfo(typeof(AGUIBinaryInputContent))); + break; + default: + throw new JsonException($"Unknown AGUIInputContent type: {value.GetType().Name}"); + } + } + + private static AGUIBinaryInputContent? DeserializeBinaryInputContent(JsonElement jsonElement, JsonSerializerOptions options) + { + AGUIBinaryInputContent? binaryContent = jsonElement.Deserialize(options.GetTypeInfo(typeof(AGUIBinaryInputContent))) as AGUIBinaryInputContent; + if (binaryContent is null) + { + return null; + } + + if (string.IsNullOrEmpty(binaryContent.Id) && + string.IsNullOrEmpty(binaryContent.Url) && + string.IsNullOrEmpty(binaryContent.Data)) + { + throw new JsonException("Binary input content must provide at least one of 'id', 'url', or 'data'."); + } + + return binaryContent; + } +} diff --git a/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIJsonSerializerContext.cs b/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIJsonSerializerContext.cs index b13a803625..a599215047 100644 --- a/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIJsonSerializerContext.cs +++ b/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIJsonSerializerContext.cs @@ -26,6 +26,10 @@ namespace Microsoft.Agents.AI.AGUI; [JsonSerializable(typeof(AGUIDeveloperMessage))] [JsonSerializable(typeof(AGUISystemMessage))] [JsonSerializable(typeof(AGUIUserMessage))] +[JsonSerializable(typeof(AGUIInputContent))] +[JsonSerializable(typeof(AGUIInputContent[]))] +[JsonSerializable(typeof(AGUITextInputContent))] +[JsonSerializable(typeof(AGUIBinaryInputContent))] [JsonSerializable(typeof(AGUIAssistantMessage))] [JsonSerializable(typeof(AGUIToolMessage))] [JsonSerializable(typeof(AGUITool))] diff --git a/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUITextInputContent.cs b/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUITextInputContent.cs new file mode 100644 index 0000000000..de6f96430b --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUITextInputContent.cs @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Text.Json.Serialization; + +#if ASPNETCORE +namespace Microsoft.Agents.AI.Hosting.AGUI.AspNetCore.Shared; +#else +namespace Microsoft.Agents.AI.AGUI.Shared; +#endif + +internal sealed class AGUITextInputContent : AGUIInputContent +{ + public AGUITextInputContent() + { + this.Type = "text"; + } + + [JsonPropertyName("text")] + public string Text { get; set; } = string.Empty; +} diff --git a/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIUserMessage.cs b/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIUserMessage.cs index e8e9f2ed57..880500df95 100644 --- a/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIUserMessage.cs +++ b/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIUserMessage.cs @@ -8,6 +8,7 @@ namespace Microsoft.Agents.AI.Hosting.AGUI.AspNetCore.Shared; namespace Microsoft.Agents.AI.AGUI.Shared; #endif +[JsonConverter(typeof(AGUIUserMessageJsonConverter))] internal sealed class AGUIUserMessage : AGUIMessage { public AGUIUserMessage() @@ -17,4 +18,7 @@ public AGUIUserMessage() [JsonPropertyName("name")] public string? Name { get; set; } + + [JsonIgnore] + public AGUIInputContent[]? InputContents { get; set; } } diff --git a/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIUserMessageJsonConverter.cs b/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIUserMessageJsonConverter.cs new file mode 100644 index 0000000000..99f0ed4e02 --- /dev/null +++ b/dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIUserMessageJsonConverter.cs @@ -0,0 +1,91 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Text.Json; +using System.Text.Json.Serialization; + +#if ASPNETCORE +namespace Microsoft.Agents.AI.Hosting.AGUI.AspNetCore.Shared; +#else +namespace Microsoft.Agents.AI.AGUI.Shared; +#endif + +internal sealed class AGUIUserMessageJsonConverter : JsonConverter +{ + public override AGUIUserMessage? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + var jsonElementTypeInfo = options.GetTypeInfo(typeof(JsonElement)); + JsonElement jsonElement = (JsonElement)JsonSerializer.Deserialize(ref reader, jsonElementTypeInfo)!; + + var message = new AGUIUserMessage(); + + if (jsonElement.TryGetProperty("id", out JsonElement idElement)) + { + message.Id = idElement.GetString(); + } + + if (jsonElement.TryGetProperty("role", out JsonElement roleElement)) + { + string? role = roleElement.GetString(); + if (!string.IsNullOrEmpty(role) && + !string.Equals(role, AGUIRoles.User, StringComparison.OrdinalIgnoreCase)) + { + throw new JsonException("AGUI user message role must be 'user'."); + } + } + + if (jsonElement.TryGetProperty("name", out JsonElement nameElement)) + { + message.Name = nameElement.GetString(); + } + + if (!jsonElement.TryGetProperty("content", out JsonElement contentElement)) + { + throw new JsonException("Missing required property 'content' for AGUIUserMessage deserialization."); + } + + switch (contentElement.ValueKind) + { + case JsonValueKind.String: + message.Content = contentElement.GetString() ?? string.Empty; + break; + case JsonValueKind.Array: + message.InputContents = contentElement.Deserialize(options.GetTypeInfo(typeof(AGUIInputContent[]))) as AGUIInputContent[]; + message.Content = string.Empty; + break; + default: + throw new JsonException("AGUI user message content must be a string or an array."); + } + + return message; + } + + public override void Write(Utf8JsonWriter writer, AGUIUserMessage value, JsonSerializerOptions options) + { + writer.WriteStartObject(); + + if (value.Id is not null) + { + writer.WriteString("id", value.Id); + } + + writer.WriteString("role", AGUIRoles.User); + + if (value.InputContents is { Length: > 0 }) + { + writer.WritePropertyName("content"); + JsonSerializer.Serialize(writer, value.InputContents, options.GetTypeInfo(typeof(AGUIInputContent[]))); + } + else + { + writer.WriteString("content", value.Content); + } + + if (value.Name is not null) + { + writer.WriteString("name", value.Name); + } + + writer.WriteEndObject(); + } +} diff --git a/dotnet/tests/Microsoft.Agents.AI.AGUI.UnitTests/AGUIChatClientTests.cs b/dotnet/tests/Microsoft.Agents.AI.AGUI.UnitTests/AGUIChatClientTests.cs index ede2c07d37..92cf9575a1 100644 --- a/dotnet/tests/Microsoft.Agents.AI.AGUI.UnitTests/AGUIChatClientTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.AGUI.UnitTests/AGUIChatClientTests.cs @@ -5,6 +5,7 @@ using System.Linq; using System.Net; using System.Net.Http; +using System.Text; using System.Text.Json; using System.Threading; using System.Threading.Tasks; @@ -1289,7 +1290,7 @@ public async Task GetStreamingResponseAsync_ExtractsStateFromDataContent_AndRemo // Arrange var stateData = new { counter = 42, status = "active" }; string stateJson = JsonSerializer.Serialize(stateData); - byte[] stateBytes = System.Text.Encoding.UTF8.GetBytes(stateJson); + byte[] stateBytes = Encoding.UTF8.GetBytes(stateJson); var dataContent = new DataContent(stateBytes, "application/json"); var captureHandler = new StateCapturingTestDelegatingHandler(); @@ -1359,7 +1360,7 @@ public async Task GetStreamingResponseAsync_WithNoStateDataContent_SendsEmptySta public async Task GetStreamingResponseAsync_WithMalformedStateJson_ThrowsInvalidOperationExceptionAsync() { // Arrange - byte[] invalidJson = System.Text.Encoding.UTF8.GetBytes("{invalid json"); + byte[] invalidJson = Encoding.UTF8.GetBytes("{invalid json"); var dataContent = new DataContent(invalidJson, "application/json"); using HttpClient httpClient = this.CreateMockHttpClient([]); @@ -1389,7 +1390,7 @@ public async Task GetStreamingResponseAsync_WithEmptyStateObject_SendsEmptyObjec // Arrange var emptyState = new { }; string stateJson = JsonSerializer.Serialize(emptyState); - byte[] stateBytes = System.Text.Encoding.UTF8.GetBytes(stateJson); + byte[] stateBytes = Encoding.UTF8.GetBytes(stateJson); var dataContent = new DataContent(stateBytes, "application/json"); var captureHandler = new StateCapturingTestDelegatingHandler(); @@ -1425,12 +1426,12 @@ public async Task GetStreamingResponseAsync_OnlyProcessesDataContentFromLastMess // Arrange var oldState = new { counter = 10 }; string oldStateJson = JsonSerializer.Serialize(oldState); - byte[] oldStateBytes = System.Text.Encoding.UTF8.GetBytes(oldStateJson); + byte[] oldStateBytes = Encoding.UTF8.GetBytes(oldStateJson); var oldDataContent = new DataContent(oldStateBytes, "application/json"); var newState = new { counter = 20 }; string newStateJson = JsonSerializer.Serialize(newState); - byte[] newStateBytes = System.Text.Encoding.UTF8.GetBytes(newStateJson); + byte[] newStateBytes = Encoding.UTF8.GetBytes(newStateJson); var newDataContent = new DataContent(newStateBytes, "application/json"); var captureHandler = new StateCapturingTestDelegatingHandler(); @@ -1470,7 +1471,7 @@ public async Task GetStreamingResponseAsync_OnlyProcessesDataContentFromLastMess public async Task GetStreamingResponseAsync_WithNonJsonMediaType_IgnoresDataContentAsync() { // Arrange - byte[] imageData = System.Text.Encoding.UTF8.GetBytes("fake image data"); + byte[] imageData = Encoding.UTF8.GetBytes("fake image data"); var dataContent = new DataContent(imageData, "image/png"); var captureHandler = new StateCapturingTestDelegatingHandler(); @@ -1500,6 +1501,94 @@ public async Task GetStreamingResponseAsync_WithNonJsonMediaType_IgnoresDataCont Assert.Equal(1, captureHandler.CapturedMessageCount); } + [Fact] + public async Task GetStreamingResponseAsync_WithMultimodalUserMessage_SerializesInputContentArrayAsync() + { + // Arrange + var captureHandler = new StateCapturingTestDelegatingHandler(); + captureHandler.AddResponse( + [ + new RunStartedEvent { ThreadId = "thread1", RunId = "run1" }, + new RunFinishedEvent { ThreadId = "thread1", RunId = "run1" } + ]); + using HttpClient httpClient = new(captureHandler); + + var chatClient = new AGUIChatClient(httpClient, "http://localhost/agent", null, AGUIJsonSerializerContext.Default.Options); + List messages = + [ + new(ChatRole.User, + [ + new TextContent("What is in this image?"), + new DataContent(Encoding.UTF8.GetBytes("png-bytes"), "image/png") { Name = "pixel.png" } + ]) + ]; + + // Act + await foreach (var _ in chatClient.GetStreamingResponseAsync(messages, null)) + { + // Just consume the stream + } + + // Assert + Assert.NotNull(captureHandler.CapturedInput); + AGUIUserMessage userMessage = Assert.IsType(captureHandler.CapturedInput.Messages.Single()); + Assert.NotNull(userMessage.InputContents); + Assert.Equal(2, userMessage.InputContents.Length); + + AGUITextInputContent textInput = Assert.IsType(userMessage.InputContents[0]); + Assert.Equal("What is in this image?", textInput.Text); + + AGUIBinaryInputContent binaryInput = Assert.IsType(userMessage.InputContents[1]); + Assert.Equal("image/png", binaryInput.MimeType); + Assert.Equal("pixel.png", binaryInput.Filename); + Assert.Equal(Convert.ToBase64String(Encoding.UTF8.GetBytes("png-bytes")), binaryInput.Data); + } + + [Fact] + public async Task GetStreamingResponseAsync_WithUriAndHostedFileUserContent_SerializesInputContentArrayAsync() + { + // Arrange + var captureHandler = new StateCapturingTestDelegatingHandler(); + captureHandler.AddResponse( + [ + new RunStartedEvent { ThreadId = "thread1", RunId = "run1" }, + new RunFinishedEvent { ThreadId = "thread1", RunId = "run1" } + ]); + using HttpClient httpClient = new(captureHandler); + + var chatClient = new AGUIChatClient(httpClient, "http://localhost/agent", null, AGUIJsonSerializerContext.Default.Options); + List messages = + [ + new(ChatRole.User, + [ + new TextContent("Inspect these assets"), + new UriContent("https://example.com/image.png", "image/png"), + new HostedFileContent("file_123") { MediaType = "application/pdf", Name = "doc.pdf" } + ]) + ]; + + // Act + await foreach (var _ in chatClient.GetStreamingResponseAsync(messages, null)) + { + // Just consume the stream + } + + // Assert + Assert.NotNull(captureHandler.CapturedInput); + AGUIUserMessage userMessage = Assert.IsType(captureHandler.CapturedInput.Messages.Single()); + Assert.NotNull(userMessage.InputContents); + Assert.Equal(3, userMessage.InputContents.Length); + + AGUIBinaryInputContent uriInput = Assert.IsType(userMessage.InputContents[1]); + Assert.Equal("https://example.com/image.png", uriInput.Url); + Assert.Equal("image/png", uriInput.MimeType); + + AGUIBinaryInputContent hostedFileInput = Assert.IsType(userMessage.InputContents[2]); + Assert.Equal("file_123", hostedFileInput.Id); + Assert.Equal("application/pdf", hostedFileInput.MimeType); + Assert.Equal("doc.pdf", hostedFileInput.Filename); + } + [Fact] public async Task GetStreamingResponseAsync_RoundTripState_PreservesJsonStructureAsync() { @@ -1583,7 +1672,7 @@ public async Task GetStreamingResponseAsync_ReceivesStateSnapshot_AsDataContentW DataContent dataContent = (DataContent)stateUpdate.Contents[0]; Assert.Equal("application/json", dataContent.MediaType); - string jsonText = System.Text.Encoding.UTF8.GetString(dataContent.Data.ToArray()); + string jsonText = Encoding.UTF8.GetString(dataContent.Data.ToArray()); JsonElement deserializedState = JsonElement.Parse(jsonText); Assert.Equal("abc123", deserializedState.GetProperty("sessionId").GetString()); Assert.Equal(5, deserializedState.GetProperty("step").GetInt32()); @@ -1690,6 +1779,7 @@ internal sealed class StateCapturingTestDelegatingHandler : DelegatingHandler public bool RequestWasMade { get; private set; } public JsonElement? CapturedState { get; private set; } public int CapturedMessageCount { get; private set; } + public RunAgentInput? CapturedInput { get; private set; } public void AddResponse(BaseEvent[] events) { @@ -1709,6 +1799,7 @@ protected override async Task SendAsync(HttpRequestMessage RunAgentInput? input = JsonSerializer.Deserialize(requestBody, AGUIJsonSerializerContext.Default.RunAgentInput); if (input != null) { + this.CapturedInput = input; if (input.State.ValueKind is not JsonValueKind.Undefined and not JsonValueKind.Null) { this.CapturedState = input.State; diff --git a/dotnet/tests/Microsoft.Agents.AI.AGUI.UnitTests/AGUIChatMessageExtensionsTests.cs b/dotnet/tests/Microsoft.Agents.AI.AGUI.UnitTests/AGUIChatMessageExtensionsTests.cs index bc3a73fb4c..9f4bfff5a1 100644 --- a/dotnet/tests/Microsoft.Agents.AI.AGUI.UnitTests/AGUIChatMessageExtensionsTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.AGUI.UnitTests/AGUIChatMessageExtensionsTests.cs @@ -3,6 +3,7 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Text; using System.Text.Json.Serialization; using Microsoft.Agents.AI.AGUI.Shared; using Microsoft.Extensions.AI; @@ -31,6 +32,10 @@ public sealed class WeatherResponse [JsonSerializable(typeof(Dictionary))] internal sealed partial class CustomTypesContext : JsonSerializerContext; +internal sealed class UnsupportedUserContent : AIContent +{ +} + /// /// Unit tests for the class. /// @@ -170,6 +175,247 @@ public void AsAGUIMessages_WithMultipleMessages_PreservesOrder() Assert.Equal("Third", ((AGUIUserMessage)aguiMessages[2]).Content); } + [Fact] + public void AsChatMessages_WithMultimodalUserMessage_MapsTextAndBinaryDataInOrder() + { + // Arrange + List aguiMessages = + [ + new AGUIUserMessage + { + Id = "msg1", + InputContents = + [ + new AGUITextInputContent { Text = "Describe this image" }, + new AGUIBinaryInputContent + { + MimeType = "image/png", + Filename = "pixel.png", + Data = Convert.ToBase64String(Encoding.UTF8.GetBytes("png-bytes")) + } + ] + } + ]; + + // Act + ChatMessage message = aguiMessages.AsChatMessages(AGUIJsonSerializerContext.Default.Options).Single(); + + // Assert + Assert.Equal(ChatRole.User, message.Role); + Assert.Equal(2, message.Contents.Count); + TextContent textContent = Assert.IsType(message.Contents[0]); + Assert.Equal("Describe this image", textContent.Text); + + DataContent dataContent = Assert.IsType(message.Contents[1]); + Assert.Equal("image/png", dataContent.MediaType); + Assert.Equal("pixel.png", dataContent.Name); + Assert.Equal("png-bytes", Encoding.UTF8.GetString(dataContent.Data.ToArray())); + } + + [Fact] + public void AsChatMessages_WithBinaryUrl_MapsToUriContent() + { + // Arrange + List aguiMessages = + [ + new AGUIUserMessage + { + Id = "msg1", + InputContents = + [ + new AGUIBinaryInputContent + { + MimeType = "image/png", + Url = "https://example.com/image.png" + } + ] + } + ]; + + // Act + ChatMessage message = aguiMessages.AsChatMessages(AGUIJsonSerializerContext.Default.Options).Single(); + + // Assert + UriContent uriContent = Assert.IsType(message.Contents.Single()); + Assert.Equal("image/png", uriContent.MediaType); + Assert.Equal("https://example.com/image.png", uriContent.Uri.ToString()); + } + + [Fact] + public void AsChatMessages_WithBinaryId_MapsToHostedFileContent() + { + // Arrange + List aguiMessages = + [ + new AGUIUserMessage + { + Id = "msg1", + InputContents = + [ + new AGUIBinaryInputContent + { + MimeType = "image/png", + Id = "file_123", + Filename = "hosted.png" + } + ] + } + ]; + + // Act + ChatMessage message = aguiMessages.AsChatMessages(AGUIJsonSerializerContext.Default.Options).Single(); + + // Assert + HostedFileContent hostedFileContent = Assert.IsType(message.Contents.Single()); + Assert.Equal("file_123", hostedFileContent.FileId); + Assert.Equal("image/png", hostedFileContent.MediaType); + Assert.Equal("hosted.png", hostedFileContent.Name); + } + + [Fact] + public void AsChatMessages_WithNamedUserMessage_MapsNameToAuthorName() + { + // Arrange + List aguiMessages = + [ + new AGUIUserMessage + { + Id = "msg1", + Name = "alice", + Content = "Hello" + } + ]; + + // Act + ChatMessage message = aguiMessages.AsChatMessages(AGUIJsonSerializerContext.Default.Options).Single(); + + // Assert + Assert.Equal("alice", message.AuthorName); + } + + [Fact] + public void AsAGUIMessages_WithMultimodalUserMessage_SerializesAsInputContentArray() + { + // Arrange + List chatMessages = + [ + new(ChatRole.User, + [ + new TextContent("What is in this image?"), + new DataContent(Encoding.UTF8.GetBytes("png-bytes"), "image/png") { Name = "pixel.png" } + ]) + { + MessageId = "msg1" + } + ]; + + // Act + AGUIUserMessage message = Assert.IsType(chatMessages.AsAGUIMessages(AGUIJsonSerializerContext.Default.Options).Single()); + + // Assert + Assert.NotNull(message.InputContents); + Assert.Equal(2, message.InputContents.Length); + + AGUITextInputContent textInput = Assert.IsType(message.InputContents[0]); + Assert.Equal("What is in this image?", textInput.Text); + + AGUIBinaryInputContent binaryInput = Assert.IsType(message.InputContents[1]); + Assert.Equal("image/png", binaryInput.MimeType); + Assert.Equal("pixel.png", binaryInput.Filename); + Assert.Equal(Convert.ToBase64String(Encoding.UTF8.GetBytes("png-bytes")), binaryInput.Data); + } + + [Fact] + public void AsAGUIMessages_WithNamedUserMessage_PreservesNameForTextContent() + { + // Arrange + List chatMessages = + [ + new ChatMessage(ChatRole.User, "Hello") + { + MessageId = "msg1", + AuthorName = "alice" + } + ]; + + // Act + AGUIUserMessage message = Assert.IsType(chatMessages.AsAGUIMessages(AGUIJsonSerializerContext.Default.Options).Single()); + + // Assert + Assert.Equal("alice", message.Name); + Assert.Equal("Hello", message.Content); + } + + [Fact] + public void AsAGUIMessages_WithNamedMultimodalUserMessage_PreservesName() + { + // Arrange + List chatMessages = + [ + new(ChatRole.User, + [ + new TextContent("What is in this image?"), + new DataContent(Encoding.UTF8.GetBytes("png-bytes"), "image/png") + ]) + { + MessageId = "msg1", + AuthorName = "alice" + } + ]; + + // Act + AGUIUserMessage message = Assert.IsType(chatMessages.AsAGUIMessages(AGUIJsonSerializerContext.Default.Options).Single()); + + // Assert + Assert.Equal("alice", message.Name); + Assert.NotNull(message.InputContents); + } + + [Fact] + public void AsAGUIMessages_WithUnsupportedUserContent_ThrowsInvalidOperationException() + { + // Arrange + List chatMessages = + [ + new(ChatRole.User, [new UnsupportedUserContent()]) + { + MessageId = "msg1" + } + ]; + + // Act & Assert + InvalidOperationException ex = Assert.Throws(() => chatMessages.AsAGUIMessages(AGUIJsonSerializerContext.Default.Options).ToList()); + Assert.Contains("Unsupported user AI content type", ex.Message); + } + + [Fact] + public void AsAGUIMessages_WithHostedFileWithoutMediaType_DoesNotSerializeEmptyMimeType() + { + // Arrange + List chatMessages = + [ + new(ChatRole.User, + [ + new HostedFileContent("file_123") + { + Name = "hosted.png" + } + ]) + { + MessageId = "msg1" + } + ]; + + // Act + AGUIUserMessage message = Assert.IsType(chatMessages.AsAGUIMessages(AGUIJsonSerializerContext.Default.Options).Single()); + string json = System.Text.Json.JsonSerializer.Serialize(message, AGUIJsonSerializerContext.Default.AGUIUserMessage); + System.Text.Json.JsonElement jsonElement = System.Text.Json.JsonElement.Parse(json); + System.Text.Json.JsonElement binaryContent = jsonElement.GetProperty("content")[0]; + + // Assert + Assert.False(binaryContent.TryGetProperty("mimeType", out _)); + } + [Fact] public void AsAGUIMessages_PreservesMessageId_WhenPresent() { diff --git a/dotnet/tests/Microsoft.Agents.AI.AGUI.UnitTests/AGUIJsonSerializerContextTests.cs b/dotnet/tests/Microsoft.Agents.AI.AGUI.UnitTests/AGUIJsonSerializerContextTests.cs index 33f259a681..838c50a024 100644 --- a/dotnet/tests/Microsoft.Agents.AI.AGUI.UnitTests/AGUIJsonSerializerContextTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.AGUI.UnitTests/AGUIJsonSerializerContextTests.cs @@ -804,6 +804,74 @@ public void AGUIUserMessage_SerializesAndDeserializes_Correctly() Assert.Equal("Hello, assistant!", deserialized.Content); } + [Fact] + public void AGUIUserMessage_WithInputContentArray_SerializesAndDeserializes_Correctly() + { + // Arrange + var originalMessage = new AGUIUserMessage + { + Id = "user2", + InputContents = + [ + new AGUITextInputContent { Text = "What is in this image?" }, + new AGUIBinaryInputContent + { + MimeType = "image/png", + Data = "aGVsbG8=", + Filename = "sample.png" + } + ] + }; + + // Act + string json = JsonSerializer.Serialize(originalMessage, AGUIJsonSerializerContext.Default.AGUIUserMessage); + JsonElement jsonElement = JsonElement.Parse(json); + var deserialized = JsonSerializer.Deserialize(json, AGUIJsonSerializerContext.Default.AGUIUserMessage); + + // Assert + Assert.Equal(JsonValueKind.Array, jsonElement.GetProperty("content").ValueKind); + Assert.NotNull(deserialized); + Assert.NotNull(deserialized.InputContents); + Assert.Equal(2, deserialized.InputContents.Length); + Assert.IsType(deserialized.InputContents[0]); + Assert.IsType(deserialized.InputContents[1]); + } + + [Fact] + public void AGUIUserMessage_Deserialize_WithNonUserRole_ThrowsJsonException() + { + // Arrange + const string Json = """ + { + "id": "user3", + "role": "assistant", + "content": "Hello" + } + """; + + // Act & Assert + Assert.Throws(() => JsonSerializer.Deserialize(Json, AGUIJsonSerializerContext.Default.AGUIUserMessage)); + } + + [Fact] + public void AGUIUserMessage_Serialize_AlwaysWritesUserRole() + { + // Arrange + AGUIUserMessage message = new() + { + Id = "user4", + Role = AGUIRoles.Assistant, + Content = "Hello" + }; + + // Act + string json = JsonSerializer.Serialize(message, AGUIJsonSerializerContext.Default.AGUIUserMessage); + JsonElement jsonElement = JsonElement.Parse(json); + + // Assert + Assert.Equal(AGUIRoles.User, jsonElement.GetProperty("role").GetString()); + } + [Fact] public void AGUISystemMessage_SerializesAndDeserializes_Correctly() { diff --git a/dotnet/tests/Microsoft.Agents.AI.Hosting.AGUI.AspNetCore.IntegrationTests/ForwardedPropertiesTests.cs b/dotnet/tests/Microsoft.Agents.AI.Hosting.AGUI.AspNetCore.IntegrationTests/ForwardedPropertiesTests.cs index 60d430d23c..1f0ab0a7a8 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Hosting.AGUI.AspNetCore.IntegrationTests/ForwardedPropertiesTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Hosting.AGUI.AspNetCore.IntegrationTests/ForwardedPropertiesTests.cs @@ -265,6 +265,55 @@ public async Task ForwardedProps_WithMixedTypes_AreCorrectlyParsedAsync() fakeAgent.ReceivedForwardedProperties.GetProperty("objectProp").GetProperty("nested").GetString().Should().Be("value"); } + [Fact] + public async Task MultimodalUserMessage_IsAcceptedAndPassedToAgentAsync() + { + // Arrange + FakeForwardedPropsAgent fakeAgent = new(); + await this.SetupTestServerAsync(fakeAgent); + + const string RequestJson = """ + { + "threadId": "thread-1", + "runId": "run-1", + "messages": [ + { + "id": "m1", + "role": "user", + "content": [ + { "type": "text", "text": "What is in this image?" }, + { + "type": "binary", + "mimeType": "image/png", + "filename": "pixel.png", + "data": "aGVsbG8=" + } + ] + } + ], + "context": [] + } + """; + + using StringContent content = new(RequestJson, Encoding.UTF8, "application/json"); + + // Act + HttpResponseMessage response = await this._client!.PostAsync(new Uri("/agent", UriKind.Relative), content); + + // Assert + response.IsSuccessStatusCode.Should().BeTrue(); + fakeAgent.ReceivedMessages.Should().HaveCount(1); + ChatMessage message = fakeAgent.ReceivedMessages[0]; + message.Role.Should().Be(ChatRole.User); + message.Contents.Should().HaveCount(2); + message.Contents[0].Should().BeOfType(); + message.Contents[1].Should().BeOfType(); + + DataContent dataContent = (DataContent)message.Contents[1]; + dataContent.MediaType.Should().Be("image/png"); + dataContent.Name.Should().Be("pixel.png"); + } + private async Task SetupTestServerAsync(FakeForwardedPropsAgent fakeAgent) { WebApplicationBuilder builder = WebApplication.CreateBuilder(); @@ -303,9 +352,11 @@ public FakeForwardedPropsAgent() public override string? Description => "Agent for forwarded properties testing"; public JsonElement ReceivedForwardedProperties { get; private set; } + public List ReceivedMessages { get; } = []; protected override Task RunCoreAsync(IEnumerable messages, AgentSession? session = null, AgentRunOptions? options = null, CancellationToken cancellationToken = default) { + this.CaptureMessages(messages); return this.RunCoreStreamingAsync(messages, session, options, cancellationToken).ToAgentResponseAsync(cancellationToken); } @@ -315,6 +366,8 @@ protected override async IAsyncEnumerable RunCoreStreamingA AgentRunOptions? options = null, [EnumeratorCancellation] CancellationToken cancellationToken = default) { + this.CaptureMessages(messages); + // Extract forwarded properties from ChatOptions.AdditionalProperties (set by AG-UI hosting layer) if (options is ChatClientAgentRunOptions { ChatOptions.AdditionalProperties: { } properties } && properties.TryGetValue("ag_ui_forwarded_properties", out object? propsObj) && @@ -364,4 +417,10 @@ public FakeAgentSession(AgentSessionStateBag stateBag) : base(stateBag) } public override object? GetService(Type serviceType, object? serviceKey = null) => null; + + private void CaptureMessages(IEnumerable messages) + { + this.ReceivedMessages.Clear(); + this.ReceivedMessages.AddRange(messages); + } } diff --git a/dotnet/tests/Microsoft.Agents.AI.Hosting.AGUI.AspNetCore.UnitTests/AGUIEndpointRouteBuilderExtensionsTests.cs b/dotnet/tests/Microsoft.Agents.AI.Hosting.AGUI.AspNetCore.UnitTests/AGUIEndpointRouteBuilderExtensionsTests.cs index 84a20e1938..33e16cb94c 100644 --- a/dotnet/tests/Microsoft.Agents.AI.Hosting.AGUI.AspNetCore.UnitTests/AGUIEndpointRouteBuilderExtensionsTests.cs +++ b/dotnet/tests/Microsoft.Agents.AI.Hosting.AGUI.AspNetCore.UnitTests/AGUIEndpointRouteBuilderExtensionsTests.cs @@ -191,6 +191,63 @@ AIAgent factory(IEnumerable messages, IEnumerable tools, IE Assert.Equal("Second", capturedMessages[1].Text); } + [Fact] + public async Task MapAGUIAgent_WithMultimodalUserMessage_ConvertsInputContentArrayAsync() + { + // Arrange + List? capturedMessages = null; + + AIAgent factory(IEnumerable messages, IEnumerable tools, IEnumerable> context, JsonElement props) + { + capturedMessages = messages.ToList(); + return new TestAgent(); + } + + const string Json = """ + { + "threadId": "thread1", + "runId": "run1", + "messages": [ + { + "id": "m1", + "role": "user", + "content": [ + { "type": "text", "text": "What is in this image?" }, + { + "type": "binary", + "mimeType": "image/png", + "filename": "pixel.png", + "data": "aGVsbG8=" + } + ] + } + ] + } + """; + + DefaultHttpContext httpContext = new(); + httpContext.Request.Body = new MemoryStream(Encoding.UTF8.GetBytes(Json)); + httpContext.Response.Body = new MemoryStream(); + + RequestDelegate handler = this.CreateRequestDelegate(factory); + + // Act + await handler(httpContext); + + // Assert + Assert.NotNull(capturedMessages); + ChatMessage message = Assert.Single(capturedMessages); + Assert.Equal(ChatRole.User, message.Role); + Assert.Equal(2, message.Contents.Count); + + TextContent textContent = Assert.IsType(message.Contents[0]); + Assert.Equal("What is in this image?", textContent.Text); + + DataContent dataContent = Assert.IsType(message.Contents[1]); + Assert.Equal("image/png", dataContent.MediaType); + Assert.Equal("pixel.png", dataContent.Name); + } + [Fact] public async Task MapAGUIAgent_ProducesValidAGUIEventStream_WithRunStartAndFinishAsync() {