.NET: Fix AG-UI multimodal user message handling

dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIBinaryInputContent.cs

@@ -0,0 +1,33 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Text.Json.Serialization;
+
+#if ASPNETCORE
+namespace Microsoft.Agents.AI.Hosting.AGUI.AspNetCore.Shared;
+#else
+namespace Microsoft.Agents.AI.AGUI.Shared;
+#endif
+
+internal sealed class AGUIBinaryInputContent : AGUIInputContent
+{
+    public AGUIBinaryInputContent()
+    {
+        this.Type = "binary";
+    }
+
+    [JsonPropertyName("mimeType")]
+    [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)]
+    public string? MimeType { get; set; }
+
+    [JsonPropertyName("id")]
+    public string? Id { get; set; }
+
+    [JsonPropertyName("url")]
+    public string? Url { get; set; }
+
+    [JsonPropertyName("data")]
+    public string? Data { get; set; }
+
+    [JsonPropertyName("filename")]
+    public string? Filename { get; set; }
+}

dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIChatMessageExtensions.cs

@@ -94,15 +94,27 @@ public static IEnumerable<ChatMessage> AsChatMessages(
                     {
                         AGUIDeveloperMessage dev => dev.Content,
                         AGUISystemMessage sys => sys.Content,
-                        AGUIUserMessage user => user.Content,
+                        AGUIUserMessage => string.Empty,
                         AGUIAssistantMessage asst => asst.Content,
                         _ => string.Empty
                     };
 
-                    yield return new ChatMessage(role, content)
+                    if (message is AGUIUserMessage userMessage)
                     {
-                        MessageId = message.Id
-                    };
+                        yield return new ChatMessage(role, MapUserContents(userMessage))
+                        {
+                            MessageId = message.Id,
+                            AuthorName = userMessage.Name
+                        };
+                    }
+                    else
+                    {
+                        yield return new ChatMessage(role, content)
+                        {
+                            MessageId = message.Id
+                        };
+                    }
+
                     break;
                 }
             }
@@ -137,7 +149,7 @@ public static IEnumerable<AGUIMessage> AsAGUIMessages(
                 {
                     AGUIRoles.Developer => new AGUIDeveloperMessage { Id = message.MessageId, Content = message.Text ?? string.Empty },
                     AGUIRoles.System => new AGUISystemMessage { Id = message.MessageId, Content = message.Text ?? string.Empty },
-                    AGUIRoles.User => new AGUIUserMessage { Id = message.MessageId, Content = message.Text ?? string.Empty },
+                    AGUIRoles.User => MapUserMessage(message),
                     _ => throw new InvalidOperationException($"Unknown role: {message.Role.Value}")
                 };
             }
@@ -213,4 +225,137 @@ public static ChatRole MapChatRole(string role) =>
         string.Equals(role, AGUIRoles.Developer, StringComparison.OrdinalIgnoreCase) ? s_developerChatRole :
         string.Equals(role, AGUIRoles.Tool, StringComparison.OrdinalIgnoreCase) ? ChatRole.Tool :
         throw new InvalidOperationException($"Unknown chat role: {role}");
+
+    private static List<AIContent> MapUserContents(AGUIUserMessage userMessage)
+    {
+        if (userMessage.InputContents is not { Length: > 0 })
+        {
+            return [new TextContent(userMessage.Content)];
+        }
+
+        List<AIContent> contents = [];
+        foreach (AGUIInputContent inputContent in userMessage.InputContents)
+        {
+            switch (inputContent)
+            {
+                case AGUITextInputContent textInput:
+                    contents.Add(new TextContent(textInput.Text));
+                    break;
+                case AGUIBinaryInputContent binaryInput:
+                    contents.Add(MapBinaryInput(binaryInput));
+                    break;
+                default:
+                    throw new InvalidOperationException($"Unsupported AG-UI input content type '{inputContent.GetType().Name}'.");
+            }
+        }
+
+        return contents;
+    }
+
+    private static AIContent MapBinaryInput(AGUIBinaryInputContent binaryInput)
+    {
+        if (!string.IsNullOrEmpty(binaryInput.Data))
+        {
+            try
+            {
+                return new DataContent(Convert.FromBase64String(binaryInput.Data), binaryInput.MimeType ?? string.Empty)
+                {
+                    Name = binaryInput.Filename
+                };
+            }
+            catch (FormatException ex)
+            {
+                throw new InvalidOperationException("AG-UI binary input content contains invalid base64 data.", ex);
+            }
+        }
+
+        if (!string.IsNullOrEmpty(binaryInput.Url))
+        {
+            return new UriContent(binaryInput.Url, binaryInput.MimeType ?? string.Empty);
+        }
+
+        if (!string.IsNullOrEmpty(binaryInput.Id))
+        {
+            HostedFileContent hostedFileContent = new(binaryInput.Id)
+            {
+                Name = binaryInput.Filename
+            };
+
+            if (!string.IsNullOrEmpty(binaryInput.MimeType))
+            {
+                hostedFileContent.MediaType = binaryInput.MimeType;
+            }
+
+            return hostedFileContent;
+        }
+
+        throw new InvalidOperationException("AG-UI binary input content must include id, url, or data.");
+    }
+
+    private static AGUIUserMessage MapUserMessage(ChatMessage message)
+    {
+        List<AGUIInputContent> inputContents = [];
+        foreach (AIContent content in message.Contents)
+        {
+            switch (content)
+            {
+                case TextContent textContent:
+                    inputContents.Add(new AGUITextInputContent { Text = textContent.Text });
+                    break;
+                case DataContent dataContent:
+                    inputContents.Add(new AGUIBinaryInputContent
+                    {
+                        MimeType = dataContent.MediaType,
+                        Data = dataContent.Base64Data.ToString(),
+                        Filename = dataContent.Name
+                    });
+                    break;
+                case UriContent uriContent:
+                    inputContents.Add(new AGUIBinaryInputContent
+                    {
+                        MimeType = uriContent.MediaType,
+                        Url = uriContent.Uri.ToString()
+                    });
+                    break;
+                case HostedFileContent hostedFileContent:
+                    inputContents.Add(new AGUIBinaryInputContent
+                    {
+                        MimeType = hostedFileContent.MediaType,
+                        Id = hostedFileContent.FileId,
+                        Filename = hostedFileContent.Name
+                    });
+                    break;
+                default:
+                    throw new InvalidOperationException($"Unsupported user AI content type '{content.GetType().Name}'.");
+            }
+        }
+
+        if (inputContents.Count == 1 &&
+            inputContents[0] is AGUITextInputContent textInputContent)
+        {
+            return new AGUIUserMessage
+            {
+                Id = message.MessageId,
+                Name = message.AuthorName,
+                Content = textInputContent.Text
+            };
+        }
+
+        if (inputContents.Count > 0)
+        {
+            return new AGUIUserMessage
+            {
+                Id = message.MessageId,
+                Name = message.AuthorName,
+                InputContents = [.. inputContents]
+            };
+        }
+
+        return new AGUIUserMessage
+        {
+            Id = message.MessageId,
+            Name = message.AuthorName,
+            Content = message.Text ?? string.Empty
+        };
+    }
 }

dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIInputContent.cs

@@ -0,0 +1,16 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Text.Json.Serialization;
+
+#if ASPNETCORE
+namespace Microsoft.Agents.AI.Hosting.AGUI.AspNetCore.Shared;
+#else
+namespace Microsoft.Agents.AI.AGUI.Shared;
+#endif
+
+[JsonConverter(typeof(AGUIInputContentJsonConverter))]
+internal abstract class AGUIInputContent
+{
+    [JsonPropertyName("type")]
+    public string Type { get; set; } = string.Empty;
+}

dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIInputContentJsonConverter.cs

@@ -0,0 +1,79 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+#if ASPNETCORE
+namespace Microsoft.Agents.AI.Hosting.AGUI.AspNetCore.Shared;
+#else
+namespace Microsoft.Agents.AI.AGUI.Shared;
+#endif
+
+internal sealed class AGUIInputContentJsonConverter : JsonConverter<AGUIInputContent>
+{
+    private const string TypeDiscriminatorPropertyName = "type";
+
+    public override bool CanConvert(Type typeToConvert) =>
+        typeof(AGUIInputContent).IsAssignableFrom(typeToConvert);
+
+    public override AGUIInputContent Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options)
+    {
+        var jsonElementTypeInfo = options.GetTypeInfo(typeof(JsonElement));
+        JsonElement jsonElement = (JsonElement)JsonSerializer.Deserialize(ref reader, jsonElementTypeInfo)!;
+
+        if (!jsonElement.TryGetProperty(TypeDiscriminatorPropertyName, out JsonElement discriminatorElement))
+        {
+            throw new JsonException($"Missing required property '{TypeDiscriminatorPropertyName}' for AGUIInputContent deserialization");
+        }
+
+        string? discriminator = discriminatorElement.GetString();
+
+        AGUIInputContent? result = discriminator switch
+        {
+            "text" => jsonElement.Deserialize(options.GetTypeInfo(typeof(AGUITextInputContent))) as AGUITextInputContent,
+            "binary" => DeserializeBinaryInputContent(jsonElement, options),
+            _ => throw new JsonException($"Unknown AGUIInputContent type discriminator: '{discriminator}'")
+        };
+
+        if (result is null)
+        {
+            throw new JsonException($"Failed to deserialize AGUIInputContent with type discriminator: '{discriminator}'");
+        }
+
+        return result;
+    }
+
+    public override void Write(Utf8JsonWriter writer, AGUIInputContent value, JsonSerializerOptions options)
+    {
+        switch (value)
+        {
+            case AGUITextInputContent text:
+                JsonSerializer.Serialize(writer, text, options.GetTypeInfo(typeof(AGUITextInputContent)));
+                break;
+            case AGUIBinaryInputContent binary:
+                JsonSerializer.Serialize(writer, binary, options.GetTypeInfo(typeof(AGUIBinaryInputContent)));
+                break;
+            default:
+                throw new JsonException($"Unknown AGUIInputContent type: {value.GetType().Name}");
+        }
+    }
+
+    private static AGUIBinaryInputContent? DeserializeBinaryInputContent(JsonElement jsonElement, JsonSerializerOptions options)
+    {
+        AGUIBinaryInputContent? binaryContent = jsonElement.Deserialize(options.GetTypeInfo(typeof(AGUIBinaryInputContent))) as AGUIBinaryInputContent;
+        if (binaryContent is null)
+        {
+            return null;
+        }
+
+        if (string.IsNullOrEmpty(binaryContent.Id) &&
+            string.IsNullOrEmpty(binaryContent.Url) &&
+            string.IsNullOrEmpty(binaryContent.Data))
+        {
+            throw new JsonException("Binary input content must provide at least one of 'id', 'url', or 'data'.");
+        }
+
+        return binaryContent;
+    }
+}

dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIJsonSerializerContext.cs

@@ -26,6 +26,10 @@ namespace Microsoft.Agents.AI.AGUI;
 [JsonSerializable(typeof(AGUIDeveloperMessage))]
 [JsonSerializable(typeof(AGUISystemMessage))]
 [JsonSerializable(typeof(AGUIUserMessage))]
+[JsonSerializable(typeof(AGUIInputContent))]
+[JsonSerializable(typeof(AGUIInputContent[]))]
+[JsonSerializable(typeof(AGUITextInputContent))]
+[JsonSerializable(typeof(AGUIBinaryInputContent))]
 [JsonSerializable(typeof(AGUIAssistantMessage))]
 [JsonSerializable(typeof(AGUIToolMessage))]
 [JsonSerializable(typeof(AGUITool))]

dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUITextInputContent.cs

@@ -0,0 +1,20 @@
+// Copyright (c) Microsoft. All rights reserved.
+
+using System.Text.Json.Serialization;
+
+#if ASPNETCORE
+namespace Microsoft.Agents.AI.Hosting.AGUI.AspNetCore.Shared;
+#else
+namespace Microsoft.Agents.AI.AGUI.Shared;
+#endif
+
+internal sealed class AGUITextInputContent : AGUIInputContent
+{
+    public AGUITextInputContent()
+    {
+        this.Type = "text";
+    }
+
+    [JsonPropertyName("text")]
+    public string Text { get; set; } = string.Empty;
+}

dotnet/src/Microsoft.Agents.AI.AGUI/Shared/AGUIUserMessage.cs

@@ -8,6 +8,7 @@ namespace Microsoft.Agents.AI.Hosting.AGUI.AspNetCore.Shared;
 namespace Microsoft.Agents.AI.AGUI.Shared;
 #endif
 
+[JsonConverter(typeof(AGUIUserMessageJsonConverter))]
 internal sealed class AGUIUserMessage : AGUIMessage
 {
     public AGUIUserMessage()
@@ -17,4 +18,7 @@ public AGUIUserMessage()
 
     [JsonPropertyName("name")]
     public string? Name { get; set; }
+
+    [JsonIgnore]
+    public AGUIInputContent[]? InputContents { get; set; }
 }