diff --git a/src/MaIN.Core.UnitTests/ChatContextTests.cs b/src/MaIN.Core.UnitTests/ChatContextTests.cs index d83f4040..261c6982 100644 --- a/src/MaIN.Core.UnitTests/ChatContextTests.cs +++ b/src/MaIN.Core.UnitTests/ChatContextTests.cs @@ -88,7 +88,7 @@ public async Task CompleteAsync_ShouldCallChatService() }; - _mockChatService.Setup(s => s.Completions(It.IsAny(), It.IsAny(), It.IsAny(), null)) + _mockChatService.Setup(s => s.Completions(It.IsAny(), It.IsAny(), It.IsAny(), null, It.IsAny())) .ReturnsAsync(chatResult); _chatContext.WithMessage("User message"); @@ -98,7 +98,7 @@ public async Task CompleteAsync_ShouldCallChatService() var result = await _chatContext.CompleteAsync(); // Assert - _mockChatService.Verify(s => s.Completions(It.IsAny(), false, false, null), Times.Once); + _mockChatService.Verify(s => s.Completions(It.IsAny(), false, false, null, It.IsAny()), Times.Once); Assert.Equal(chatResult, result); } @@ -128,6 +128,6 @@ await _chatContext.WithModel(model) .CompleteAsync(); // Assert - _mockChatService.Verify(s => s.Completions(It.Is(c => c.ModelId == _testModelId && c.ModelInstance == model), It.IsAny(), It.IsAny(), null), Times.Once); + _mockChatService.Verify(s => s.Completions(It.Is(c => c.ModelId == _testModelId && c.ModelInstance == model), It.IsAny(), It.IsAny(), null, It.IsAny()), Times.Once); } } diff --git a/src/MaIN.Core/Hub/Contexts/ChatContext.cs b/src/MaIN.Core/Hub/Contexts/ChatContext.cs index 01bc9477..e62e2e35 100644 --- a/src/MaIN.Core/Hub/Contexts/ChatContext.cs +++ b/src/MaIN.Core/Hub/Contexts/ChatContext.cs @@ -198,7 +198,8 @@ public IChatConfigurationBuilder DisableCache() public async Task CompleteAsync( bool translate = false, // Move to WithTranslate bool interactive = false, // Move to WithInteractive - Func? changeOfValue = null) + Func? changeOfValue = null, + CancellationToken cancellationToken = default) { if (_chat.ModelInstance is null) { @@ -219,7 +220,7 @@ public async Task CompleteAsync( { await _chatService.Create(_chat); } - var result = await _chatService.Completions(_chat, translate, interactive, changeOfValue); + var result = await _chatService.Completions(_chat, translate, interactive, changeOfValue, cancellationToken); _files = []; return result; } diff --git a/src/MaIN.Core/Hub/Contexts/Interfaces/ChatContext/IChatConfigurationBuilder.cs b/src/MaIN.Core/Hub/Contexts/Interfaces/ChatContext/IChatConfigurationBuilder.cs index fa9d24a9..5c3c1788 100644 --- a/src/MaIN.Core/Hub/Contexts/Interfaces/ChatContext/IChatConfigurationBuilder.cs +++ b/src/MaIN.Core/Hub/Contexts/Interfaces/ChatContext/IChatConfigurationBuilder.cs @@ -104,5 +104,5 @@ public interface IChatConfigurationBuilder : IChatActions /// A flag indicating whether the chat session should be interactive. Default is false. /// An optional callback invoked whenever a new token or update is received during streaming. /// A object containing the result of the completed chat session. - Task CompleteAsync(bool translate = false, bool interactive = false, Func? changeOfValue = null); + Task CompleteAsync(bool translate = false, bool interactive = false, Func? changeOfValue = null, CancellationToken cancellationToken = default); } \ No newline at end of file diff --git a/src/MaIN.Domain/Entities/Chat.cs b/src/MaIN.Domain/Entities/Chat.cs index 892a6ff4..60a23314 100644 --- a/src/MaIN.Domain/Entities/Chat.cs +++ b/src/MaIN.Domain/Entities/Chat.cs @@ -9,25 +9,31 @@ public class Chat { public string Id { get; init; } = string.Empty; public required string Name { get; init; } + private string? _modelId; public required string ModelId { - get => _modelInstance?.Id ?? string.Empty; + get => _modelInstance?.Id ?? _modelId ?? string.Empty; set { + _modelId = value; if (string.IsNullOrEmpty(value)) { _modelInstance = null; return; } - _modelInstance = ModelRegistry.GetById(value); + ModelRegistry.TryGetById(value, out _modelInstance); } } private AIModel? _modelInstance; public AIModel? ModelInstance { get => _modelInstance; - set => (_modelInstance, ModelId) = (value, value?.Id ?? string.Empty); + set + { + _modelInstance = value; + _modelId = value?.Id ?? string.Empty; + } } public List Messages { get; set; } = []; public ChatType Type { get; set; } = ChatType.Conversation; diff --git a/src/MaIN.Domain/Entities/Message.cs b/src/MaIN.Domain/Entities/Message.cs index 5c54f71b..8efeebeb 100644 --- a/src/MaIN.Domain/Entities/Message.cs +++ b/src/MaIN.Domain/Entities/Message.cs @@ -19,7 +19,19 @@ public Message() public List Tokens { get; set; } = []; public bool Tool { get; init; } public DateTime Time { get; set; } - public byte[]? Image { get; init; } + public List? Images { get; set; } + + // Backward-compat wrapper – single image access + public byte[]? Image + { + get => Images?.Count > 0 ? Images[0] : null; + set + { + if (value == null) Images = null; + else Images = [value]; + } + } + public byte[]? Speech { get; set; } public List? Files { get; set; } public Dictionary Properties { get; set; } = []; diff --git a/src/MaIN.Domain/Models/Abstract/AIModel.cs b/src/MaIN.Domain/Models/Abstract/AIModel.cs index 610fc8e8..38296866 100644 --- a/src/MaIN.Domain/Models/Abstract/AIModel.cs +++ b/src/MaIN.Domain/Models/Abstract/AIModel.cs @@ -34,6 +34,9 @@ public abstract record AIModel( /// Checks if model supports vision/image input. public bool HasVision => this is IVisionModel; + + /// Checks if model generates images from text prompts. + public bool HasImageGeneration => this is IImageGenerationModel; } /// Base class for local models. diff --git a/src/MaIN.Domain/Models/Abstract/IModelCapabilities.cs b/src/MaIN.Domain/Models/Abstract/IModelCapabilities.cs index bb703b45..b35cb5dc 100644 --- a/src/MaIN.Domain/Models/Abstract/IModelCapabilities.cs +++ b/src/MaIN.Domain/Models/Abstract/IModelCapabilities.cs @@ -45,3 +45,8 @@ public interface IEmbeddingModel /// Interface for models that support text-to-speech. /// public interface ITTSModel; + +/// +/// Interface for models that generate images from text prompts. +/// +public interface IImageGenerationModel; diff --git a/src/MaIN.Domain/Models/Concrete/CloudModels.cs b/src/MaIN.Domain/Models/Concrete/CloudModels.cs index e03b57aa..eaf0ff8e 100644 --- a/src/MaIN.Domain/Models/Concrete/CloudModels.cs +++ b/src/MaIN.Domain/Models/Concrete/CloudModels.cs @@ -31,7 +31,14 @@ public sealed record DallE3() : CloudModel( BackendType.OpenAi, "DALL-E 3", 4000, - "Advanced image generation model from OpenAI"); + "Advanced image generation model from OpenAI"), IImageGenerationModel; + +public sealed record GptImage1() : CloudModel( + "gpt-image-1", + BackendType.OpenAi, + "GPT Image 1", + 4000, + "OpenAI's latest image generation model"), IImageGenerationModel; // ===== Anthropic Models ===== @@ -74,6 +81,13 @@ public sealed record Grok3Beta() : CloudModel( ModelDefaults.DefaultMaxContextWindow, "xAI latest Grok model in beta testing phase"); +public sealed record GrokImage() : CloudModel( + "grok-2-image", + BackendType.Xai, + "Grok 2 Image", + 4000, + "xAI image generation model"), IImageGenerationModel; + // ===== GroqCloud Models ===== public sealed record Llama3_1_8bInstant() : CloudModel( diff --git a/src/MaIN.Services/Services/LLMService/Utils/LLMApiRegistry.cs b/src/MaIN.Domain/Models/Concrete/LLMApiRegistry.cs similarity index 60% rename from src/MaIN.Services/Services/LLMService/Utils/LLMApiRegistry.cs rename to src/MaIN.Domain/Models/Concrete/LLMApiRegistry.cs index 302e73cf..726aa434 100644 --- a/src/MaIN.Services/Services/LLMService/Utils/LLMApiRegistry.cs +++ b/src/MaIN.Domain/Models/Concrete/LLMApiRegistry.cs @@ -1,4 +1,6 @@ -namespace MaIN.Services.Services.LLMService.Utils; +using MaIN.Domain.Configuration; + +namespace MaIN.Domain.Models.Concrete; public static class LLMApiRegistry { @@ -9,6 +11,18 @@ public static class LLMApiRegistry public static readonly LLMApiRegistryEntry Anthropic = new("Anthropic", "ANTHROPIC_API_KEY"); public static readonly LLMApiRegistryEntry Xai = new("Xai", "XAI_API_KEY"); public static readonly LLMApiRegistryEntry Ollama = new("Ollama", "OLLAMA_API_KEY"); + + public static LLMApiRegistryEntry? GetEntry(BackendType backendType) => backendType switch + { + BackendType.OpenAi => OpenAi, + BackendType.Gemini => Gemini, + BackendType.DeepSeek => Deepseek, + BackendType.GroqCloud => Groq, + BackendType.Anthropic => Anthropic, + BackendType.Xai => Xai, + BackendType.Ollama => Ollama, + _ => null + }; } public record LLMApiRegistryEntry(string ApiName, string ApiKeyEnvName); \ No newline at end of file diff --git a/src/MaIN.InferPage/Components/App.razor b/src/MaIN.InferPage/Components/App.razor index 2543f9aa..889b8ac3 100644 --- a/src/MaIN.InferPage/Components/App.razor +++ b/src/MaIN.InferPage/Components/App.razor @@ -9,12 +9,39 @@ + + + \ No newline at end of file diff --git a/src/MaIN.InferPage/Components/Layout/MainLayout.razor b/src/MaIN.InferPage/Components/Layout/MainLayout.razor index 0481fa40..b0f6917a 100644 --- a/src/MaIN.InferPage/Components/Layout/MainLayout.razor +++ b/src/MaIN.InferPage/Components/Layout/MainLayout.razor @@ -1,5 +1,4 @@ @inherits LayoutComponentBase -
diff --git a/src/MaIN.InferPage/Components/Layout/NavBar.razor b/src/MaIN.InferPage/Components/Layout/NavBar.razor index 5216c58e..c64a16eb 100644 --- a/src/MaIN.InferPage/Components/Layout/NavBar.razor +++ b/src/MaIN.InferPage/Components/Layout/NavBar.razor @@ -1,15 +1,23 @@ @using Microsoft.FluentUI.AspNetCore.Components.Icons.Regular +@using MaIN.Domain.Configuration @inject NavigationManager _navigationManager +@inject IJSRuntime JS @rendermode @(new InteractiveServerRenderMode(prerender: false))
- M.A.I.N - M.A.I.N + @GetBackendDisplayName() + @Utils.Model @@ -21,24 +29,50 @@ Color="#000" Style="margin-left: 10px">Reasoning ✨ } + @if (Utils.Visual) + { + Visual 🎨 + }
- - - - + + + +
@code { private DesignThemeModes Mode { get; set; } + private string AccentColor => Mode == DesignThemeModes.Dark ? "#00ffcc" : "#00cca3"; + private bool _isChangingTheme = false; + + protected override async Task OnAfterRenderAsync(bool firstRender) + { + if (firstRender) + { + var stored = await JS.InvokeAsync("themeManager.load"); + Mode = stored == "dark" ? DesignThemeModes.Dark : DesignThemeModes.Light; + StateHasChanged(); + } + } private void SetTheme() { + if (_isChangingTheme) return; + _isChangingTheme = true; Mode = Mode == DesignThemeModes.Dark ? DesignThemeModes.Light : DesignThemeModes.Dark; + _isChangingTheme = false; } private void Reload(MouseEventArgs obj) @@ -46,4 +80,18 @@ _navigationManager.Refresh(true); } -} \ No newline at end of file + private string GetBackendColor() + { + return Utils.IsLocal ? "#6b7280" : "#10a37f"; + } + + private string GetBackendDisplayName() + { + return Utils.BackendType switch + { + BackendType.Self => "Local", + BackendType.Ollama when !Utils.HasApiKey => "Local Ollama", + _ => Utils.BackendType.ToString() + }; + } +} diff --git a/src/MaIN.InferPage/Components/Pages/Home.razor b/src/MaIN.InferPage/Components/Pages/Home.razor index 4e4ad60c..3c10c131 100644 --- a/src/MaIN.InferPage/Components/Pages/Home.razor +++ b/src/MaIN.InferPage/Components/Pages/Home.razor @@ -1,15 +1,15 @@ @page "/" @rendermode @(new InteractiveServerRenderMode(prerender: true)) @inject IJSRuntime JS +@implements IDisposable @using MaIN.Core.Hub -@using MaIN.Core.Hub.Contexts @using MaIN.Core.Hub.Contexts.Interfaces.ChatContext +@using MaIN.Domain.Configuration @using MaIN.Domain.Entities @using MaIN.Domain.Exceptions @using MaIN.Domain.Models @using MaIN.Domain.Models.Abstract @using Markdig -@using Microsoft.FluentUI.AspNetCore.Components.Icons.Regular @using Message = MaIN.Domain.Entities.Message @using MessageType = MaIN.Domain.Entities.MessageType @@ -18,302 +18,627 @@ - - - -
- @foreach (var conversation in Messages) +
+ @if (_isDragging) + { +
+
+ + Drop files here +
+
+ } + + + +
+ @foreach (var conversation in Messages) + { + @if (conversation.Message.Role != "System") { -
- @if (conversation.Message.Role != "System") - { - @if (Chat.Visual) + + @if (conversation.Message.Role == "User") { - - @(conversation.Message.Role == "User" ? "User" : Utils.Model) - - @if (conversation.Message.Role == "User") + @if (conversation.AttachedFiles.Any() || conversation.AttachedImages.Any()) { - - @conversation.Message.Content - - } - else - { - -
- - imageResponse - -
- +
+ @foreach (var image in conversation.AttachedImages) + { +
+ @image.Name +
+ } + @foreach (var fileName in conversation.AttachedFiles) + { + + + @fileName + + } +
} +
+ @((MarkupString)Markdown.ToHtml(conversation.Message.Content ?? string.Empty, _markdownPipeline)) +
} else { - @if (conversation.Message.Role == "User") + @if (conversation.Message.Images?.Any() == true) { - - User - +
+ @foreach (var imageBytes in conversation.Message.Images) + { + var b64 = Convert.ToBase64String(imageBytes); +
+ + generated image + +
+ + + + + + +
+
+ } +
} else { - - - @Utils.Model - +
@if (_reasoning && conversation.Message.Role == "Assistant") { - +
+ + + + @if (conversation.ShowReason) + { +
+ @((MarkupString)Markdown.ToHtml(GetReasoningContent(conversation.Message), _markdownPipeline)) +
+ } +
+
} - + @((MarkupString)Markdown.ToHtml(GetMessageContent(conversation.Message), _markdownPipeline)) +
} - - - @if (conversation.Message.Role == "User") - { - @conversation.Message.Content - } - else - { -
- @if (conversation.ShowReason) - { -
- @((MarkupString)Markdown.ToHtml(string.Concat(conversation.Message.Tokens.Where(x => x.Type == TokenType.Reason).Select(x => x.Text)), - new MarkdownPipelineBuilder() - .UseAdvancedExtensions() - .Build())) -
-
- } - @((MarkupString)Markdown.ToHtml(string.Concat(conversation.Message.Tokens.Where(x => x.Type == TokenType.Message).Select(x => x.Text)), - new MarkdownPipelineBuilder() - .UseAdvancedExtensions() - .Build())) -
- } - - - } - } + + } + } + @if (_isLoading) + { + @if (Utils.Visual) + { + This might take a while... } - @if (_isLoading) + else if (_incomingMessage != null || _incomingReasoning != null) { - @if (Chat.Visual) + + @if (_isThinking) + { + + @((MarkupString)Markdown.ToHtml(_incomingReasoning ?? string.Empty, _markdownPipeline)) + + } + else + { + @((MarkupString)Markdown.ToHtml(_incomingMessage ?? string.Empty, _markdownPipeline)) + } + + } + } +
+
+
+ @if (_selectedImages.Any() || _selectedFiles.Any()) + { +
+ @foreach (var image in _selectedImages) { - @_displayName - This might take a while... - +
+ @image.File.Name + + + +
} - else + @foreach (var file in _selectedFiles) { - - @_displayName - - @if (_isThinking) - { - Thinking... - } - - @if (_incomingMessage != null || _incomingReasoning != null) - { - - @if (_isThinking) - { - - @((MarkupString)Markdown.ToHtml(_incomingReasoning!, - new MarkdownPipelineBuilder() - .UseAdvancedExtensions() - .Build())) - - } - else - { - @((MarkupString)Markdown.ToHtml(_incomingMessage!, - new MarkdownPipelineBuilder() - .UseAdvancedExtensions() - .Build())) - } - - - } +
+ @file.Name + + + +
} - } -
-
+
+ }
- - - - + + +
+ +
+
+
+ + + OnClick="@HandleSend"> -
- - - - -@* ReSharper disable once UnassignedField.Compiler *@ + + +
+
+ @code { - private string _prompt = string.Empty; private bool _isLoading; private bool _isThinking; + private bool _isDragging; private bool _reasoning; + private bool _preserveScroll; + private string _accentColor = "#00cca3"; private string? _errorMessage; - private string? _incomingMessage = null; - private string? _incomingReasoning = null; - private readonly string? _displayName = Utils.Model; - private IChatMessageBuilder? ctxBuilder; + private string? _incomingMessage; + private string? _incomingReasoning; + private IChatMessageBuilder? ctx; + private CancellationTokenSource? _cancellationTokenSource; private Chat Chat { get; } = new() { Name = "MaIN Infer", ModelId = Utils.Model! }; private List Messages { get; set; } = new(); private ElementReference? _bottomElement; - private int _inputKey = 0; + private ElementReference _editorRef; + private DotNetObjectReference? _dotNetRef; + private List _selectedFiles = new(); + private List<(FileInfo File, string Base64Preview)> _selectedImages = new(); + private int _inputKey; + + private static readonly HashSet ImageExtensions = [".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp"]; + + private readonly MarkdownPipeline _markdownPipeline = new MarkdownPipelineBuilder() + .UseAdvancedExtensions() + .UseSoftlineBreakAsHardlineBreak() + .Build(); protected override async Task OnAfterRenderAsync(bool firstRender) { - if (!firstRender) + if (firstRender) + { + var theme = await JS.InvokeAsync("themeManager.load"); + _accentColor = (theme == "dark" || theme == "system-dark") ? "#00ffcc" : "#00cca3"; + await JS.InvokeVoidAsync("scrollManager.attachScrollListener", "messages-container"); + await JS.InvokeVoidAsync("scrollManager.restoreScrollPosition", "messages-container"); + + _dotNetRef = DotNetObjectReference.Create(this); + await JS.InvokeVoidAsync("editorManager.attachPasteHandler", _editorRef, _dotNetRef); + await JS.InvokeVoidAsync("editorManager.attachDropZone", "chat-container", _dotNetRef); + + StateHasChanged(); + } + else if (_preserveScroll) { + _preserveScroll = false; await JS.InvokeVoidAsync("scrollManager.restoreScrollPosition", "messages-container"); } } protected override Task OnInitializedAsync() { - ctxBuilder = Utils.Visual - ? AIHub.Chat().EnableVisual() - : Utils.Path != null - ? AIHub.Chat().WithCustomModel(model: Utils.Model!, path: Utils.Path) - : AIHub.Chat().WithModel(Utils.Model!); //If that grows with different chat types we can consider switch ex + AIModel? model = null; - if (Utils.DeepSeek) + try { - _reasoning = Utils.Model!.ToLower().Contains("reasoner"); - Utils.Reason = _reasoning; + if (Utils.BackendType == BackendType.Self && Utils.Path != null) + { + model = ModelRegistry.TryGetById(Utils.Model!, out var foundModel1) + ? foundModel1! + : new GenericLocalModel($"{Utils.Model}.gguf"); + } + else + { + model = ModelRegistry.TryGetById(Utils.Model!, out var foundModel1) + ? foundModel1! + : new GenericCloudModel(Id: Utils.Model!, Backend: Utils.BackendType); + } + + ctx = Utils.Visual + ? AIHub.Chat().WithModel(model).EnableVisual() + : AIHub.Chat().WithModel(model); + } + catch (MaINCustomException ex) + { + _errorMessage = ex.PublicErrorMessage; } - else if (!Utils.OpenAi) + catch (Exception ex) { - var model = ModelRegistry.TryGetById(Utils.Model!, out var foundModel) ? foundModel : null; - _reasoning = !Utils.Visual && model?.HasReasoning == true; - Utils.Reason = _reasoning; + _errorMessage = ex.Message; } + _reasoning = !Utils.Visual && model?.HasReasoning == true; + Utils.Reason = _reasoning; + return base.OnInitializedAsync(); } - private async Task CheckEnterKey(KeyboardEventArgs e) + private async Task HandleKeyDown(KeyboardEventArgs e) { if (e is { Key: "Enter", ShiftKey: false }) { - _prompt = _prompt.Replace("\n", string.Empty); - await SendAsync(_prompt); + await HandleSend(); + } + } + + private async Task HandleSend() + { + if (_isLoading) return; + + var msg = await JS.InvokeAsync("editorManager.getInnerText", _editorRef); + if (!string.IsNullOrWhiteSpace(msg) || _selectedFiles.Any() || _selectedImages.Any()) + { + await JS.InvokeVoidAsync("editorManager.clearContent", _editorRef); + await SendAsync(msg?.Trim() ?? string.Empty); + } + } + + private async Task HandleFileSelected(InputFileChangeEventArgs e) + { + foreach (var file in e.GetMultipleFiles(10)) + { + var ms = new MemoryStream(); + try + { + await file.OpenReadStream(20 * 1024 * 1024).CopyToAsync(ms); + ms.Position = 0; + + var extension = Path.GetExtension(file.Name).ToLowerInvariant(); + var fileInfo = new FileInfo + { + Name = file.Name, + Extension = extension, + StreamContent = ms + }; + + if (ImageExtensions.Contains(extension)) + { + var base64 = Convert.ToBase64String(ms.ToArray()); + ms.Position = 0; + _selectedImages.Add((fileInfo, base64)); + } + else + { + _selectedFiles.Add(fileInfo); + } + } + catch (Exception ex) + { + await ms.DisposeAsync(); + _errorMessage = $"Failed to read file {file.Name}: {ex.Message}"; + } + } + StateHasChanged(); + } + + private void RemoveFile(FileInfo file) + { + file.StreamContent?.Dispose(); + _selectedFiles.Remove(file); + } + + private void RemoveImage((FileInfo File, string Base64Preview) image) + { + image.File.StreamContent?.Dispose(); + _selectedImages.Remove(image); + } + + [JSInvokable] + public async Task OnFileReceived(string fileName, string extension, string base64Data) + { + try + { + var bytes = Convert.FromBase64String(base64Data); + var ms = new MemoryStream(bytes); + var fileInfo = new FileInfo + { + Name = fileName, + Extension = extension, + StreamContent = ms + }; + + if (ImageExtensions.Contains(extension.ToLowerInvariant())) + { + _selectedImages.Add((fileInfo, base64Data)); + } + else + { + _selectedFiles.Add(fileInfo); + } + } + catch (Exception ex) + { + _errorMessage = $"Failed to load file {fileName}: {ex.Message}"; } + + _isDragging = false; + await InvokeAsync(StateHasChanged); + } + + [JSInvokable] + public async Task OnDragEnter() + { + _isDragging = true; + await InvokeAsync(StateHasChanged); + } + + [JSInvokable] + public async Task OnDragLeave() + { + _isDragging = false; + await InvokeAsync(StateHasChanged); + } + + private void HandleStop() + { + _cancellationTokenSource?.Cancel(); } private async Task SendAsync(string msg) { - if (!string.IsNullOrWhiteSpace(msg)) + if (string.IsNullOrWhiteSpace(msg) && !_selectedFiles.Any() && !_selectedImages.Any()) { + return; + } + + _cancellationTokenSource?.Dispose(); + _cancellationTokenSource = new CancellationTokenSource(); + var cancellationToken = _cancellationTokenSource.Token; + + _isLoading = true; + StateHasChanged(); + + var attachedFiles = new List(_selectedFiles); + var attachedImages = new List<(FileInfo File, string Base64Preview)>(_selectedImages); + try + { + _selectedFiles.Clear(); + _selectedImages.Clear(); + _inputKey++; StateHasChanged(); - var newMsg = new Message { Role = "User", Content = msg, Type = Utils.OpenAi || Utils.Gemini ? MessageType.CloudLLM : MessageType.LocalLLM }; + + var newMsg = new Message + { + Role = "User", + Content = msg, + Type = GetMessageType() + }; Chat.Messages.Add(newMsg); - Messages.Add(new MessageExt() + + Messages.Add(new MessageExt { - Message = newMsg + Message = newMsg, + AttachedFiles = attachedFiles.Select(f => f.Name).ToList(), + AttachedImages = attachedImages.Select(i => (i.File.Name, i.Base64Preview)).ToList() }); + Chat.ModelId = Utils.Model!; - _isLoading = true; Chat.Visual = Utils.Visual; - _inputKey++; - _prompt = string.Empty; - StateHasChanged(); + bool wasAtBottom = await JS.InvokeAsync("scrollManager.isAtBottom", "messages-container"); - try + StateHasChanged(); + + if (wasAtBottom) + await JS.InvokeVoidAsync("scrollManager.scrollToBottomSmooth", "messages-container"); + + var request = ctx!.WithMessage(msg); + + // Combine files and images - images go as files, ExtractImageFromFiles will handle them + var allFiles = new List(attachedFiles); + allFiles.AddRange(attachedImages.Select(i => i.File)); + + if (allFiles.Count != 0) { - await ctxBuilder!.WithMessage(msg) - .CompleteAsync(changeOfValue: async message => - { - if (message?.Type == TokenType.Reason) - { - _isThinking = true; - _incomingReasoning += message.Text; - } - else if (message?.Type == TokenType.Message) - { - _isThinking = false; - _incomingMessage += message.Text; - } + request.WithFiles(allFiles); + } - StateHasChanged(); - if (wasAtBottom) - { - await JS.InvokeVoidAsync("scrollManager.scrollToBottomSmooth", _bottomElement); - } - }); - - _isLoading = false; - var currentChat = (await ctxBuilder.GetCurrentChat()); - Chat.Messages.Add(currentChat.Messages.Last()); - Messages = Chat.Messages.Select(x => new MessageExt() + cancellationToken.ThrowIfCancellationRequested(); + + var completionTask = request.CompleteAsync(changeOfValue: async message => + { + if (cancellationToken.IsCancellationRequested) { - Message = x - }).ToList(); - _incomingReasoning = null; - _incomingMessage = null; - await JS.InvokeVoidAsync("scrollManager.scrollToBottomSmooth", _bottomElement); - StateHasChanged(); + return; + } - } - catch (Exception ex) + if (message?.Type == TokenType.Reason) + { + _isThinking = true; + _incomingReasoning += message.Text; + } + else if (message?.Type == TokenType.Message) + { + _isThinking = false; + _incomingMessage += message.Text; + } + + await InvokeAsync(StateHasChanged); + if (wasAtBottom) + { + await InvokeAsync(async () => await JS.InvokeVoidAsync("scrollManager.scrollToBottomSmooth", "messages-container")); + } + }, cancellationToken: cancellationToken); + + await completionTask.WaitAsync(cancellationToken); + + var currentChat = await ctx.GetCurrentChat(); + Chat.Messages.Add(currentChat.Messages.Last()); + + await JS.InvokeVoidAsync("scrollManager.saveScrollPosition", "messages-container"); + _preserveScroll = true; + RebuildMessagesWithFiles(); + _incomingReasoning = null; + _incomingMessage = null; + } + catch (OperationCanceledException) + { + bool hasPartialResponse = !string.IsNullOrWhiteSpace(_incomingMessage) || !string.IsNullOrWhiteSpace(_incomingReasoning); + + if (hasPartialResponse) { - _errorMessage = null; - StateHasChanged(); + var partialMsg = new Message + { + Role = "Assistant", + Content = _incomingMessage ?? string.Empty, + Type = GetMessageType(), + Time = DateTime.Now + }; + + if (!string.IsNullOrWhiteSpace(_incomingReasoning)) + partialMsg.Tokens.Add(new LLMTokenValue { Text = _incomingReasoning, Type = TokenType.Reason }); - _errorMessage = ex is MaINCustomException maInException - ? $"{maInException.PublicErrorMessage}" - : $"{ex.Message}"; + if (!string.IsNullOrWhiteSpace(_incomingMessage)) + partialMsg.Tokens.Add(new LLMTokenValue { Text = _incomingMessage, Type = TokenType.Message }); - StateHasChanged(); + Chat.Messages.Add(partialMsg); + RebuildMessagesWithFiles(); } - finally + else { - _isLoading = false; - _isThinking = false; + if (Chat.Messages.Count > 0 && Chat.Messages.Last().Role == "User") + Chat.Messages.RemoveAt(Chat.Messages.Count - 1); + + if (Messages.Count > 0 && Messages.Last().Message.Role == "User") + Messages.RemoveAt(Messages.Count - 1); } + + _incomingReasoning = null; + _incomingMessage = null; + StateHasChanged(); + } + catch (Exception ex) + { + _errorMessage = null; + StateHasChanged(); + + _errorMessage = ex is MaINCustomException maInException + ? maInException.PublicErrorMessage + : ex.Message; + + StateHasChanged(); + } + finally + { + foreach (var attachment in attachedFiles) + attachment.StreamContent?.Dispose(); + attachedFiles.Clear(); + + foreach (var image in attachedImages) + image.File.StreamContent?.Dispose(); + attachedImages.Clear(); + + _isLoading = false; + _isThinking = false; + StateHasChanged(); + } + } + + private void HandleInput(ChangeEventArgs obj) + { + // Handled by JS on send to ensure accuracy + } + + private string GetMessageContent(Message message) + { + var tokensContent = string.Concat(message.Tokens.Where(x => x.Type == TokenType.Message).Select(x => x.Text)); + return !string.IsNullOrEmpty(tokensContent) ? tokensContent : message.Content ?? string.Empty; + } + + private string GetReasoningContent(Message message) + { + return string.Concat(message.Tokens.Where(x => x.Type == TokenType.Reason).Select(x => x.Text)); + } + + private MessageType GetMessageType() + { + return Utils.IsLocal + ? MessageType.LocalLLM + : MessageType.CloudLLM; + } + + private async Task ToggleReasoning(MessageExt conversation) + { + await JS.InvokeVoidAsync("scrollManager.saveScrollPosition", "messages-container"); + _preserveScroll = true; + conversation.ShowReason = !conversation.ShowReason; + } + + private void RebuildMessagesWithFiles() + { + var existingFilesMap = Messages + .Where(m => m.AttachedFiles.Any()) + .ToDictionary(m => m.Message, m => m.AttachedFiles); + + var existingImagesMap = Messages + .Where(m => m.AttachedImages.Any()) + .ToDictionary(m => m.Message, m => m.AttachedImages); + + var existingReasonMap = Messages + .ToDictionary(m => m.Message, m => m.ShowReason); + + Messages = Chat.Messages.Select(x => new MessageExt + { + Message = x, + AttachedFiles = existingFilesMap.TryGetValue(x, out var files) ? files : new List(), + AttachedImages = existingImagesMap.TryGetValue(x, out var images) ? images : new List<(string, string)>(), + ShowReason = existingReasonMap.TryGetValue(x, out var show) && show + }).ToList(); + } + + private async Task CopyImageToClipboard(string base64) + { + try + { + await JS.InvokeVoidAsync("editorManager.copyImageToClipboard", base64); + } + catch + { + // silently ignore — clipboard API may not be available in all browsers } } - private void Callback(ChangeEventArgs obj) + public void Dispose() { - _prompt = obj.Value?.ToString()!; + _cancellationTokenSource?.Dispose(); + _dotNetRef?.Dispose(); } -} \ No newline at end of file +} diff --git a/src/MaIN.InferPage/Program.cs b/src/MaIN.InferPage/Program.cs index 3ae7d8e5..581e0b7b 100644 --- a/src/MaIN.InferPage/Program.cs +++ b/src/MaIN.InferPage/Program.cs @@ -1,15 +1,18 @@ using MaIN.Core; using MaIN.Domain.Configuration; -using MaIN.Domain.Models; +using MaIN.Domain.Models.Concrete; +using MaIN.Domain.Models.Abstract; using Microsoft.FluentUI.AspNetCore.Components; using MaIN.InferPage.Components; -using MaIN.Services.Services.LLMService.Utils; using Utils = MaIN.InferPage.Utils; -using MaIN.Domain.Models.Abstract; var builder = WebApplication.CreateBuilder(args); builder.Services.AddRazorComponents() - .AddInteractiveServerComponents(); + .AddInteractiveServerComponents() + .AddHubOptions(options => + { + options.MaximumReceiveMessageSize = 10 * 1024 * 1024; // 10 MB + }); builder.Services.AddFluentUIComponents(); try @@ -18,88 +21,66 @@ var modelPathArg = builder.Configuration["path"]; var backendArg = builder.Configuration["backend"]; - if (!string.IsNullOrEmpty(modelArg)) - { - Utils.Model = modelArg; - if (string.IsNullOrEmpty(modelPathArg)) + if (backendArg != null) + { + Utils.BackendType = backendArg.ToLower() switch + { + "openai" => BackendType.OpenAi, + "gemini" => BackendType.Gemini, + "deepseek" => BackendType.DeepSeek, + "groqcloud" => BackendType.GroqCloud, + "anthropic" => BackendType.Anthropic, + "xai" => BackendType.Xai, + "ollama" => BackendType.Ollama, + _ => BackendType.Self + }; + + if (Utils.BackendType != BackendType.Self) { - Console.WriteLine("Error: A model path must be provided using --path when a model is specified."); - return; + var apiKeyVariable = LLMApiRegistry.GetEntry(Utils.BackendType)?.ApiKeyEnvName ?? string.Empty; + var key = Environment.GetEnvironmentVariable(apiKeyVariable); + + if (string.IsNullOrEmpty(key) && !string.IsNullOrEmpty(apiKeyVariable)) + { + Console.Write($"Please enter your {Utils.BackendType.ToString()} API key: "); + key = Console.ReadLine(); + + if (!string.IsNullOrWhiteSpace(key)) + { + Utils.HasApiKey = true; + Environment.SetEnvironmentVariable(apiKeyVariable, key); + } + } } + } + + if (!string.IsNullOrEmpty(modelArg)) + { + Utils.Model = modelArg; Utils.Path = modelPathArg; - var envModelsPath = Environment.GetEnvironmentVariable("MaIN_ModelsPath"); - if (string.IsNullOrEmpty(envModelsPath)) + if (Utils.BackendType == BackendType.Self) { - Console.Write("Please enter the MaIN_ModelsPath: "); - envModelsPath = Console.ReadLine(); - Environment.SetEnvironmentVariable("MaIN_ModelsPath", envModelsPath); + if (string.IsNullOrEmpty(modelPathArg)) + { + Console.WriteLine("Error: A model path must be provided using --path when a local model is specified."); + return; + } + + var envModelsPath = Environment.GetEnvironmentVariable("MaIN_ModelsPath"); + if (string.IsNullOrEmpty(envModelsPath)) + { + Console.Write("Please enter the MaIN_ModelsPath: "); + envModelsPath = Console.ReadLine(); + Environment.SetEnvironmentVariable("MaIN_ModelsPath", envModelsPath); + } } } else { Console.WriteLine("No model argument provided. Continuing without model configuration."); } - - if (backendArg != null) - { - var apiKeyVariable = ""; - var apiName = ""; - - switch (backendArg.ToLower()) - { - case "openai": - Utils.OpenAi = true; - apiKeyVariable = LLMApiRegistry.OpenAi.ApiKeyEnvName; - apiName = LLMApiRegistry.OpenAi.ApiName; - break; - - case "gemini": - Utils.Gemini = true; - apiKeyVariable = LLMApiRegistry.Gemini.ApiKeyEnvName; - apiName = LLMApiRegistry.Gemini.ApiName; - break; - - case "deepseek": - Utils.DeepSeek = true; - apiKeyVariable = LLMApiRegistry.Deepseek.ApiKeyEnvName; - apiName = LLMApiRegistry.Deepseek.ApiName; - break; - - case "groqcloud": - Utils.GroqCloud = true; - apiKeyVariable = LLMApiRegistry.Groq.ApiKeyEnvName; - apiName = LLMApiRegistry.Groq.ApiName; - break; - - case "anthropic": - Utils.Anthropic = true; - apiKeyVariable = LLMApiRegistry.Anthropic.ApiKeyEnvName; - apiName = LLMApiRegistry.Anthropic.ApiName; - break; - - case "xai": - Utils.Xai = true; - apiKeyVariable = LLMApiRegistry.Xai.ApiKeyEnvName; - apiName = LLMApiRegistry.Xai.ApiName; - break; - - case "ollama": - Utils.Ollama = true; - apiKeyVariable = LLMApiRegistry.Ollama.ApiKeyEnvName; - apiName = LLMApiRegistry.Ollama.ApiName; - break; - } - - var key = Environment.GetEnvironmentVariable(apiKeyVariable); - if (string.IsNullOrEmpty(key) && !string.IsNullOrEmpty(apiName) && !string.IsNullOrEmpty(apiKeyVariable)) - { - Console.Write($"Please enter your {apiName} API key: "); - key = Console.ReadLine(); - Environment.SetEnvironmentVariable(apiKeyVariable, key); - } - } } catch (Exception ex) { @@ -107,63 +88,21 @@ return; } -if (Utils.OpenAi) +if (Utils.BackendType != BackendType.Self) { builder.Services.AddMaIN(builder.Configuration, settings => { - settings.BackendType = BackendType.OpenAi; - }); -} -else if (Utils.Gemini) -{ - builder.Services.AddMaIN(builder.Configuration, settings => - { - settings.BackendType = BackendType.Gemini; - }); -} -else if (Utils.DeepSeek) -{ - builder.Services.AddMaIN(builder.Configuration, settings => - { - settings.BackendType = BackendType.DeepSeek; - }); -} -else if (Utils.GroqCloud) -{ - builder.Services.AddMaIN(builder.Configuration, settings => - { - settings.BackendType = BackendType.GroqCloud; - }); -} -else if(Utils.Anthropic) -{ - builder.Services.AddMaIN(builder.Configuration, settings => - { - settings.BackendType = BackendType.Anthropic; - }); -} -else if (Utils.Xai) -{ - builder.Services.AddMaIN(builder.Configuration, settings => - { - settings.BackendType = BackendType.Xai; - }); -} -else if (Utils.Ollama) -{ - builder.Services.AddMaIN(builder.Configuration, settings => - { - settings.BackendType = BackendType.Ollama; + settings.BackendType = Utils.BackendType; }); } else { - if (Utils.Path is null && !ModelRegistry.Exists(Utils.Model!)) + if (Utils.Path == null && !ModelRegistry.Exists(Utils.Model!)) { Console.WriteLine($"Model: {Utils.Model} is not supported"); Environment.Exit(0); } - + builder.Services.AddMaIN(builder.Configuration); } @@ -183,5 +122,4 @@ app.MapRazorComponents() .AddInteractiveServerRenderMode(); - app.Run(); diff --git a/src/MaIN.InferPage/Properties/launchSettings.json b/src/MaIN.InferPage/Properties/launchSettings.json index c42ec2a7..89082b7a 100644 --- a/src/MaIN.InferPage/Properties/launchSettings.json +++ b/src/MaIN.InferPage/Properties/launchSettings.json @@ -14,7 +14,7 @@ "commandName": "Project", "dotnetRunMessages": true, "launchBrowser": true, - "applicationUrl": "https://localhost:7113;http://localhost:5555", + "applicationUrl": "https://localhost:7113", "environmentVariables": { "ASPNETCORE_ENVIRONMENT": "Development" } diff --git a/src/MaIN.InferPage/Utils.cs b/src/MaIN.InferPage/Utils.cs index c00c6f59..7782b764 100644 --- a/src/MaIN.InferPage/Utils.cs +++ b/src/MaIN.InferPage/Utils.cs @@ -1,25 +1,42 @@ +using MaIN.Domain.Configuration; using MaIN.Domain.Entities; +using MaIN.Domain.Models.Abstract; namespace MaIN.InferPage; public static class Utils { - public static string? Model = "gemma2:2b"; - public static bool Visual => VisualModels.Contains(Model); - private static readonly string[] VisualModels = ["FLUX.1_Shnell", "FLUX.1", "dall-e-3", "dall-e", "imagen", "imagen-3"]; //user might type different names - public static bool OpenAi { get; set; } - public static bool Gemini { get; set; } - public static bool DeepSeek { get; set; } - public static bool GroqCloud { get; set; } - public static bool Anthropic { get; set; } - public static bool Xai { get; set; } - public static bool Ollama { get; set; } + public static BackendType BackendType { get; set; } = BackendType.Self; + public static bool HasApiKey { get; set; } public static string? Path { get; set; } + public static bool IsLocal => BackendType == BackendType.Self || (BackendType == BackendType.Ollama && !HasApiKey); + public static string? Model = "gemma3-4b"; public static bool Reason { get; set; } + public static bool Visual + { + get + { + if (string.IsNullOrEmpty(Model)) return false; + if (ModelRegistry.TryGetById(Model, out var m)) + return m is IImageGenerationModel; + return ImageGenerationModels.Contains(Model); // fallback for unregistered models (e.g. FLUX via separate server) + } + } + + private static readonly HashSet ImageGenerationModels = + [ + "FLUX.1_Shnell", "FLUX.1", + "dall-e-3", "dall-e", + "gpt-image-1", + "imagen", "imagen-3", "imagen-4", "imagen-4-fast", + "grok-2-image" + ]; } public class MessageExt { public required Message Message { get; set; } public bool ShowReason { get; set; } + public List AttachedFiles { get; set; } = new(); + public List<(string Name, string Base64)> AttachedImages { get; set; } = new(); } diff --git a/src/MaIN.InferPage/wwwroot/app.css b/src/MaIN.InferPage/wwwroot/app.css index f2f5dd6a..eb29837b 100644 --- a/src/MaIN.InferPage/wwwroot/app.css +++ b/src/MaIN.InferPage/wwwroot/app.css @@ -1,5 +1,28 @@ @import url('https://fonts.googleapis.com/css2?family=Tiny5&display=swap'); /* Importing a coding font */ +/* Kolor akcentu ciemny motyw */ +body[data-theme="dark"], +body[data-theme="system-dark"] { + --accent-base-color: #00ffcc !important; + --accent-fill-rest: #00ffcc !important; +} + +/* Lekko szare tło + kolor akcentu jasny motyw */ +body[data-theme="light"], +body[data-theme="system-light"], +body:not([data-theme="dark"]):not([data-theme="system-dark"]) { + --accent-base-color: #00cca3 !important; + --accent-fill-rest: #00cca3 !important; + --light-bg: #f0f0f0; + --neutral-layer-1: var(--light-bg) !important; + --neutral-fill-layer-rest: var(--light-bg) !important; + --neutral-fill-input-rest: var(--light-bg) !important; + --neutral-layer-2: #e8e8e8 !important; + --neutral-layer-3: #e0e0e0 !important; + --neutral-layer-4: #d8d8d8 !important; + --neutral-fill-input-hover: #ebebeb !important; + --neutral-fill-input-active: #e8e8e8 !important; +} body { --body-font: "Segoe UI Variable", "Segoe UI", sans-serif; @@ -210,8 +233,8 @@ code { font-weight: 400; font-size: 45px; font-style: normal; - color: var(--accent-fill-rest); - text-shadow: 0 0 5px #00ffcc; + color: var(--accent-base-color); + text-shadow: 0 0 5px var(--accent-base-color); } .navbar { diff --git a/src/MaIN.InferPage/wwwroot/editor.js b/src/MaIN.InferPage/wwwroot/editor.js new file mode 100644 index 00000000..485afa0c --- /dev/null +++ b/src/MaIN.InferPage/wwwroot/editor.js @@ -0,0 +1,118 @@ +window.editorManager = { + getInnerText: (element) => { + return element.innerText; + }, + clearContent: (element) => { + element.innerText = ""; + }, + clickElement: (id) => { + const el = document.getElementById(id); + if (el) el.click(); + }, + attachPasteHandler: (element, dotNetHelper) => { + // Handle paste only + element.addEventListener('paste', async (e) => { + let imageFile = null; + + if (e.clipboardData?.files?.length > 0) { + for (const file of e.clipboardData.files) { + if (file.type.startsWith('image/')) { + imageFile = file; + break; + } + } + } + + if (!imageFile && e.clipboardData?.items) { + for (const item of e.clipboardData.items) { + if (item.type.startsWith('image/')) { + imageFile = item.getAsFile(); + break; + } + } + } + + if (!imageFile) return; + + e.preventDefault(); + await editorManager._processFile(imageFile, dotNetHelper); + }); + }, + attachDropZone: (containerId, dotNetHelper) => { + const container = document.getElementById(containerId); + if (!container) return; + + let dragCounter = 0; + + container.addEventListener('dragenter', async (e) => { + e.preventDefault(); + e.stopPropagation(); + dragCounter++; + if (dragCounter === 1) { + await dotNetHelper.invokeMethodAsync('OnDragEnter'); + } + }); + + container.addEventListener('dragleave', async (e) => { + e.preventDefault(); + e.stopPropagation(); + dragCounter--; + if (dragCounter === 0) { + await dotNetHelper.invokeMethodAsync('OnDragLeave'); + } + }); + + container.addEventListener('dragover', (e) => { + e.preventDefault(); + e.stopPropagation(); + }); + + container.addEventListener('drop', async (e) => { + e.preventDefault(); + e.stopPropagation(); + dragCounter = 0; + + const files = e.dataTransfer?.files; + if (!files || files.length === 0) { + await dotNetHelper.invokeMethodAsync('OnDragLeave'); + return; + } + + for (const file of files) { + await editorManager._processFile(file, dotNetHelper); + } + + // Ensure overlay is always dismissed after drop + try { await dotNetHelper.invokeMethodAsync('OnDragLeave'); } catch {} + }); + }, + _processFile: async (file, dotNetHelper) => { + try { + const base64 = await new Promise((resolve, reject) => { + const reader = new FileReader(); + reader.onload = () => resolve(reader.result.split(',')[1]); + reader.onerror = () => reject(reader.error); + reader.readAsDataURL(file); + }); + + let extension = ''; + const lastDot = file.name.lastIndexOf('.'); + if (lastDot > 0) { + extension = file.name.substring(lastDot); + } else if (file.type) { + extension = '.' + file.type.split('/')[1].replace('jpeg', 'jpg'); + } + + const fileName = file.name || `file-${Date.now()}${extension}`; + + await dotNetHelper.invokeMethodAsync('OnFileReceived', fileName, extension, base64); + } catch (err) { + try { await dotNetHelper.invokeMethodAsync('OnDragLeave'); } catch {} + } + }, + copyImageToClipboard: async (base64) => { + const res = await fetch(`data:image/png;base64,${base64}`); + const blob = await res.blob(); + await navigator.clipboard.write([new ClipboardItem({ 'image/png': blob })]); + } +}; diff --git a/src/MaIN.InferPage/wwwroot/home.css b/src/MaIN.InferPage/wwwroot/home.css index f1101655..f9df84df 100644 --- a/src/MaIN.InferPage/wwwroot/home.css +++ b/src/MaIN.InferPage/wwwroot/home.css @@ -1,19 +1,248 @@ -.input-container { - width: 100%; +body { + height: 100vh; display: flex; - padding: 5px; + flex-direction: column; + overflow: hidden; +} + +/* Main chat container */ +.chat-container { + height: 100%; + display: flex; + flex-direction: column; + flex-grow: 1; + gap: 4px; + padding-top: 4px; + position: relative; +} + +/* Drop overlay */ +.drop-overlay { + position: absolute; + top: 0; + left: 0; + right: 0; + bottom: 0; + background: rgba(0, 204, 163, 0.15); + border: 3px dashed var(--accent-base-color); + border-radius: 8px; + z-index: 1000; + display: flex; + align-items: center; + justify-content: center; + pointer-events: none; + backdrop-filter: blur(2px); +} + +.drop-overlay-content { + display: flex; + flex-direction: column; + align-items: center; + gap: 12px; + padding: 24px 48px; + background: var(--neutral-layer-1); + border-radius: 12px; + box-shadow: 0 4px 20px rgba(0, 0, 0, 0.2); +} + +.drop-overlay-content span { + font-size: 18px; + font-weight: 500; + color: var(--accent-base-color); +} + +.messages-container>div { + font-size: 72px; +} + +.navbar { +} + +.content { + flex-grow: 1; + display: flex; + flex-direction: column; + overflow: hidden; } .messages-container { flex-grow: 1; overflow-y: auto; padding: 10px; - max-height: 80vh; - min-height: 80vh; display: flex; flex-direction: column; } +.chat-input-section { + display: flex; + flex-direction: column; + margin: 10px 15px 20px 15px; +} + +.input-container { + display: flex; + padding: 8px 12px; + align-items: flex-end; + gap: 8px; + background-color: var(--neutral-fill-input-rest); + border-radius: 26px; + border: 1px solid var(--neutral-stroke-rest); + position: relative; + z-index: 2; +} + +.attachment-wrapper { + position: relative; + flex-shrink: 0; + cursor: pointer; +} + +.file-input-overlay { + position: absolute; + top: 0; + left: 0; + width: 100%; + height: 100%; + opacity: 0; + cursor: pointer; + z-index: 10; +} + +.attachment-button { + flex-shrink: 0; + margin-bottom: 2px; +} + +.input-content-wrapper { + flex-grow: 1; + min-width: 0; + display: flex; + flex-direction: column; +} + +.selected-attachments-container { + display: flex; + flex-wrap: wrap; + gap: 8px; + padding: 4px 12px; + margin-bottom: 2px; + width: 100%; + align-items: flex-end; +} + +/* Image preview thumbnail */ +.image-preview { + position: relative; + width: 48px; + height: 48px; + border-radius: 6px; + overflow: hidden; + border: 1px solid var(--neutral-stroke-rest); + background-color: var(--neutral-layer-1); +} + +.image-preview img { + width: 100%; + height: 100%; + object-fit: cover; +} + +.image-dismiss-button { + position: absolute; + top: 2px; + right: 2px; + width: 16px; + height: 16px; + border-radius: 50%; + display: flex; + align-items: center; + justify-content: center; + cursor: pointer; + opacity: 0.6; +} + +.image-dismiss-button:hover { + opacity: 1; +} + +.file-badge { + max-width: 200px; + height: 26px; + border-radius: 5px; + background-color: var(--neutral-layer-1); + border: 1px solid var(--neutral-stroke-rest); + display: flex; + align-items: center; + justify-content: space-between; + padding: 0 8px; + font-size: 12px; + color: var(--neutral-foreground-rest); + user-select: none; +} + +.file-name { + flex: 1; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + margin-right: 4px; +} + +.dismiss-button { + display: flex; + align-items: center; + justify-content: center; + cursor: pointer; + opacity: 0.6; + flex-shrink: 0; +} + +.dismiss-button:hover { + opacity: 1; + color: var(--error-foreground-rest); +} + +.chat-input { + width: 100%; + min-height: 24px; + max-height: 50vh; + overflow-y: auto; + white-space: pre-wrap; + word-break: break-word; + outline: none; + color: var(--neutral-foreground-rest); + font-size: 1rem; + line-height: 1.5; + padding: 4px 0; +} + +.chat-input[contenteditable]:empty::before { + content: attr(data-placeholder); + color: var(--neutral-foreground-hint); + pointer-events: none; + display: block; +} + +.chat-input[contenteditable][disabled="true"] { + opacity: 0.5; + cursor: not-allowed; +} + +.send-button { + flex-shrink: 0; + margin-bottom: 2px; +} + +.stop-button { + flex-shrink: 0; + margin-bottom: 2px; + color: var(--error-foreground-rest); +} + +.stop-button:hover { + color: var(--error-fill-hover); +} + .message-role-bot { align-self: flex-start; margin-bottom: 5px; @@ -48,6 +277,37 @@ font-size: smaller !important; } +.reasoning-row { + display: flex; + flex-direction: row; + align-items: flex-start; + gap: 8px; + margin-bottom: 6px; +} + +.brain-toggle { + display: inline-flex; + align-items: center; + flex-shrink: 0; + margin-top: 10px; + cursor: pointer; + opacity: 0.85; + transition: opacity 0.15s; +} + +.brain-toggle:hover { + opacity: 1; +} + +.reasoning-text { + flex: 1; + min-width: 0; +} + +.reasoning-hr { + margin: 6px 0; +} + .message-card-img { margin-bottom: 15px; width: 80%; @@ -59,6 +319,45 @@ margin: 0; } +.attached-files-display { + display: flex; + flex-wrap: wrap; + gap: 6px; + margin-bottom: 10px; + padding-bottom: 8px; + border-bottom: 1px solid var(--neutral-stroke-rest); + align-items: center; +} + +/* History image preview (in sent messages) */ +.history-image-preview { + width: 40px; + height: 40px; + border-radius: 4px; + overflow: hidden; + border: 1px solid var(--neutral-stroke-rest); + background-color: var(--neutral-layer-1); + flex-shrink: 0; +} + +.history-image-preview img { + width: 100%; + height: 100%; + object-fit: cover; +} + +.attached-file-tag { + display: inline-flex; + align-items: center; + gap: 4px; + padding: 4px 8px; + background-color: var(--neutral-fill-secondary-rest); + border: 1px solid var(--neutral-stroke-rest); + border-radius: 4px; + font-size: 11px; + color: var(--neutral-foreground-rest); +} + .error-notification-wrapper { position: fixed; top: 30px; @@ -80,7 +379,6 @@ color: #ffffff !important; border: none !important; font-size: 1.1rem !important; - overflow-wrap: break-word !important; word-wrap: break-word !important; word-break: break-word !important; @@ -107,4 +405,59 @@ 20%, 80% { transform: translate3d(2px, 0, 0); } 30%, 50%, 70% { transform: translate3d(-4px, 0, 0); } 40%, 60% { transform: translate3d(4px, 0, 0); } -} \ No newline at end of file +} + +/* Generated image display */ +.generated-images { + display: flex; + flex-direction: column; + gap: 12px; + width: 100%; +} + +.image-wrapper { + position: relative; + width: 100%; +} + +/* Fix: is inline by default — make it block so width: 100% on works correctly */ +.image-wrapper > a { + display: block; + width: 100%; +} + +.generated-image { + width: 100%; + height: auto; + border-radius: 8px; + display: block; + cursor: zoom-in; +} + +/* Action buttons overlaid on bottom-left corner of the image */ +.image-actions { + display: flex; + gap: 6px; + position: absolute; + bottom: 10px; + left: 10px; +} + +.image-action-btn { + display: inline-flex; + align-items: center; + justify-content: center; + padding: 8px; + border-radius: 8px; + opacity: 0.85; + transition: opacity 0.15s, background-color 0.15s; + color: var(--accent-base-color); + text-decoration: none; + cursor: pointer; + background-color: rgba(0, 0, 0, 0.4); +} + +.image-action-btn:hover { + opacity: 1; + background-color: rgba(0, 0, 0, 0.6); +} diff --git a/src/MaIN.InferPage/wwwroot/scroll.js b/src/MaIN.InferPage/wwwroot/scroll.js index a2bc21a9..f05bfbb9 100644 --- a/src/MaIN.InferPage/wwwroot/scroll.js +++ b/src/MaIN.InferPage/wwwroot/scroll.js @@ -1,16 +1,25 @@ window.scrollManager = { - isUserScrolling: false, + userScrolledUp: false, + isProgrammaticScroll: false, + _savedScrollTop: null, saveScrollPosition: (containerId) => { const container = document.getElementById(containerId); if (!container) return; - sessionStorage.setItem("scrollTop", container.scrollTop); + window.scrollManager._savedScrollTop = container.scrollTop; + container.style.overflowY = 'hidden'; }, restoreScrollPosition: (containerId) => { const container = document.getElementById(containerId); if (!container) return; - container.scrollTop = 9999; + if (window.scrollManager._savedScrollTop !== null) { + container.scrollTop = window.scrollManager._savedScrollTop; + window.scrollManager._savedScrollTop = null; + } else { + container.scrollTop = container.scrollHeight; + } + container.style.overflowY = ''; }, isAtBottom: (containerId) => { @@ -19,24 +28,35 @@ window.scrollManager = { return container.scrollHeight - container.scrollTop <= container.clientHeight + 50; }, - scrollToBottomSmooth: (bottomElement) => { - if (!bottomElement) return; - if (!window.scrollManager.isUserScrolling) { - bottomElement.scrollIntoView({ behavior: 'smooth' }); - } + scrollToBottomSmooth: (containerId) => { + if (window.scrollManager.userScrolledUp) return; + const container = document.getElementById(containerId); + if (!container) return; + window.scrollManager.isProgrammaticScroll = true; + container.scrollTop = container.scrollHeight; + window.scrollManager.isProgrammaticScroll = false; }, attachScrollListener: (containerId) => { const container = document.getElementById(containerId); if (!container) return; + container.addEventListener("wheel", (e) => { + if (e.deltaY < 0) { + window.scrollManager.userScrolledUp = true; + } + }); + + container.addEventListener("touchmove", () => { + window.scrollManager.userScrolledUp = true; + }); + container.addEventListener("scroll", () => { - window.scrollManager.isUserScrolling = - container.scrollHeight - container.scrollTop > container.clientHeight + 50; + if (window.scrollManager.isProgrammaticScroll) return; + const atBottom = container.scrollHeight - container.scrollTop <= container.clientHeight + 50; + if (atBottom) { + window.scrollManager.userScrolledUp = false; + } }); } }; - -document.addEventListener("DOMContentLoaded", () => { - window.scrollManager.attachScrollListener("bottom"); -}); diff --git a/src/MaIN.Services/Services/Abstract/IChatService.cs b/src/MaIN.Services/Services/Abstract/IChatService.cs index c5ffb5f6..0bbd013f 100644 --- a/src/MaIN.Services/Services/Abstract/IChatService.cs +++ b/src/MaIN.Services/Services/Abstract/IChatService.cs @@ -11,7 +11,8 @@ Task Completions( Chat chat, bool translatePrompt = false, bool interactiveUpdates = false, - Func? changeOfValue = null); + Func? changeOfValue = null, + CancellationToken cancellationToken = default); Task Delete(string id); Task GetById(string id); Task> GetAll(); diff --git a/src/MaIN.Services/Services/ChatService.cs b/src/MaIN.Services/Services/ChatService.cs index 2003b291..7fa7853d 100644 --- a/src/MaIN.Services/Services/ChatService.cs +++ b/src/MaIN.Services/Services/ChatService.cs @@ -30,13 +30,14 @@ public async Task Completions( Chat chat, bool translate = false, bool interactiveUpdates = false, - Func? changeOfValue = null) + Func? changeOfValue = null, + CancellationToken cancellationToken = default) { if (chat.ModelId == ImageGenService.LocalImageModels.FLUX) { chat.Visual = true; // TODO: add IImageGenModel interface and check for that instead } - chat.Backend ??= settings.BackendType; + chat.Backend = settings.BackendType; chat.Messages.Where(x => x.Type == MessageType.NotSet).ToList() .ForEach(x => x.Type = chat.Backend != BackendType.Self ? MessageType.CloudLLM : MessageType.LocalLLM); @@ -59,13 +60,13 @@ public async Task Completions( }))]; } - var result = chat.Visual - ? await imageGenServiceFactory.CreateService(chat.Backend.Value)!.Send(chat) + var result = chat.Visual + ? await imageGenServiceFactory.CreateService(chat.Backend.Value)!.Send(chat) : await llmServiceFactory.CreateService(chat.Backend.Value).Send(chat, new ChatRequestOptions() { InteractiveUpdates = interactiveUpdates, TokenCallback = changeOfValue - }); + }, cancellationToken); if (translate) { diff --git a/src/MaIN.Services/Services/ImageGenServices/GeminiImageGenService.cs b/src/MaIN.Services/Services/ImageGenServices/GeminiImageGenService.cs index dd223e6b..bae97232 100644 --- a/src/MaIN.Services/Services/ImageGenServices/GeminiImageGenService.cs +++ b/src/MaIN.Services/Services/ImageGenServices/GeminiImageGenService.cs @@ -7,6 +7,7 @@ using System.Net.Http.Json; using System.Text.Json.Serialization; using MaIN.Domain.Exceptions; +using MaIN.Domain.Models.Concrete; using MaIN.Services.Services.LLMService.Utils; namespace MaIN.Services.Services.ImageGenServices; diff --git a/src/MaIN.Services/Services/ImageGenServices/ImageGenDalleService.cs b/src/MaIN.Services/Services/ImageGenServices/ImageGenDalleService.cs index 066bdafd..96c66cbb 100644 --- a/src/MaIN.Services/Services/ImageGenServices/ImageGenDalleService.cs +++ b/src/MaIN.Services/Services/ImageGenServices/ImageGenDalleService.cs @@ -3,6 +3,7 @@ using MaIN.Domain.Configuration; using MaIN.Domain.Entities; using MaIN.Domain.Exceptions; +using MaIN.Domain.Models.Concrete; using MaIN.Services.Constants; using MaIN.Services.Services.Abstract; using MaIN.Services.Services.LLMService.Utils; diff --git a/src/MaIN.Services/Services/ImageGenServices/XaiImageGenService.cs b/src/MaIN.Services/Services/ImageGenServices/XaiImageGenService.cs index 3b931b44..bf10ff53 100644 --- a/src/MaIN.Services/Services/ImageGenServices/XaiImageGenService.cs +++ b/src/MaIN.Services/Services/ImageGenServices/XaiImageGenService.cs @@ -7,6 +7,7 @@ using System.Net.Http.Json; using System.Text.Json; using MaIN.Domain.Exceptions; +using MaIN.Domain.Models.Concrete; using MaIN.Services.Services.LLMService.Utils; namespace MaIN.Services.Services.ImageGenServices; diff --git a/src/MaIN.Services/Services/LLMService/AnthropicService.cs b/src/MaIN.Services/Services/LLMService/AnthropicService.cs index 03724568..bbf1d879 100644 --- a/src/MaIN.Services/Services/LLMService/AnthropicService.cs +++ b/src/MaIN.Services/Services/LLMService/AnthropicService.cs @@ -12,6 +12,7 @@ using MaIN.Domain.Configuration; using MaIN.Domain.Entities.Tools; using MaIN.Domain.Exceptions; +using MaIN.Domain.Models.Concrete; using MaIN.Services.Services.LLMService.Utils; namespace MaIN.Services.Services.LLMService; @@ -25,6 +26,7 @@ public sealed class AnthropicService( { private readonly MaINSettings _settings = settings ?? throw new ArgumentNullException(nameof(settings)); + private static readonly HashSet AnthropicImageExtensions = [".jpg", ".jpeg", ".png", ".gif", ".webp"]; private static readonly ConcurrentDictionary> SessionCache = new(); private const string CompletionsUrl = ServiceConstants.ApiUrls.AnthropicChatMessages; @@ -63,12 +65,14 @@ private void ValidateApiKey() return null; var apiKey = GetApiKey(); + + var lastMessage = chat.Messages.Last(); + await ExtractImageFromFiles(lastMessage); + var conversation = GetOrCreateConversation(chat, options.CreateSession); var resultBuilder = new StringBuilder(); var tokens = new List(); - var lastMessage = chat.Messages.Last(); - if (HasFiles(lastMessage)) { var result = ChatHelper.ExtractMemoryOptions(lastMessage); @@ -631,6 +635,42 @@ private static bool HasFiles(Message message) return message.Files != null && message.Files.Count > 0; } + private static async Task ExtractImageFromFiles(Message message) + { + if (message.Files == null || message.Files.Count == 0) + return; + + var imageFiles = message.Files + .Where(f => AnthropicImageExtensions.Contains(f.Extension.ToLowerInvariant())) + .ToList(); + + if (imageFiles.Count == 0) + return; + + var imageBytesList = new List(); + foreach (var imageFile in imageFiles) + { + if (imageFile.StreamContent != null) + { + using var ms = new MemoryStream(); + imageFile.StreamContent.Position = 0; + await imageFile.StreamContent.CopyToAsync(ms); + imageBytesList.Add(ms.ToArray()); + } + else if (imageFile.Path != null) + { + imageBytesList.Add(await File.ReadAllBytesAsync(imageFile.Path)); + } + + message.Files.Remove(imageFile); + } + + message.Images = imageBytesList; + + if (message.Files.Count == 0) + message.Files = null; + } + private async Task ProcessStreamingChatAsync( Chat chat, List conversation, diff --git a/src/MaIN.Services/Services/LLMService/DeepSeekService.cs b/src/MaIN.Services/Services/LLMService/DeepSeekService.cs index f632f138..f64eb3df 100644 --- a/src/MaIN.Services/Services/LLMService/DeepSeekService.cs +++ b/src/MaIN.Services/Services/LLMService/DeepSeekService.cs @@ -10,6 +10,7 @@ using System.Text.Json; using System.Text.Json.Serialization; using MaIN.Domain.Exceptions; +using MaIN.Domain.Models.Concrete; using MaIN.Services.Services.LLMService.Utils; namespace MaIN.Services.Services.LLMService; @@ -56,7 +57,7 @@ protected override void ValidateApiKey() CancellationToken cancellationToken = default) { var lastMsg = chat.Messages.Last(); - var filePaths = await DocumentProcessor.ConvertToFilesContent(memoryOptions); + var filePaths = await DocumentProcessor.ConvertToFilesContent(memoryOptions, cancellationToken); var message = new Message() { Role = ServiceConstants.Roles.User, @@ -66,7 +67,7 @@ protected override void ValidateApiKey() chat.Messages.Last().Content = message.Content; chat.Messages.Last().Files = []; - var result = await Send(chat, new ChatRequestOptions(), cancellationToken); + var result = await Send(chat, requestOptions, cancellationToken); chat.Messages.Last().Content = lastMsg.Content; return result; } diff --git a/src/MaIN.Services/Services/LLMService/GeminiService.cs b/src/MaIN.Services/Services/LLMService/GeminiService.cs index 5d741498..677fd85f 100644 --- a/src/MaIN.Services/Services/LLMService/GeminiService.cs +++ b/src/MaIN.Services/Services/LLMService/GeminiService.cs @@ -11,6 +11,7 @@ using MaIN.Domain.Entities; using MaIN.Domain.Exceptions; using MaIN.Domain.Models; +using MaIN.Domain.Models.Concrete; using MaIN.Services.Services.LLMService.Utils; using MaIN.Services.Utils; diff --git a/src/MaIN.Services/Services/LLMService/GroqCloudService.cs b/src/MaIN.Services/Services/LLMService/GroqCloudService.cs index c64f6593..e580780f 100644 --- a/src/MaIN.Services/Services/LLMService/GroqCloudService.cs +++ b/src/MaIN.Services/Services/LLMService/GroqCloudService.cs @@ -2,6 +2,7 @@ using MaIN.Domain.Configuration; using MaIN.Domain.Entities; using MaIN.Domain.Exceptions; +using MaIN.Domain.Models.Concrete; using MaIN.Services.Services.Abstract; using Microsoft.Extensions.Logging; using MaIN.Services.Services.LLMService.Memory; @@ -50,7 +51,7 @@ protected override void ValidateApiKey() CancellationToken cancellationToken = default) { var lastMsg = chat.Messages.Last(); - var filePaths = await DocumentProcessor.ConvertToFilesContent(memoryOptions); + var filePaths = await DocumentProcessor.ConvertToFilesContent(memoryOptions, cancellationToken); var message = new Message() { Role = ServiceConstants.Roles.User, @@ -60,7 +61,7 @@ protected override void ValidateApiKey() chat.Messages.Last().Content = message.Content; chat.Messages.Last().Files = []; - var result = await Send(chat, new ChatRequestOptions(), cancellationToken); + var result = await Send(chat, requestOptions, cancellationToken); chat.Messages.Last().Content = lastMsg.Content; return result; } diff --git a/src/MaIN.Services/Services/LLMService/LLMService.cs b/src/MaIN.Services/Services/LLMService/LLMService.cs index 88c6669f..2591e8a3 100644 --- a/src/MaIN.Services/Services/LLMService/LLMService.cs +++ b/src/MaIN.Services/Services/LLMService/LLMService.cs @@ -130,10 +130,12 @@ public Task CleanSessionCache(string? id) MemoryAnswer result; + var tokens = new List(); + if (requestOptions.InteractiveUpdates || requestOptions.TokenCallback != null) { var responseBuilder = new StringBuilder(); - + var searchOptions = new SearchOptions { Stream = true @@ -147,24 +149,27 @@ public Task CleanSessionCache(string? id) if (!string.IsNullOrEmpty(chunk.Result)) { responseBuilder.Append(chunk.Result); - + var tokenValue = new LLMTokenValue { Text = chunk.Result, Type = TokenType.Message }; - + + tokens.Add(tokenValue); + if (requestOptions.InteractiveUpdates) { await notificationService.DispatchNotification( NotificationMessageBuilder.CreateChatCompletion(chat.Id, tokenValue, false), ServiceConstants.Notifications.ReceiveMessageUpdate); } - - requestOptions.TokenCallback?.Invoke(tokenValue); + + if (requestOptions.TokenCallback != null) + await requestOptions.TokenCallback(tokenValue); } } - + result = new MemoryAnswer { Question = userMessage.Content, @@ -184,9 +189,9 @@ await notificationService.DispatchNotification( options: searchOptions, cancellationToken: cancellationToken); } - + await km.DeleteIndexAsync(cancellationToken: cancellationToken); - + if (disableCache) { llmModel.Dispose(); @@ -205,6 +210,7 @@ await notificationService.DispatchNotification( Message = new Message { Content = memoryService.CleanResponseText(result.Result), + Tokens = tokens, Role = nameof(AuthorRole.Assistant), Type = MessageType.LocalLLM, } diff --git a/src/MaIN.Services/Services/LLMService/Memory/DocumentProcessor.cs b/src/MaIN.Services/Services/LLMService/Memory/DocumentProcessor.cs index 9e7b0358..da604f56 100644 --- a/src/MaIN.Services/Services/LLMService/Memory/DocumentProcessor.cs +++ b/src/MaIN.Services/Services/LLMService/Memory/DocumentProcessor.cs @@ -31,7 +31,7 @@ public static string ProcessDocument(string filePath) }; } - public static async Task ConvertToFilesContent(ChatMemoryOptions options) + public static async Task ConvertToFilesContent(ChatMemoryOptions options, CancellationToken cancellationToken = default) { var files = new List(); foreach (var fData in options.FilesData) @@ -41,16 +41,16 @@ public static async Task ConvertToFilesContent(ChatMemoryOptions optio foreach (var sData in options.StreamData) { - var path = Path.GetTempPath() + $".{sData.Key}"; + var path = Path.Combine(Path.GetTempPath(), sData.Key); await using var fileStream = new FileStream(path, FileMode.Create, FileAccess.Write); - await sData.Value.CopyToAsync(fileStream); + await sData.Value.CopyToAsync(fileStream, cancellationToken); files.Add(path); } foreach (var txt in options.TextData) { - var path = Path.GetTempPath() + $".{txt.Key}.txt"; - await File.WriteAllTextAsync(path, txt.Value); + var path = Path.Combine(Path.GetTempPath(), $"{txt.Key}.txt"); + await File.WriteAllTextAsync(path, txt.Value, cancellationToken); files.Add(path); } @@ -60,8 +60,8 @@ public static async Task ConvertToFilesContent(ChatMemoryOptions optio foreach (var web in options.WebUrls) { var path = Path.Combine(Path.GetTempPath(), $"{Guid.NewGuid()}.html"); - var html = await client.GetStringAsync(web); - await File.WriteAllTextAsync(path, html); + var html = await client.GetStringAsync(web, cancellationToken); + await File.WriteAllTextAsync(path, html, cancellationToken); files.Add(path); } } diff --git a/src/MaIN.Services/Services/LLMService/OllamaService.cs b/src/MaIN.Services/Services/LLMService/OllamaService.cs index 7c10e8aa..74da470e 100644 --- a/src/MaIN.Services/Services/LLMService/OllamaService.cs +++ b/src/MaIN.Services/Services/LLMService/OllamaService.cs @@ -1,6 +1,7 @@ using System.Text; using MaIN.Domain.Configuration; using MaIN.Domain.Entities; +using MaIN.Domain.Models.Concrete; using MaIN.Services.Constants; using MaIN.Services.Services.Abstract; using MaIN.Services.Services.LLMService.Memory; @@ -57,7 +58,7 @@ protected override void ValidateApiKey() chat.Messages.Last().Content = message.Content; chat.Messages.Last().Files = []; - var result = await Send(chat, new ChatRequestOptions(), cancellationToken); + var result = await Send(chat, requestOptions, cancellationToken); chat.Messages.Last().Content = lastMsg.Content; return result; } diff --git a/src/MaIN.Services/Services/LLMService/OpenAiCompatibleService.cs b/src/MaIN.Services/Services/LLMService/OpenAiCompatibleService.cs index 740e4989..355e8391 100644 --- a/src/MaIN.Services/Services/LLMService/OpenAiCompatibleService.cs +++ b/src/MaIN.Services/Services/LLMService/OpenAiCompatibleService.cs @@ -28,16 +28,13 @@ public abstract class OpenAiCompatibleService( ILogger? logger = null) : ILLMService { - private readonly INotificationService _notificationService = - notificationService ?? throw new ArgumentNullException(nameof(notificationService)); - - private readonly IHttpClientFactory _httpClientFactory = - httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); + private readonly INotificationService _notificationService = notificationService ?? throw new ArgumentNullException(nameof(notificationService)); + private readonly IHttpClientFactory _httpClientFactory = httpClientFactory ?? throw new ArgumentNullException(nameof(httpClientFactory)); private static readonly ConcurrentDictionary> SessionCache = new(); + private static readonly HashSet ImageExtensions = [".jpg", ".jpeg", ".png", ".gif", ".webp", ".bmp", ".tiff", ".tif", ".heic", ".heif", ".avif"]; - private static readonly JsonSerializerOptions DefaultJsonSerializerOptions = - new() { PropertyNameCaseInsensitive = true }; + private static readonly JsonSerializerOptions DefaultJsonSerializerOptions = new() { PropertyNameCaseInsensitive = true }; private const string ToolCallsProperty = "ToolCalls"; private const string ToolCallIdProperty = "ToolCallId"; @@ -63,10 +60,11 @@ public abstract class OpenAiCompatibleService( List tokens = new(); string apiKey = GetApiKey(); + var lastMessage = chat.Messages.Last(); + await ExtractImageFromFiles(lastMessage); + List conversation = GetOrCreateConversation(chat, options.CreateSession); StringBuilder resultBuilder = new(); - - var lastMessage = chat.Messages.Last(); if (HasFiles(lastMessage)) { var result = ChatHelper.ExtractMemoryOptions(lastMessage); @@ -74,15 +72,7 @@ public abstract class OpenAiCompatibleService( resultBuilder.Append(memoryResult!.Message.Content); lastMessage.MarkProcessed(); UpdateSessionCache(chat.Id, resultBuilder.ToString(), options.CreateSession); - if (options.TokenCallback != null) - { - await InvokeTokenCallbackAsync(options.TokenCallback, new LLMTokenValue() - { - Text = resultBuilder.ToString(), - Type = TokenType.FullAnswer - }); - } - return CreateChatResult(chat, resultBuilder.ToString(), tokens); + return CreateChatResult(chat, resultBuilder.ToString(), memoryResult.Message.Tokens); } if (chat.ToolsConfiguration?.Tools != null && chat.ToolsConfiguration.Tools.Any()) @@ -454,23 +444,84 @@ await _notificationService.DispatchNotification( return null; var kernel = memoryFactory.CreateMemoryWithOpenAi(GetApiKey(), chat.MemoryParams); - await memoryService.ImportDataToMemory((kernel, null), memoryOptions, cancellationToken); - var userQuery = chat.Messages.Last().Content; + var lastMessage = chat.Messages.Last(); + var userQuery = lastMessage.Content; + if (chat.MemoryParams.Grammar != null) { var jsonGrammarConverter = new GrammarToJsonConverter(); var jsonGrammar = jsonGrammarConverter.ConvertToJson(chat.MemoryParams.Grammar); userQuery = $"{userQuery} | Respond only using the following JSON format: \n{jsonGrammar}\n. Do not add explanations, code tags, or any extra content."; } - + + // If there are images, use SearchAsync + regular chat with images + if (HasImages(lastMessage)) + { + var searchResult = await kernel.SearchAsync(userQuery, cancellationToken: cancellationToken); + await kernel.DeleteIndexAsync(cancellationToken: cancellationToken); + + // Build context from search results + var contextBuilder = new StringBuilder(); + foreach (var citation in searchResult.Results.SelectMany(r => r.Partitions)) + { + contextBuilder.AppendLine(citation.Text); + } + + // Create a temporary message with context + original query + var contextEnhancedContent = string.IsNullOrEmpty(contextBuilder.ToString()) + ? userQuery + : $"Use the following context to answer the question:\n\n{contextBuilder}\n\nQuestion: {userQuery}"; + + // Create conversation with context-enhanced message + var conversation = new List + { + new(ServiceConstants.Roles.User, contextEnhancedContent) + { + OriginalMessage = new Message + { + Role = "User", + Content = contextEnhancedContent, + Type = MessageType.CloudLLM, + Images = lastMessage.Images + } + } + }; + + var tokens = new List(); + var resultBuilder = new StringBuilder(); + + if (requestOptions.InteractiveUpdates || requestOptions.TokenCallback != null) + { + await ProcessStreamingChatAsync(chat, conversation, GetApiKey(), tokens, resultBuilder, requestOptions, cancellationToken); + } + else + { + await ProcessNonStreamingChatAsync(chat, conversation, GetApiKey(), resultBuilder, requestOptions, cancellationToken); + } + + var finalToken = new LLMTokenValue { Text = resultBuilder.ToString(), Type = TokenType.FullAnswer }; + tokens.Add(finalToken); + + if (requestOptions.InteractiveUpdates) + { + await _notificationService.DispatchNotification( + NotificationMessageBuilder.CreateChatCompletion(chat.Id, finalToken, true), + ServiceConstants.Notifications.ReceiveMessageUpdate); + } + + return CreateChatResult(chat, resultBuilder.ToString(), tokens); + } + + // No images - use standard AskAsync flow MemoryAnswer retrievedContext; + var standardTokens = new List(); if (requestOptions.InteractiveUpdates || requestOptions.TokenCallback != null) { var responseBuilder = new StringBuilder(); - + var searchOptions = new SearchOptions { Stream = true @@ -484,13 +535,15 @@ await _notificationService.DispatchNotification( if (!string.IsNullOrEmpty(chunk.Result)) { responseBuilder.Append(chunk.Result); - + var tokenValue = new LLMTokenValue { Text = chunk.Result, Type = TokenType.Message }; + standardTokens.Add(tokenValue); + if (requestOptions.InteractiveUpdates) { await notificationService.DispatchNotification( @@ -498,10 +551,10 @@ await notificationService.DispatchNotification( ServiceConstants.Notifications.ReceiveMessageUpdate); } - requestOptions.TokenCallback?.Invoke(tokenValue); + await InvokeTokenCallbackAsync(requestOptions.TokenCallback, tokenValue); } } - + retrievedContext = new MemoryAnswer { Question = userQuery, @@ -521,9 +574,9 @@ await notificationService.DispatchNotification( options: searchOptions, cancellationToken: cancellationToken); } - + await kernel.DeleteIndexAsync(cancellationToken: cancellationToken); - return CreateChatResult(chat, retrievedContext.Result, []); + return CreateChatResult(chat, retrievedContext.Result, standardTokens); } public virtual async Task GetCurrentModels() @@ -581,6 +634,42 @@ private void UpdateSessionCache(string chatId, string assistantResponse, bool cr } } + private static async Task ExtractImageFromFiles(Message message) + { + if (message.Files == null || message.Files.Count == 0) + return; + + var imageFiles = message.Files + .Where(f => ImageExtensions.Contains(f.Extension.ToLowerInvariant())) + .ToList(); + + if (imageFiles.Count == 0) + return; + + var imageBytesList = new List(); + foreach (var imageFile in imageFiles) + { + if (imageFile.StreamContent != null) + { + using var ms = new MemoryStream(); + imageFile.StreamContent.Position = 0; + await imageFile.StreamContent.CopyToAsync(ms); + imageBytesList.Add(ms.ToArray()); + } + else if (imageFile.Path != null) + { + imageBytesList.Add(await File.ReadAllBytesAsync(imageFile.Path)); + } + + message.Files.Remove(imageFile); + } + + message.Images = imageBytesList; + + if (message.Files.Count == 0) + message.Files = null; + } + private static bool HasFiles(Message message) { return message.Files != null && message.Files.Count > 0; @@ -631,6 +720,8 @@ private async Task ProcessStreamingChatAsync( while (!reader.EndOfStream) { + cancellationToken.ThrowIfCancellationRequested(); + var line = await reader.ReadLineAsync(cancellationToken); if (string.IsNullOrWhiteSpace(line)) continue; @@ -915,7 +1006,7 @@ private static async Task InvokeTokenCallbackAsync(Func? ca private static bool HasImages(Message message) { - return message.Image != null && message.Image.Length > 0; + return message.Images?.Count > 0; } private static object BuildMessageContent(Message message, ImageType imageType) @@ -936,10 +1027,10 @@ private static object BuildMessageContent(Message message, ImageType imageType) }); } - if (message.Image != null && message.Image.Length > 0) + foreach (var imageBytes in message.Images!) { - var base64Data = Convert.ToBase64String(message.Image); - var mimeType = DetectImageMimeType(message.Image); + var base64Data = Convert.ToBase64String(imageBytes); + var mimeType = DetectImageMimeType(imageBytes); switch (imageType) { @@ -979,17 +1070,17 @@ private static string DetectImageMimeType(byte[] imageBytes) if (imageBytes[0] == 0xFF && imageBytes[1] == 0xD8) return "image/jpeg"; - + if (imageBytes.Length >= 8 && imageBytes[0] == 0x89 && imageBytes[1] == 0x50 && imageBytes[2] == 0x4E && imageBytes[3] == 0x47) return "image/png"; - + if (imageBytes.Length >= 6 && imageBytes[0] == 0x47 && imageBytes[1] == 0x49 && imageBytes[2] == 0x46 && imageBytes[3] == 0x38) return "image/gif"; - + if (imageBytes.Length >= 12 && imageBytes[0] == 0x52 && imageBytes[1] == 0x49 && imageBytes[2] == 0x46 && imageBytes[3] == 0x46 && @@ -997,6 +1088,25 @@ private static string DetectImageMimeType(byte[] imageBytes) imageBytes[10] == 0x42 && imageBytes[11] == 0x50) return "image/webp"; + // HEIC/HEIF format (iPhone photos) + if (imageBytes.Length >= 12 && + imageBytes[4] == 0x66 && imageBytes[5] == 0x74 && + imageBytes[6] == 0x79 && imageBytes[7] == 0x70) + { + // Check for heic/heif brands + if ((imageBytes[8] == 0x68 && imageBytes[9] == 0x65 && imageBytes[10] == 0x69 && imageBytes[11] == 0x63) || + (imageBytes[8] == 0x68 && imageBytes[9] == 0x65 && imageBytes[10] == 0x69 && imageBytes[11] == 0x66)) + return "image/heic"; + } + + // AVIF format + if (imageBytes.Length >= 12 && + imageBytes[4] == 0x66 && imageBytes[5] == 0x74 && + imageBytes[6] == 0x79 && imageBytes[7] == 0x70 && + imageBytes[8] == 0x61 && imageBytes[9] == 0x76 && + imageBytes[10] == 0x69 && imageBytes[11] == 0x66) + return "image/avif"; + return "image/jpeg"; } } diff --git a/src/MaIN.Services/Services/LLMService/OpenAiService.cs b/src/MaIN.Services/Services/LLMService/OpenAiService.cs index e64c9fd9..461cda56 100644 --- a/src/MaIN.Services/Services/LLMService/OpenAiService.cs +++ b/src/MaIN.Services/Services/LLMService/OpenAiService.cs @@ -1,5 +1,6 @@ using MaIN.Domain.Configuration; using MaIN.Domain.Exceptions; +using MaIN.Domain.Models.Concrete; using MaIN.Services.Services.Abstract; using Microsoft.Extensions.Logging; using MaIN.Services.Services.LLMService.Memory; diff --git a/src/MaIN.Services/Services/LLMService/XaiService.cs b/src/MaIN.Services/Services/LLMService/XaiService.cs index d61f9d18..69e3ce3b 100644 --- a/src/MaIN.Services/Services/LLMService/XaiService.cs +++ b/src/MaIN.Services/Services/LLMService/XaiService.cs @@ -7,6 +7,7 @@ using Microsoft.Extensions.Logging; using System.Text; using MaIN.Domain.Exceptions; +using MaIN.Domain.Models.Concrete; using MaIN.Services.Services.LLMService.Utils; namespace MaIN.Services.Services.LLMService; @@ -49,7 +50,7 @@ protected override void ValidateApiKey() CancellationToken cancellationToken = default) { var lastMsg = chat.Messages.Last(); - var filePaths = await DocumentProcessor.ConvertToFilesContent(memoryOptions); + var filePaths = await DocumentProcessor.ConvertToFilesContent(memoryOptions, cancellationToken); var message = new Message() { Role = ServiceConstants.Roles.User, @@ -59,7 +60,7 @@ protected override void ValidateApiKey() chat.Messages.Last().Content = message.Content; chat.Messages.Last().Files = []; - var result = await Send(chat, new ChatRequestOptions(), cancellationToken); + var result = await Send(chat, requestOptions, cancellationToken); chat.Messages.Last().Content = lastMsg.Content; return result; } diff --git a/src/MaIN.Services/Services/McpService.cs b/src/MaIN.Services/Services/McpService.cs index 4521d1b4..9572b40b 100644 --- a/src/MaIN.Services/Services/McpService.cs +++ b/src/MaIN.Services/Services/McpService.cs @@ -1,5 +1,6 @@ using MaIN.Domain.Configuration; using MaIN.Domain.Entities; +using MaIN.Domain.Models.Concrete; using MaIN.Services.Services.Abstract; using MaIN.Services.Services.LLMService.Utils; using MaIN.Services.Services.Models;