From d64cd6270cd7f07e11a20046021c560a5b2c9969 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Sat, 28 Feb 2026 22:52:28 +0000 Subject: [PATCH 1/7] Replace JSON.NET with System.Text.Json across the codebase Remove all Newtonsoft.Json serialization from the application layer, replacing it with System.Text.Json (STJ). NEST still brings in Newtonsoft transitively, but all application-level serialization now uses STJ exclusively. Key changes: - Add ElasticSystemTextJsonSerializer as custom IElasticsearchSerializer for NEST - Add EmptyCollectionModifier to omit empty collections during serialization - Add ObjectToInferredTypesConverter to handle JObject/JToken from NEST reads - Add JsonNodeExtensions as STJ equivalents of JObject helpers for event upgraders - Add IJsonOnDeserialized to Event model to merge [JsonExtensionData] into Data dict - Add [JsonPropertyName] attributes to V1 webhook models for PascalCase compat - Migrate all event upgraders from JObject to JsonObject (System.Text.Json.Nodes) - Migrate all plugins from ISerializer/JsonSerializerOptions DI injection - Use case-insensitive deserialization for DataDictionary.GetValue() from JsonElement - Use semantic comparison (JsonNode.DeepEquals) in tests for fixture validation - Remove DataObjectConverter, ElasticJsonNetSerializer, and related Newtonsoft classes - Remove Foundatio.JsonNet, NEST.JsonNetSerializer, FluentRest.NewtonsoftJson packages Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- AGENTS.md | 23 + src/Exceptionless.Core/Bootstrapper.cs | 30 +- .../Exceptionless.Core.csproj | 2 +- .../Extensions/DataDictionaryExtensions.cs | 131 ++--- .../Extensions/ErrorExtensions.cs | 6 +- .../Extensions/EventExtensions.cs | 46 +- .../Extensions/JsonExtensions.cs | 266 +---------- .../Extensions/JsonNodeExtensions.cs | 447 ++++++++++++++++++ .../Extensions/PersistentEventExtensions.cs | 18 +- .../Extensions/ProjectExtensions.cs | 17 +- .../Extensions/RequestInfoExtensions.cs | 13 +- .../Jobs/CloseInactiveSessionsJob.cs | 10 +- .../Jobs/EventNotificationsJob.cs | 10 +- src/Exceptionless.Core/Jobs/EventPostsJob.cs | 10 +- src/Exceptionless.Core/Jobs/WebHooksJob.cs | 19 +- src/Exceptionless.Core/Mail/Mailer.cs | 16 +- src/Exceptionless.Core/Models/Event.cs | 49 +- .../Models/Messaging/ReleaseNotification.cs | 2 +- src/Exceptionless.Core/Models/SlackToken.cs | 38 +- src/Exceptionless.Core/Models/Stack.cs | 2 - .../Models/StackSummaryModel.cs | 2 +- src/Exceptionless.Core/Models/SummaryData.cs | 2 +- .../Default/JsonEventParserPlugin.cs | 33 +- .../Default/LegacyErrorParserPlugin.cs | 12 +- .../Default/03_ManualStackingPlugin.cs | 12 +- .../Default/0_ThrottleBotsPlugin.cs | 12 +- .../Default/10_NotFoundPlugin.cs | 12 +- .../EventProcessor/Default/20_ErrorPlugin.cs | 14 +- .../Default/30_SimpleErrorPlugin.cs | 12 +- .../Default/40_RequestInfoPlugin.cs | 16 +- .../Default/45_EnvironmentInfoPlugin.cs | 14 +- .../EventProcessor/Default/50_GeoPlugin.cs | 14 +- .../Default/70_SessionPlugin.cs | 16 +- .../Default/80_AngularPlugin.cs | 12 +- .../90_RemovePrivateInformationPlugin.cs | 12 +- .../EventUpgrader/Default/GetVersion.cs | 8 +- .../Default/V1R500_EventUpgrade.cs | 11 +- .../Default/V1R844_EventUpgrade.cs | 22 +- .../Default/V1R850_EventUpgrade.cs | 10 +- .../EventUpgrader/Default/V2_EventUpgrade.cs | 77 +-- .../EventUpgrader/EventUpgraderContext.cs | 19 +- .../05_ManualStackingFormattingPlugin.cs | 8 +- .../Default/10_SimpleErrorFormattingPlugin.cs | 22 +- .../Default/20_ErrorFormattingPlugin.cs | 22 +- .../Default/30_NotFoundFormattingPlugin.cs | 16 +- .../Default/40_UsageFormattingPlugin.cs | 10 +- .../Default/50_SessionFormattingPlugin.cs | 8 +- .../Default/60_LogFormattingPlugin.cs | 14 +- .../Default/99_DefaultFormattingPlugin.cs | 12 +- .../Formatting/FormattingPluginBase.cs | 12 +- .../WebHook/Default/005_SlackPlugin.cs | 12 +- .../WebHook/Default/010_VersionOnePlugin.cs | 64 ++- .../ExceptionlessElasticConfiguration.cs | 11 +- .../Serialization/DataObjectConverter.cs | 197 -------- .../DynamicTypeContractResolver.cs | 38 -- ...ConnectionSettingsAwareContractResolver.cs | 34 -- .../Serialization/ElasticJsonNetSerializer.cs | 41 -- .../ElasticSystemTextJsonSerializer.cs | 314 ++++++++++++ .../Serialization/EmptyCollectionModifier.cs | 66 +++ .../ExceptionlessNamingStrategy.cs | 12 - .../JsonSerializerOptionsExtensions.cs | 30 +- ...UnderscorePropertyNamesContractResolver.cs | 30 -- .../ObjectToInferredTypesConverter.cs | 41 +- .../Services/SlackService.cs | 4 +- .../Utility/ErrorSignature.cs | 10 +- .../Utility/ExtensibleObject.cs | 18 +- src/Exceptionless.Core/Utility/TypeHelper.cs | 7 +- .../Controllers/EventController.cs | 8 +- .../Controllers/ProjectController.cs | 5 +- .../Exceptionless.Web.csproj | 1 - .../Controllers/EventControllerTests.cs | 19 +- .../Exceptionless.Tests.csproj | 2 + tests/Exceptionless.Tests/Mail/MailerTests.cs | 4 +- .../Pipeline/EventPipelineTests.cs | 32 +- .../Plugins/EventParserTests.cs | 17 +- .../Plugins/EventUpgraderTests.cs | 12 +- tests/Exceptionless.Tests/Plugins/GeoTests.cs | 43 +- .../Plugins/SummaryDataTests.cs | 34 +- .../Plugins/WebHookDataTests.cs | 67 ++- .../Repositories/EventRepositoryTests.cs | 8 +- .../Repositories/ProjectRepositoryTests.cs | 7 +- .../Repositories/StackRepositoryTests.cs | 5 +- .../Serializer/Models/DataDictionaryTests.cs | 67 ++- .../Models/PersistentEventSerializerTests.cs | 19 +- .../Serializer/SerializerTests.cs | 139 +++--- .../Utility/DataBuilder.cs | 11 +- 86 files changed, 1767 insertions(+), 1283 deletions(-) create mode 100644 src/Exceptionless.Core/Extensions/JsonNodeExtensions.cs delete mode 100644 src/Exceptionless.Core/Serialization/DataObjectConverter.cs delete mode 100644 src/Exceptionless.Core/Serialization/DynamicTypeContractResolver.cs delete mode 100644 src/Exceptionless.Core/Serialization/ElasticConnectionSettingsAwareContractResolver.cs delete mode 100644 src/Exceptionless.Core/Serialization/ElasticJsonNetSerializer.cs create mode 100644 src/Exceptionless.Core/Serialization/ElasticSystemTextJsonSerializer.cs create mode 100644 src/Exceptionless.Core/Serialization/EmptyCollectionModifier.cs delete mode 100644 src/Exceptionless.Core/Serialization/ExceptionlessNamingStrategy.cs delete mode 100644 src/Exceptionless.Core/Serialization/LowerCaseUnderscorePropertyNamesContractResolver.cs diff --git a/AGENTS.md b/AGENTS.md index 30d6e030e3..95e1361746 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -82,3 +82,26 @@ pr-reviewer → security pre-screen (before build!) → dependency audit - Never commit secrets — use environment variables - NuGet feeds are in `NuGet.Config` — don't add sources - Prefer additive documentation updates — don't replace strategic docs wholesale, extend them + +## Serialization Architecture + +The project uses **System.Text.Json (STJ)** exclusively. NEST still brings in Newtonsoft.Json transitively, but all application-level serialization uses STJ: + +| Component | Serializer | Notes | +| -------------- | --------------------------------- | -------------------------------------------- | +| Elasticsearch | `ElasticSystemTextJsonSerializer` | Custom `IElasticsearchSerializer` using STJ | +| Event Upgrader | `System.Text.Json.Nodes` | JsonObject/JsonArray for mutable DOM | +| Data Storage | `SystemTextJsonSerializer` | Via Foundatio's STJ support | +| API | STJ (built-in) | ASP.NET Core default with custom options | + +**Key files:** + +- `ElasticSystemTextJsonSerializer.cs` - Custom `IElasticsearchSerializer` for NEST +- `JsonNodeExtensions.cs` - STJ equivalents of JObject helpers +- `ObjectToInferredTypesConverter.cs` - Handles JObject/JToken from NEST during STJ serialization +- `V*_EventUpgrade.cs` - Event version upgraders using JsonObject + +**Security:** + +- Safe JSON encoding used everywhere (escapes `<`, `>`, `&`, `'` for XSS protection) +- No `UnsafeRelaxedJsonEscaping` in the codebase diff --git a/src/Exceptionless.Core/Bootstrapper.cs b/src/Exceptionless.Core/Bootstrapper.cs index f264439dca..229ffc835a 100644 --- a/src/Exceptionless.Core/Bootstrapper.cs +++ b/src/Exceptionless.Core/Bootstrapper.cs @@ -24,7 +24,6 @@ using Exceptionless.Core.Services; using Exceptionless.Core.Utility; using Exceptionless.Core.Validation; -using Exceptionless.Serializer; using FluentValidation; using Foundatio.Caching; using Foundatio.Extensions.Hosting.Jobs; @@ -53,27 +52,7 @@ public class Bootstrapper { public static void RegisterServices(IServiceCollection services, AppOptions appOptions) { - // PERF: Work towards getting rid of JSON.NET. - Newtonsoft.Json.JsonConvert.DefaultSettings = () => new Newtonsoft.Json.JsonSerializerSettings - { - DateParseHandling = Newtonsoft.Json.DateParseHandling.DateTimeOffset - }; - - services.AddSingleton(_ => GetJsonContractResolver()); - services.AddSingleton(s => - { - // NOTE: These settings may need to be synced in the Elastic Configuration. - var settings = new Newtonsoft.Json.JsonSerializerSettings - { - MissingMemberHandling = Newtonsoft.Json.MissingMemberHandling.Ignore, - DateParseHandling = Newtonsoft.Json.DateParseHandling.DateTimeOffset, - ContractResolver = s.GetRequiredService() - }; - - settings.AddModelConverters(s.GetRequiredService>()); - return settings; - }); - + // Register System.Text.Json options with Exceptionless defaults (snake_case, null handling) services.AddSingleton(_ => new JsonSerializerOptions().ConfigureExceptionlessDefaults()); services.AddSingleton(s => s.GetRequiredService()); @@ -278,13 +257,6 @@ public static void AddHostedJobs(IServiceCollection services, ILoggerFactory log logger.LogWarning("Jobs running in process"); } - public static DynamicTypeContractResolver GetJsonContractResolver() - { - var resolver = new DynamicTypeContractResolver(new LowerCaseUnderscorePropertyNamesContractResolver()); - resolver.UseDefaultResolverFor(typeof(DataDictionary), typeof(SettingsDictionary), typeof(VersionOnePlugin.VersionOneWebHookStack), typeof(VersionOnePlugin.VersionOneWebHookEvent)); - return resolver; - } - private static IQueue CreateQueue(IServiceProvider container, TimeSpan? workItemTimeout = null) where T : class { var loggerFactory = container.GetRequiredService(); diff --git a/src/Exceptionless.Core/Exceptionless.Core.csproj b/src/Exceptionless.Core/Exceptionless.Core.csproj index a706da2023..55f4524012 100644 --- a/src/Exceptionless.Core/Exceptionless.Core.csproj +++ b/src/Exceptionless.Core/Exceptionless.Core.csproj @@ -24,8 +24,8 @@ + - diff --git a/src/Exceptionless.Core/Extensions/DataDictionaryExtensions.cs b/src/Exceptionless.Core/Extensions/DataDictionaryExtensions.cs index 6ebf49a9e8..acc3c7bc8b 100644 --- a/src/Exceptionless.Core/Extensions/DataDictionaryExtensions.cs +++ b/src/Exceptionless.Core/Extensions/DataDictionaryExtensions.cs @@ -1,18 +1,27 @@ using System.Text.Json; using System.Text.Json.Nodes; using Exceptionless.Core.Models; +using Foundatio.Serializer; namespace Exceptionless.Core.Extensions; public static class DataDictionaryExtensions { + /// + /// Options for deserializing JsonElement values that may use PascalCase or snake_case + /// property names. Uses case-insensitive matching without a naming policy so both formats work. + /// + private static readonly JsonSerializerOptions CaseInsensitiveOptions = new() + { + PropertyNameCaseInsensitive = true + }; /// /// Retrieves a typed value from the , deserializing if necessary. /// /// The target type to deserialize to. /// The data dictionary containing the value. /// The key of the value to retrieve. - /// The JSON serializer options to use for deserialization. + /// The text serializer to use for deserialization. /// The deserialized value, or default if deserialization fails. /// Thrown when the key is not found in the dictionary. /// @@ -20,16 +29,16 @@ public static class DataDictionaryExtensions /// /// Direct type match - returns value directly /// - extracts root element and deserializes - /// - deserializes using provided options - /// - deserializes using provided options - /// - re-serializes to JSON then deserializes (for ObjectToInferredTypesConverter output) - /// of objects - re-serializes to JSON then deserializes - /// - uses ToObject for Elasticsearch compatibility (data read from Elasticsearch uses JSON.NET) - /// JSON string - parses and deserializes + /// - extracts raw JSON and deserializes via ITextSerializer + /// - extracts JSON string and deserializes via ITextSerializer + /// - re-serializes to JSON then deserializes via ITextSerializer + /// of objects - re-serializes to JSON then deserializes via ITextSerializer + /// - uses ToObject for Elasticsearch compatibility + /// JSON string - deserializes via ITextSerializer /// Fallback - attempts type conversion via ToType /// /// - public static T? GetValue(this DataDictionary extendedData, string key, JsonSerializerOptions options) + public static T? GetValue(this DataDictionary extendedData, string key, ITextSerializer serializer) { if (!extendedData.TryGetValue(key, out object? data)) throw new KeyNotFoundException($"Key \"{key}\" not found in the dictionary."); @@ -42,10 +51,37 @@ public static class DataDictionaryExtensions data = jsonDocument.RootElement; // JsonElement (from STJ deserialization when ObjectToInferredTypesConverter wasn't used) - if (data is JsonElement jsonElement && - TryDeserialize(jsonElement, options, out T? jsonElementResult)) + if (data is JsonElement jsonElement) { - return jsonElementResult; + try + { + // Fast-path for string type + if (typeof(T) == typeof(string)) + { + object? s = jsonElement.ValueKind switch + { + JsonValueKind.String => jsonElement.GetString(), + JsonValueKind.Number => jsonElement.GetRawText(), + JsonValueKind.True => "true", + JsonValueKind.False => "false", + JsonValueKind.Null => null, + _ => jsonElement.GetRawText() + }; + + return (T?)s; + } + + // Deserialize directly from JsonElement using case-insensitive matching. + // This handles both snake_case (from Elasticsearch) and PascalCase (from + // [JsonExtensionData] which preserves original property names). + var result = jsonElement.Deserialize(CaseInsensitiveOptions); + if (result is not null) + return result; + } + catch + { + // Ignored - fall through to next handler + } } // JsonNode (JsonObject/JsonArray/JsonValue) @@ -53,7 +89,8 @@ public static class DataDictionaryExtensions { try { - var result = jsonNode.Deserialize(options); + string jsonString = jsonNode.ToJsonString(); + var result = serializer.Deserialize(jsonString); if (result is not null) return result; } @@ -64,15 +101,18 @@ public static class DataDictionaryExtensions } // Dictionary from ObjectToInferredTypesConverter - // Re-serialize to JSON then deserialize to target type with proper naming policy + // Re-serialize to JSON then deserialize to target type via ITextSerializer if (data is Dictionary dictionary) { try { - string dictJson = JsonSerializer.Serialize(dictionary, options); - var result = JsonSerializer.Deserialize(dictJson, options); - if (result is not null) - return result; + string? dictJson = serializer.SerializeToString(dictionary); + if (dictJson is not null) + { + var result = serializer.Deserialize(dictJson); + if (result is not null) + return result; + } } catch { @@ -85,10 +125,13 @@ public static class DataDictionaryExtensions { try { - string listJson = JsonSerializer.Serialize(list, options); - var result = JsonSerializer.Deserialize(listJson, options); - if (result is not null) - return result; + string? listJson = serializer.SerializeToString(list); + if (listJson is not null) + { + var result = serializer.Deserialize(listJson); + if (result is not null) + return result; + } } catch { @@ -111,12 +154,12 @@ public static class DataDictionaryExtensions } } - // JSON string + // JSON string - deserialize via ITextSerializer if (data is string json && json.IsJson()) { try { - var result = JsonSerializer.Deserialize(json, options); + var result = serializer.Deserialize(json); if (result is not null) return result; } @@ -142,49 +185,9 @@ public static class DataDictionaryExtensions return default; } - private static bool TryDeserialize(JsonElement element, JsonSerializerOptions options, out T? result) - { - result = default; - - try - { - // Fast-path for common primitives where the element isn't an object/array - // (Deserialize also works for these, but this avoids some edge cases and allocations) - if (typeof(T) == typeof(string)) - { - object? s = element.ValueKind switch - { - JsonValueKind.String => element.GetString(), - JsonValueKind.Number => element.GetRawText(), - JsonValueKind.True => "true", - JsonValueKind.False => "false", - JsonValueKind.Null => null, - _ => element.GetRawText() - }; - - result = (T?)s; - return true; - } - - // General case - var deserialized = element.Deserialize(options); - if (deserialized is not null) - { - result = deserialized; - return true; - } - } - catch - { - // Ignored - } - - return false; - } - public static void RemoveSensitiveData(this DataDictionary extendedData) { - string[] removeKeys = extendedData.Keys.Where(k => k.StartsWith('-')).ToArray(); + string[] removeKeys = [.. extendedData.Keys.Where(k => k.StartsWith('-'))]; foreach (string key in removeKeys) extendedData.Remove(key); } diff --git a/src/Exceptionless.Core/Extensions/ErrorExtensions.cs b/src/Exceptionless.Core/Extensions/ErrorExtensions.cs index 634f1fa8fc..48ff51704d 100644 --- a/src/Exceptionless.Core/Extensions/ErrorExtensions.cs +++ b/src/Exceptionless.Core/Extensions/ErrorExtensions.cs @@ -1,6 +1,6 @@ -using System.Text.Json; using Exceptionless.Core.Models; using Exceptionless.Core.Models.Data; +using Foundatio.Serializer; namespace Exceptionless.Core.Extensions; @@ -59,9 +59,9 @@ public static StackingTarget GetStackingTarget(this Error error) }; } - public static StackingTarget? GetStackingTarget(this Event ev, JsonSerializerOptions options) + public static StackingTarget? GetStackingTarget(this Event ev, ITextSerializer serializer) { - var error = ev.GetError(options); + var error = ev.GetError(serializer); return error?.GetStackingTarget(); } diff --git a/src/Exceptionless.Core/Extensions/EventExtensions.cs b/src/Exceptionless.Core/Extensions/EventExtensions.cs index 0797660f87..261b8a0262 100644 --- a/src/Exceptionless.Core/Extensions/EventExtensions.cs +++ b/src/Exceptionless.Core/Extensions/EventExtensions.cs @@ -1,9 +1,8 @@ using System.Text; -using System.Text.Json; using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Models.Data; -using Newtonsoft.Json; +using Foundatio.Serializer; namespace Exceptionless; @@ -14,14 +13,14 @@ public static bool HasError(this Event ev) return ev.Data is not null && ev.Data.ContainsKey(Event.KnownDataKeys.Error); } - public static Error? GetError(this Event ev, JsonSerializerOptions options) + public static Error? GetError(this Event ev, ITextSerializer serializer) { if (!ev.HasError()) return null; try { - return ev.Data!.GetValue(Event.KnownDataKeys.Error, options); + return ev.Data!.GetValue(Event.KnownDataKeys.Error, serializer); } catch (Exception) { @@ -36,14 +35,14 @@ public static bool HasSimpleError(this Event ev) return ev.Data is not null && ev.Data.ContainsKey(Event.KnownDataKeys.SimpleError); } - public static SimpleError? GetSimpleError(this Event ev, JsonSerializerOptions options) + public static SimpleError? GetSimpleError(this Event ev, ITextSerializer serializer) { if (!ev.HasSimpleError()) return null; try { - return ev.Data!.GetValue(Event.KnownDataKeys.SimpleError, options); + return ev.Data!.GetValue(Event.KnownDataKeys.SimpleError, serializer); } catch (Exception) { @@ -53,14 +52,14 @@ public static bool HasSimpleError(this Event ev) return null; } - public static RequestInfo? GetRequestInfo(this Event ev, JsonSerializerOptions options) + public static RequestInfo? GetRequestInfo(this Event ev, ITextSerializer serializer) { if (ev.Data is null || !ev.Data.ContainsKey(Event.KnownDataKeys.RequestInfo)) return null; try { - return ev.Data.GetValue(Event.KnownDataKeys.RequestInfo, options); + return ev.Data.GetValue(Event.KnownDataKeys.RequestInfo, serializer); } catch (Exception) { @@ -70,14 +69,14 @@ public static bool HasSimpleError(this Event ev) return null; } - public static EnvironmentInfo? GetEnvironmentInfo(this Event ev, JsonSerializerOptions options) + public static EnvironmentInfo? GetEnvironmentInfo(this Event ev, ITextSerializer serializer) { if (ev.Data is null || !ev.Data.ContainsKey(Event.KnownDataKeys.EnvironmentInfo)) return null; try { - return ev.Data.GetValue(Event.KnownDataKeys.EnvironmentInfo, options); + return ev.Data.GetValue(Event.KnownDataKeys.EnvironmentInfo, serializer); } catch (Exception) { @@ -183,14 +182,14 @@ public static void AddRequestInfo(this Event ev, RequestInfo request) /// /// Gets the user info object from extended data. /// - public static UserInfo? GetUserIdentity(this Event ev, JsonSerializerOptions options) + public static UserInfo? GetUserIdentity(this Event ev, ITextSerializer serializer) { if (ev.Data is null || !ev.Data.ContainsKey(Event.KnownDataKeys.UserInfo)) return null; try { - return ev.Data.GetValue(Event.KnownDataKeys.UserInfo, options); + return ev.Data.GetValue(Event.KnownDataKeys.UserInfo, serializer); } catch (Exception) { @@ -219,14 +218,14 @@ public static void SetVersion(this Event ev, string? version) ev.Data[Event.KnownDataKeys.Version] = version.Trim(); } - public static SubmissionClient? GetSubmissionClient(this Event ev, JsonSerializerOptions options) + public static SubmissionClient? GetSubmissionClient(this Event ev, ITextSerializer serializer) { if (ev.Data is null || !ev.Data.ContainsKey(Event.KnownDataKeys.SubmissionClient)) return null; try { - return ev.Data.GetValue(Event.KnownDataKeys.SubmissionClient, options); + return ev.Data.GetValue(Event.KnownDataKeys.SubmissionClient, serializer); } catch (Exception) { @@ -241,14 +240,14 @@ public static bool HasLocation(this Event ev) return ev.Data != null && ev.Data.ContainsKey(Event.KnownDataKeys.Location); } - public static Location? GetLocation(this Event ev, JsonSerializerOptions options) + public static Location? GetLocation(this Event ev, ITextSerializer serializer) { if (ev.Data is null || !ev.Data.ContainsKey(Event.KnownDataKeys.Location)) return null; try { - return ev.Data.GetValue(Event.KnownDataKeys.Location, options); + return ev.Data.GetValue(Event.KnownDataKeys.Location, serializer); } catch (Exception) { @@ -301,14 +300,14 @@ public static void SetEnvironmentInfo(this Event ev, EnvironmentInfo? environmen /// /// Gets the stacking info from extended data. /// - public static ManualStackingInfo? GetManualStackingInfo(this Event ev, JsonSerializerOptions options) + public static ManualStackingInfo? GetManualStackingInfo(this Event ev, ITextSerializer serializer) { if (ev.Data is null || !ev.Data.ContainsKey(Event.KnownDataKeys.ManualStackingInfo)) return null; try { - return ev.Data.GetValue(Event.KnownDataKeys.ManualStackingInfo, options); + return ev.Data.GetValue(Event.KnownDataKeys.ManualStackingInfo, serializer); } catch (Exception) { @@ -423,14 +422,14 @@ public static void RemoveUserIdentity(this Event ev) /// /// Gets the user description from extended data. /// - public static UserDescription? GetUserDescription(this Event ev, JsonSerializerOptions options) + public static UserDescription? GetUserDescription(this Event ev, ITextSerializer serializer) { if (ev.Data is null || !ev.Data.ContainsKey(Event.KnownDataKeys.UserDescription)) return null; try { - return ev.Data.GetValue(Event.KnownDataKeys.UserDescription, options); + return ev.Data.GetValue(Event.KnownDataKeys.UserDescription, serializer); } catch (Exception) { @@ -469,8 +468,11 @@ public static void SetUserDescription(this Event ev, UserDescription description ev.Data[Event.KnownDataKeys.UserDescription] = description; } - public static byte[] GetBytes(this Event ev, JsonSerializerSettings settings) + /// + /// Serializes an event to UTF-8 JSON bytes using the specified serializer. + /// + public static byte[] GetBytes(this Event ev, ITextSerializer serializer) { - return Encoding.UTF8.GetBytes(ev.ToJson(Formatting.None, settings)); + return serializer.SerializeToBytes(ev); } } diff --git a/src/Exceptionless.Core/Extensions/JsonExtensions.cs b/src/Exceptionless.Core/Extensions/JsonExtensions.cs index 7c3e95e905..f6837f37e6 100644 --- a/src/Exceptionless.Core/Extensions/JsonExtensions.cs +++ b/src/Exceptionless.Core/Extensions/JsonExtensions.cs @@ -1,155 +1,24 @@ -using System.Collections; -using System.Collections.Concurrent; -using Exceptionless.Core.Models; -using Exceptionless.Core.Models.Data; -using Exceptionless.Core.Reflection; -using Exceptionless.Serializer; -using Microsoft.Extensions.Logging; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; -using Newtonsoft.Json.Serialization; +using System.Text.Json; namespace Exceptionless.Core.Extensions; -[System.Runtime.InteropServices.GuidAttribute("4186FC77-AF28-4D51-AAC3-49055DD855A4")] +/// +/// Extension methods for JSON operations using System.Text.Json. +/// For JsonNode/JsonObject operations, see . +/// public static class JsonExtensions { - public static bool IsNullOrEmpty(this JToken target) - { - if (target is null || target.Type == JTokenType.Null) - return true; - - if (target.Type == JTokenType.Object || target.Type == JTokenType.Array) - return !target.HasValues; - - if (target.Type != JTokenType.Property) - return false; - - var value = ((JProperty)target).Value; - if (value.Type == JTokenType.String) - return value.ToString().IsNullOrEmpty(); - - return IsNullOrEmpty(value); - } - - public static bool IsPropertyNullOrEmpty(this JObject target, string name) - { - var property = target.Property(name); - if (property is null) - return true; - - return property.Value.IsNullOrEmpty(); - } - - public static bool RemoveIfNullOrEmpty(this JObject target, string name) - { - if (!target.IsPropertyNullOrEmpty(name)) - return false; - - target.Remove(name); - return true; - } - - public static void RemoveAll(this JObject target, params string[] names) - { - foreach (string name in names) - target.Remove(name); - } - - - public static bool RemoveAllIfNullOrEmpty(this JObject target, params string[] names) - { - if (target.IsNullOrEmpty()) - return false; - - var properties = target.Descendants().OfType().Where(t => names.Contains(t.Name) && t.IsNullOrEmpty()).ToList(); - foreach (var p in properties) - p.Remove(); - - return true; - } - - public static bool Rename(this JObject target, string currentName, string newName) - { - if (String.Equals(currentName, newName)) - return true; - - var property = target.Property(currentName); - if (property is null) - return false; - - property.Replace(new JProperty(newName, property.Value)); - return true; - } - - public static bool RenameOrRemoveIfNullOrEmpty(this JObject target, string currentName, string newName) - { - var property = target.Property(currentName); - if (property is null) - return false; - - bool isNullOrEmpty = target.IsPropertyNullOrEmpty(currentName); - if (isNullOrEmpty) - { - target.Remove(property.Name); - return false; - } - - property.Replace(new JProperty(newName, property.Value)); - return true; - } - - public static void MoveOrRemoveIfNullOrEmpty(this JObject target, JObject source, params string[] names) - { - foreach (string name in names) - { - var property = source.Property(name); - if (property is null) - continue; - - bool isNullOrEmpty = source.IsPropertyNullOrEmpty(name); - source.Remove(property.Name); - - if (isNullOrEmpty) - continue; - - target.Add(name, property.Value); - } - } - - public static bool RenameAll(this JObject target, string currentName, string newName) - { - var properties = target.Descendants().OfType().Where(t => t.Name == currentName).ToList(); - foreach (var p in properties) - { - if (p.Parent is JObject parent) - parent.Rename(currentName, newName); - } - - return true; - } - - public static string? GetPropertyStringValue(this JObject target, string name) - { - if (target.IsPropertyNullOrEmpty(name)) - return null; - - return target.Property(name)?.Value.ToString(); - } - - - public static string? GetPropertyStringValueAndRemove(this JObject target, string name) - { - string? value = target.GetPropertyStringValue(name); - target.Remove(name); - return value; - } - + /// + /// Checks if a string contains JSON content (starts with { or [). + /// public static bool IsJson(this string value) { return value.GetJsonType() != JsonType.None; } + /// + /// Determines the JSON type of a string (Object, Array, or None). + /// public static JsonType GetJsonType(this string value) { if (String.IsNullOrEmpty(value)) @@ -172,120 +41,7 @@ public static JsonType GetJsonType(this string value) return JsonType.None; } - public static string ToJson(this T data, Formatting formatting = Formatting.None, JsonSerializerSettings? settings = null) - { - var serializer = settings is null ? JsonSerializer.CreateDefault() : JsonSerializer.CreateDefault(settings); - serializer.Formatting = formatting; - - using (var sw = new StringWriter()) - { - serializer.Serialize(sw, data, typeof(T)); - return sw.ToString(); - } - } - - public static List? FromJson(this JArray data, JsonSerializerSettings? settings = null) - { - var serializer = settings is null ? JsonSerializer.CreateDefault() : JsonSerializer.CreateDefault(settings); - return data.ToObject>(serializer); - } - - public static T? FromJson(this string data, JsonSerializerSettings? settings = null) - { - var serializer = settings is null ? JsonSerializer.CreateDefault() : JsonSerializer.CreateDefault(settings); - - using (var sw = new StringReader(data)) - using (var sr = new JsonTextReader(sw)) - return serializer.Deserialize(sr); - } - public static bool TryFromJson(this string data, out T? value, JsonSerializerSettings? settings = null) - { - try - { - value = data.FromJson(settings); - return true; - } - catch (Exception) - { - value = default; - return false; - } - } - - private static readonly ConcurrentDictionary _countAccessors = new(); - public static bool IsValueEmptyCollection(this JsonProperty property, object target) - { - object? value = property.ValueProvider?.GetValue(target); - if (value is null) - return true; - - if (value is ICollection collection) - return collection.Count == 0; - - if (property.PropertyType is null) - return false; - - if (!_countAccessors.ContainsKey(property.PropertyType)) - { - if (typeof(IEnumerable).IsAssignableFrom(property.PropertyType)) - { - var countProperty = property.PropertyType.GetProperty("Count"); - if (countProperty is not null) - _countAccessors.AddOrUpdate(property.PropertyType, LateBinder.GetPropertyAccessor(countProperty)); - else - _countAccessors.AddOrUpdate(property.PropertyType, null); - } - else - { - _countAccessors.AddOrUpdate(property.PropertyType, null); - } - } - - var countAccessor = _countAccessors[property.PropertyType]; - if (countAccessor is null) - return false; - - int count = (int)(countAccessor.GetValue(value) ?? 0); - return count == 0; - } - - public static void AddModelConverters(this JsonSerializerSettings settings, ILogger logger) - { - var knownEventDataTypes = new Dictionary - { - { Event.KnownDataKeys.Error, typeof(Error) }, - { Event.KnownDataKeys.EnvironmentInfo, typeof(EnvironmentInfo) }, - { Event.KnownDataKeys.Location, typeof(Location) }, - { Event.KnownDataKeys.RequestInfo, typeof(RequestInfo) }, - { Event.KnownDataKeys.SimpleError, typeof(SimpleError) }, - { Event.KnownDataKeys.SubmissionClient, typeof(SubmissionClient) }, - { Event.KnownDataKeys.ManualStackingInfo, typeof(ManualStackingInfo) }, - { Event.KnownDataKeys.UserDescription, typeof(UserDescription) }, - { Event.KnownDataKeys.UserInfo, typeof(UserInfo) } - }; - - var knownProjectDataTypes = new Dictionary - { - { Project.KnownDataKeys.SlackToken, typeof(SlackToken) } - }; - - settings.Converters.Add(new DataObjectConverter(logger)); - settings.Converters.Add(new DataObjectConverter(logger, knownProjectDataTypes)); - settings.Converters.Add(new DataObjectConverter(logger, knownEventDataTypes)); - settings.Converters.Add(new DataObjectConverter(logger, knownEventDataTypes)); - settings.Converters.Add(new DataObjectConverter(logger)); - settings.Converters.Add(new DataObjectConverter(logger)); - settings.Converters.Add(new DataObjectConverter(logger)); - settings.Converters.Add(new DataObjectConverter(logger)); - settings.Converters.Add(new DataObjectConverter(logger)); - settings.Converters.Add(new DataObjectConverter(logger)); - settings.Converters.Add(new DataObjectConverter(logger)); - settings.Converters.Add(new DataObjectConverter(logger)); - settings.Converters.Add(new DataObjectConverter(logger)); - settings.Converters.Add(new DataObjectConverter(logger)); - settings.Converters.Add(new DataObjectConverter(logger)); - } } public enum JsonType : byte diff --git a/src/Exceptionless.Core/Extensions/JsonNodeExtensions.cs b/src/Exceptionless.Core/Extensions/JsonNodeExtensions.cs new file mode 100644 index 0000000000..85b6f89e17 --- /dev/null +++ b/src/Exceptionless.Core/Extensions/JsonNodeExtensions.cs @@ -0,0 +1,447 @@ +using System.Text.Encodings.Web; +using System.Text.Json; +using System.Text.Json.Nodes; +using System.Text.Unicode; + +namespace Exceptionless.Core.Extensions; + +/// +/// Extension methods for System.Text.Json.Nodes types (JsonNode, JsonObject, JsonArray). +/// Provides helper methods for JSON manipulation during event processing and upgrades. +/// +public static class JsonNodeExtensions +{ + /// + /// XSS-safe encoder for JSON output formatting. + /// This encoder ensures proper XSS protection while allowing Unicode characters + /// for internationalization support. + /// + /// Security features: + /// - HTML-sensitive characters (<, >, &) are escaped for XSS protection + /// - Single quotes are escaped as \u0027 (per ECMAScript spec) + /// - Control characters are escaped for security + /// + private static readonly JavaScriptEncoder SafeJsonEncoder = JavaScriptEncoder.Create(new TextEncoderSettings(UnicodeRanges.All)); + + /// + /// JSON options with safe XSS encoding for tests. + /// Validates that dangerous characters (<, >, &, ') are properly escaped. + /// Production code should use from DI. + /// + internal static readonly JsonSerializerOptions SafeSerializerOptions = new() + { + Encoder = SafeJsonEncoder + }; + + /// + /// Checks if a JsonNode is null or empty (no values for objects/arrays). + /// + public static bool IsNullOrEmpty(this JsonNode? target) + { + if (target is null) + return true; + + if (target is JsonObject obj) + return obj.Count == 0; + + if (target is JsonArray arr) + return arr.Count == 0; + + if (target is JsonValue val) + { + // Check for null value + if (target.GetValueKind() == JsonValueKind.Null) + return true; + + // Check for empty string + if (target.GetValueKind() == JsonValueKind.String) + { + var strValue = val.GetValue(); + return string.IsNullOrEmpty(strValue); + } + } + + return false; + } + + /// + /// Checks if a property in a JsonObject is null or empty. + /// + public static bool IsPropertyNullOrEmpty(this JsonObject target, string name) + { + if (!target.TryGetPropertyValue(name, out var value)) + return true; + + return value.IsNullOrEmpty(); + } + + /// + /// Removes a property if it is null or empty. + /// + /// True if the property was removed, false otherwise. + public static bool RemoveIfNullOrEmpty(this JsonObject target, string name) + { + if (!target.IsPropertyNullOrEmpty(name)) + return false; + + target.Remove(name); + return true; + } + + /// + /// Removes multiple properties from a JsonObject. + /// + public static void RemoveAll(this JsonObject target, params string[] names) + { + foreach (string name in names) + target.Remove(name); + } + + /// + /// Removes all properties with the given names if they are null or empty, recursively. + /// + /// True if any properties were removed, false otherwise. + public static bool RemoveAllIfNullOrEmpty(this JsonObject target, params string[] names) + { + if (target.IsNullOrEmpty()) + return false; + + bool removed = false; + var toRemove = new List<(JsonObject parent, string name)>(); + + foreach (var descendant in target.DescendantsAndSelf().OfType()) + { + foreach (var name in names) + { + if (descendant.TryGetPropertyValue(name, out var value) && value.IsNullOrEmpty()) + { + toRemove.Add((descendant, name)); + } + } + } + + foreach (var (parent, name) in toRemove) + { + parent.Remove(name); + removed = true; + } + + return removed; + } + + /// + /// Renames a property in a JsonObject while preserving property order. + /// + /// True if the property was renamed, false if not found. + public static bool Rename(this JsonObject target, string currentName, string newName) + { + if (string.Equals(currentName, newName)) + return true; + + if (!target.TryGetPropertyValue(currentName, out var value)) + return false; + + // To preserve order, we need to rebuild the object + var properties = target.ToList(); + target.Clear(); + + foreach (var prop in properties) + { + if (prop.Key == currentName) + target.Add(newName, prop.Value); + else + target.Add(prop.Key, prop.Value); + } + + return true; + } + + /// + /// Renames a property or removes it if null or empty, preserving property order. + /// + /// True if renamed, false if removed or not found. + public static bool RenameOrRemoveIfNullOrEmpty(this JsonObject target, string currentName, string newName) + { + if (!target.TryGetPropertyValue(currentName, out var value)) + return false; + + bool isNullOrEmpty = value.IsNullOrEmpty(); + if (isNullOrEmpty) + { + target.Remove(currentName); + return false; + } + + // To preserve order, we need to rebuild the object + var properties = target.ToList(); + target.Clear(); + + foreach (var prop in properties) + { + if (prop.Key == currentName) + target.Add(newName, prop.Value); + else + target.Add(prop.Key, prop.Value); + } + + return true; + } + + /// + /// Moves properties from source to target, removing if null or empty. + /// + public static void MoveOrRemoveIfNullOrEmpty(this JsonObject target, JsonObject source, params string[] names) + { + foreach (string name in names) + { + if (!source.TryGetPropertyValue(name, out var value)) + continue; + + bool isNullOrEmpty = value.IsNullOrEmpty(); + source.Remove(name); + + if (isNullOrEmpty) + continue; + + target.Add(name, value); + } + } + + /// + /// Renames all properties with the given name recursively throughout the JSON tree. + /// + public static bool RenameAll(this JsonObject target, string currentName, string newName) + { + var objectsWithProperty = target.DescendantsAndSelf() + .OfType() + .Where(o => o.ContainsKey(currentName)) + .ToList(); + + foreach (var obj in objectsWithProperty) + { + obj.Rename(currentName, newName); + } + + return true; + } + + /// + /// Gets a string value from a property, or null if not found or empty. + /// + public static string? GetPropertyStringValue(this JsonObject target, string name) + { + if (target.IsPropertyNullOrEmpty(name)) + return null; + + if (!target.TryGetPropertyValue(name, out var value)) + return null; + + return value?.ToString(); + } + + /// + /// Gets a string value from a property and removes it. + /// + public static string? GetPropertyStringValueAndRemove(this JsonObject target, string name) + { + string? value = target.GetPropertyStringValue(name); + target.Remove(name); + return value; + } + + /// + /// Enumerates all descendant nodes of a JsonNode. + /// + public static IEnumerable Descendants(this JsonNode? node) + { + if (node is null) + yield break; + + if (node is JsonObject obj) + { + foreach (var prop in obj) + { + yield return prop.Value; + if (prop.Value is not null) + { + foreach (var desc in Descendants(prop.Value)) + yield return desc; + } + } + } + else if (node is JsonArray arr) + { + foreach (var item in arr) + { + yield return item; + if (item is not null) + { + foreach (var desc in Descendants(item)) + yield return desc; + } + } + } + } + + /// + /// Enumerates the node itself and all its descendants. + /// + public static IEnumerable DescendantsAndSelf(this JsonNode? node) + { + yield return node; + foreach (var desc in Descendants(node)) + yield return desc; + } + + /// + /// Converts an object to a JsonNode using System.Text.Json serialization. + /// + public static JsonNode? ToJsonNode(T value, JsonSerializerOptions options) + { + return JsonSerializer.SerializeToNode(value, options); + } + + /// + /// Checks if a JsonNode has any values (for objects: has properties, for arrays: has items). + /// + public static bool HasValues(this JsonNode? node) + { + return !node.IsNullOrEmpty(); + } + + /// + /// Converts a JsonNode to the specified type. + /// + public static T? ToObject(this JsonNode? node, JsonSerializerOptions options) + { + if (node is null) + return default; + + return node.Deserialize(options); + } + + /// + /// Converts a JsonArray to a List of the specified type. + /// + public static List? ToList(this JsonArray? array, JsonSerializerOptions options) + { + if (array is null) + return null; + + return array.Deserialize>(options); + } + + /// + /// Creates a JsonValue from a primitive value. + /// + public static JsonValue? CreateValue(T value) + { + return JsonValue.Create(value); + } + + /// + /// Converts a JsonNode to a pretty-printed JSON string. + /// Uses 2-space indentation. Normalizes dates to match existing data format (Z → +00:00). + /// + /// The JSON node to format. + /// Serializer options from DI. Uses WriteIndented=true and IndentSize=2. + public static string ToFormattedString(this JsonNode? node, JsonSerializerOptions options) + { + if (node is null) + return "null"; + + // Normalize the node to match existing date format before serialization + NormalizeDates(node); + + return node.ToJsonString(options); + } + + /// + /// Recursively normalizes date strings from Z format to +00:00 format + /// to match Newtonsoft.Json's default date serialization behavior. + /// + private static void NormalizeDates(JsonNode? node) + { + if (node is JsonObject obj) + { + var propertiesToUpdate = new List<(string key, string newValue)>(); + + foreach (var prop in obj) + { + if (prop.Value is JsonValue val && val.GetValueKind() == System.Text.Json.JsonValueKind.String) + { + var strValue = val.GetValue(); + if (strValue != null && IsIso8601DateWithZ(strValue)) + { + // Convert Z to +00:00 to match Newtonsoft behavior + var normalized = NormalizeDateString(strValue); + if (normalized != strValue) + { + propertiesToUpdate.Add((prop.Key, normalized)); + } + } + } + else + { + NormalizeDates(prop.Value); + } + } + + foreach (var (key, newValue) in propertiesToUpdate) + { + obj[key] = JsonValue.Create(newValue); + } + } + else if (node is JsonArray arr) + { + for (int i = 0; i < arr.Count; i++) + { + if (arr[i] is JsonValue val && val.GetValueKind() == System.Text.Json.JsonValueKind.String) + { + var strValue = val.GetValue(); + if (strValue != null && IsIso8601DateWithZ(strValue)) + { + var normalized = NormalizeDateString(strValue); + if (normalized != strValue) + { + arr[i] = JsonValue.Create(normalized); + } + } + } + else + { + NormalizeDates(arr[i]); + } + } + } + } + + /// + /// Checks if a string looks like an ISO 8601 date with Z suffix. + /// + private static bool IsIso8601DateWithZ(string value) + { + // Check for pattern like "2013-09-11T14:49:54.218Z" or "2014-03-03T11:10:56Z" + return value.Length >= 20 && + value.Length <= 28 && + value.EndsWith("Z") && + value[4] == '-' && + value[7] == '-' && + value[10] == 'T' && + value[13] == ':' && + value[16] == ':'; + } + + /// + /// Normalizes a date string from Z format to +00:00 format. + /// + private static string NormalizeDateString(string value) + { + if (DateTimeOffset.TryParse(value, out var date)) + { + // Format with explicit offset + return date.ToString("yyyy-MM-ddTHH:mm:ss.FFFFFFFzzz"); + } + return value; + } +} diff --git a/src/Exceptionless.Core/Extensions/PersistentEventExtensions.cs b/src/Exceptionless.Core/Extensions/PersistentEventExtensions.cs index c0901a893f..37ed2eaeb6 100644 --- a/src/Exceptionless.Core/Extensions/PersistentEventExtensions.cs +++ b/src/Exceptionless.Core/Extensions/PersistentEventExtensions.cs @@ -1,7 +1,7 @@ -using System.Text.Json; using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Models.Data; +using Foundatio.Serializer; namespace Exceptionless; @@ -178,7 +178,7 @@ public static bool UpdateSessionStart(this PersistentEvent ev, DateTime lastActi return true; } - public static PersistentEvent ToSessionStartEvent(this PersistentEvent source, JsonSerializerOptions jsonOptions, DateTime? lastActivityUtc = null, bool? isSessionEnd = null, bool hasPremiumFeatures = true, bool includePrivateInformation = true) + public static PersistentEvent ToSessionStartEvent(this PersistentEvent source, ITextSerializer serializer, DateTime? lastActivityUtc = null, bool? isSessionEnd = null, bool hasPremiumFeatures = true, bool includePrivateInformation = true) { var startEvent = new PersistentEvent { @@ -194,11 +194,11 @@ public static PersistentEvent ToSessionStartEvent(this PersistentEvent source, J if (sessionId is not null) startEvent.SetSessionId(sessionId); if (includePrivateInformation) - startEvent.SetUserIdentity(source.GetUserIdentity(jsonOptions)); - startEvent.SetLocation(source.GetLocation(jsonOptions)); + startEvent.SetUserIdentity(source.GetUserIdentity(serializer)); + startEvent.SetLocation(source.GetLocation(serializer)); startEvent.SetVersion(source.GetVersion()); - var ei = source.GetEnvironmentInfo(jsonOptions); + var ei = source.GetEnvironmentInfo(serializer); if (ei is not null) { startEvent.SetEnvironmentInfo(new EnvironmentInfo @@ -219,7 +219,7 @@ public static PersistentEvent ToSessionStartEvent(this PersistentEvent source, J }); } - var ri = source.GetRequestInfo(jsonOptions); + var ri = source.GetRequestInfo(serializer); if (ri is not null) { startEvent.AddRequestInfo(new RequestInfo @@ -245,19 +245,19 @@ public static PersistentEvent ToSessionStartEvent(this PersistentEvent source, J return startEvent; } - public static IEnumerable GetIpAddresses(this PersistentEvent ev, JsonSerializerOptions jsonOptions) + public static IEnumerable GetIpAddresses(this PersistentEvent ev, ITextSerializer serializer) { if (!String.IsNullOrEmpty(ev.Geo) && (ev.Geo.Contains('.') || ev.Geo.Contains(':'))) yield return ev.Geo.Trim(); - var ri = ev.GetRequestInfo(jsonOptions); + var ri = ev.GetRequestInfo(serializer); if (!String.IsNullOrEmpty(ri?.ClientIpAddress)) { foreach (string ip in ri.ClientIpAddress.Split(_commaSeparator, StringSplitOptions.RemoveEmptyEntries)) yield return ip.Trim(); } - var ei = ev.GetEnvironmentInfo(jsonOptions); + var ei = ev.GetEnvironmentInfo(serializer); if (!String.IsNullOrEmpty(ei?.IpAddress)) { foreach (string ip in ei.IpAddress.Split(_commaSeparator, StringSplitOptions.RemoveEmptyEntries)) diff --git a/src/Exceptionless.Core/Extensions/ProjectExtensions.cs b/src/Exceptionless.Core/Extensions/ProjectExtensions.cs index 3f14a0fb30..930fa466b4 100644 --- a/src/Exceptionless.Core/Extensions/ProjectExtensions.cs +++ b/src/Exceptionless.Core/Extensions/ProjectExtensions.cs @@ -1,6 +1,7 @@ using System.Text; using Exceptionless.Core.Models; using Exceptionless.DateTimeExtensions; +using Foundatio.Serializer; namespace Exceptionless.Core.Extensions; @@ -48,9 +49,21 @@ public static string BuildFilter(this IList projects) /// /// Gets the slack token from extended data. /// - public static SlackToken? GetSlackToken(this Project project) + public static SlackToken? GetSlackToken(this Project project, ITextSerializer serializer) { - return project.Data is not null && project.Data.TryGetValue(Project.KnownDataKeys.SlackToken, out object? value) ? value as SlackToken : null; + if (project.Data is null || !project.Data.ContainsKey(Project.KnownDataKeys.SlackToken)) + return null; + + try + { + return project.Data.GetValue(Project.KnownDataKeys.SlackToken, serializer); + } + catch (Exception) + { + // Ignored + } + + return null; } public static bool HasHourlyUsage(this Project project, DateTime date) diff --git a/src/Exceptionless.Core/Extensions/RequestInfoExtensions.cs b/src/Exceptionless.Core/Extensions/RequestInfoExtensions.cs index 1f992b7a58..8019f6a9ae 100644 --- a/src/Exceptionless.Core/Extensions/RequestInfoExtensions.cs +++ b/src/Exceptionless.Core/Extensions/RequestInfoExtensions.cs @@ -1,35 +1,34 @@ using System.Text; using Exceptionless.Core.Models.Data; -using Newtonsoft.Json; +using Foundatio.Serializer; namespace Exceptionless.Core.Extensions; public static class RequestInfoExtensions { - public static RequestInfo ApplyDataExclusions(this RequestInfo request, IList exclusions, int maxLength = 1000) + public static RequestInfo ApplyDataExclusions(this RequestInfo request, ITextSerializer serializer, IList exclusions, int maxLength = 1000) { request.Cookies = ApplyExclusions(request.Cookies, exclusions, maxLength); request.QueryString = ApplyExclusions(request.QueryString, exclusions, maxLength); - request.PostData = ApplyPostDataExclusions(request.PostData, exclusions, maxLength); + request.PostData = ApplyPostDataExclusions(request.PostData, serializer, exclusions, maxLength); return request; } - private static object? ApplyPostDataExclusions(object? data, IEnumerable exclusions, int maxLength) + private static object? ApplyPostDataExclusions(object? data, ITextSerializer serializer, IEnumerable exclusions, int maxLength) { if (data is null) return null; var dictionary = data as Dictionary; - if (dictionary is null && data is string) + if (dictionary is null && data is string json) { - string json = (string)data; if (!json.IsJson()) return data; try { - dictionary = JsonConvert.DeserializeObject>(json); + dictionary = serializer.Deserialize>(json); } catch (Exception) { } } diff --git a/src/Exceptionless.Core/Jobs/CloseInactiveSessionsJob.cs b/src/Exceptionless.Core/Jobs/CloseInactiveSessionsJob.cs index c968f02623..8d18510098 100644 --- a/src/Exceptionless.Core/Jobs/CloseInactiveSessionsJob.cs +++ b/src/Exceptionless.Core/Jobs/CloseInactiveSessionsJob.cs @@ -1,5 +1,4 @@ using System.Diagnostics; -using System.Text.Json; using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Repositories; @@ -9,6 +8,7 @@ using Foundatio.Lock; using Foundatio.Repositories; using Foundatio.Resilience; +using Foundatio.Serializer; using Microsoft.Extensions.Diagnostics.HealthChecks; using Microsoft.Extensions.Logging; @@ -20,11 +20,11 @@ public class CloseInactiveSessionsJob : JobWithLockBase, IHealthCheck private readonly IEventRepository _eventRepository; private readonly ICacheClient _cache; private readonly ILockProvider _lockProvider; - private readonly JsonSerializerOptions _jsonOptions; + private readonly ITextSerializer _serializer; private DateTime? _lastActivity; public CloseInactiveSessionsJob(IEventRepository eventRepository, ICacheClient cacheClient, - JsonSerializerOptions jsonOptions, + ITextSerializer serializer, TimeProvider timeProvider, IResiliencePolicyProvider resiliencePolicyProvider, ILoggerFactory loggerFactory @@ -33,7 +33,7 @@ ILoggerFactory loggerFactory _eventRepository = eventRepository; _cache = cacheClient; _lockProvider = new ThrottlingLockProvider(cacheClient, 1, TimeSpan.FromMinutes(1), timeProvider, resiliencePolicyProvider, loggerFactory); - _jsonOptions = jsonOptions; + _serializer = serializer; } protected override Task GetLockAsync(CancellationToken cancellationToken = default) @@ -130,7 +130,7 @@ protected override async Task RunInternalAsync(JobContext context) allHeartbeatKeys.Add(sessionIdKey); } - var user = session.GetUserIdentity(_jsonOptions); + var user = session.GetUserIdentity(_serializer); if (!String.IsNullOrWhiteSpace(user?.Identity)) { userIdentityKey = $"Project:{session.ProjectId}:heartbeat:{user.Identity.ToSHA1()}"; diff --git a/src/Exceptionless.Core/Jobs/EventNotificationsJob.cs b/src/Exceptionless.Core/Jobs/EventNotificationsJob.cs index e51f0f0a27..bca68d754f 100644 --- a/src/Exceptionless.Core/Jobs/EventNotificationsJob.cs +++ b/src/Exceptionless.Core/Jobs/EventNotificationsJob.cs @@ -1,4 +1,3 @@ -using System.Text.Json; using Exceptionless.Core.Configuration; using Exceptionless.Core.Extensions; using Exceptionless.Core.Mail; @@ -13,6 +12,7 @@ using Foundatio.Queues; using Foundatio.Repositories; using Foundatio.Resilience; +using Foundatio.Serializer; using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Jobs; @@ -29,7 +29,7 @@ public class EventNotificationsJob : QueueJobBase private readonly IEventRepository _eventRepository; private readonly ICacheClient _cache; private readonly UserAgentParser _parser; - private readonly JsonSerializerOptions _jsonOptions; + private readonly ITextSerializer _serializer; public EventNotificationsJob(IQueue queue, SlackService slackService, @@ -41,7 +41,7 @@ public EventNotificationsJob(IQueue queue, IEventRepository eventRepository, ICacheClient cacheClient, UserAgentParser parser, - JsonSerializerOptions jsonOptions, + ITextSerializer serializer, TimeProvider timeProvider, IResiliencePolicyProvider resiliencePolicyProvider, ILoggerFactory loggerFactory) : base(queue, timeProvider, resiliencePolicyProvider, loggerFactory) @@ -55,7 +55,7 @@ public EventNotificationsJob(IQueue queue, _eventRepository = eventRepository; _cache = cacheClient; _parser = parser; - _jsonOptions = jsonOptions; + _serializer = serializer; } protected override async Task ProcessQueueEntryAsync(QueueEntryContext context) @@ -116,7 +116,7 @@ protected override async Task ProcessQueueEntryAsync(QueueEntryContex _logger.LogTrace("Settings: new error={ReportNewErrors} critical error={ReportCriticalErrors} regression={ReportEventRegressions} new={ReportNewEvents} critical={ReportCriticalEvents}", settings.ReportNewErrors, settings.ReportCriticalErrors, settings.ReportEventRegressions, settings.ReportNewEvents, settings.ReportCriticalEvents); _logger.LogTrace("Should process: new error={ShouldReportNewError} critical error={ShouldReportCriticalError} regression={ShouldReportRegression} new={ShouldReportNewEvent} critical={ShouldReportCriticalEvent}", shouldReportNewError, shouldReportCriticalError, shouldReportRegression, shouldReportNewEvent, shouldReportCriticalEvent); } - var request = ev.GetRequestInfo(_jsonOptions); + var request = ev.GetRequestInfo(_serializer); // check for known bots if the user has elected to not report them if (shouldReport && !String.IsNullOrEmpty(request?.UserAgent)) { diff --git a/src/Exceptionless.Core/Jobs/EventPostsJob.cs b/src/Exceptionless.Core/Jobs/EventPostsJob.cs index 605ee3ee6c..87efabe916 100644 --- a/src/Exceptionless.Core/Jobs/EventPostsJob.cs +++ b/src/Exceptionless.Core/Jobs/EventPostsJob.cs @@ -13,8 +13,8 @@ using Foundatio.Queues; using Foundatio.Repositories; using Foundatio.Resilience; +using Foundatio.Serializer; using Microsoft.Extensions.Logging; -using Newtonsoft.Json; namespace Exceptionless.Core.Jobs; @@ -30,10 +30,10 @@ public class EventPostsJob : QueueJobBase private readonly UsageService _usageService; private readonly IOrganizationRepository _organizationRepository; private readonly IProjectRepository _projectRepository; - private readonly JsonSerializerSettings _jsonSerializerSettings; + private readonly ITextSerializer _serializer; private readonly AppOptions _appOptions; - public EventPostsJob(IQueue queue, EventPostService eventPostService, EventParserPluginManager eventParserPluginManager, EventPipeline eventPipeline, UsageService usageService, IOrganizationRepository organizationRepository, IProjectRepository projectRepository, JsonSerializerSettings jsonSerializerSettings, AppOptions appOptions, TimeProvider timeProvider, + public EventPostsJob(IQueue queue, EventPostService eventPostService, EventParserPluginManager eventParserPluginManager, EventPipeline eventPipeline, UsageService usageService, IOrganizationRepository organizationRepository, IProjectRepository projectRepository, ITextSerializer serializer, AppOptions appOptions, TimeProvider timeProvider, IResiliencePolicyProvider resiliencePolicyProvider, ILoggerFactory loggerFactory) : base(queue, timeProvider, resiliencePolicyProvider, loggerFactory) { _eventPostService = eventPostService; @@ -42,7 +42,7 @@ public EventPostsJob(IQueue queue, EventPostService eventPostService, _usageService = usageService; _organizationRepository = organizationRepository; _projectRepository = projectRepository; - _jsonSerializerSettings = jsonSerializerSettings; + _serializer = serializer; _appOptions = appOptions; _maximumEventPostFileSize = _appOptions.MaximumEventPostSize + 1024; @@ -302,7 +302,7 @@ private async Task RetryEventsAsync(List eventsToRetry, EventPo { try { - var stream = new MemoryStream(ev.GetBytes(_jsonSerializerSettings)); + var stream = new MemoryStream(ev.GetBytes(_serializer)); // Put this single event back into the queue so we can retry it separately. await _eventPostService.EnqueueAsync(new EventPost(false) diff --git a/src/Exceptionless.Core/Jobs/WebHooksJob.cs b/src/Exceptionless.Core/Jobs/WebHooksJob.cs index 9f616db157..c88a74e44f 100644 --- a/src/Exceptionless.Core/Jobs/WebHooksJob.cs +++ b/src/Exceptionless.Core/Jobs/WebHooksJob.cs @@ -1,4 +1,7 @@ using System.Net; +using System.Net.Http.Json; +using System.Text; +using System.Text.Json; using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Queues.Models; @@ -10,8 +13,8 @@ using Foundatio.Queues; using Foundatio.Repositories; using Foundatio.Resilience; +using Foundatio.Serializer; using Microsoft.Extensions.Logging; -using Newtonsoft.Json; namespace Exceptionless.Core.Jobs; @@ -32,7 +35,8 @@ public class WebHooksJob : QueueJobBase, IDisposable private readonly SlackService _slackService; private readonly IWebHookRepository _webHookRepository; private readonly ICacheClient _cacheClient; - private readonly JsonSerializerSettings _jsonSerializerSettings; + private readonly ITextSerializer _serializer; + private readonly JsonSerializerOptions _jsonOptions; private readonly AppOptions _appOptions; private HttpClient? _client; @@ -42,14 +46,15 @@ private HttpClient Client get => _client ??= new HttpClient(); } - public WebHooksJob(IQueue queue, IProjectRepository projectRepository, SlackService slackService, IWebHookRepository webHookRepository, ICacheClient cacheClient, JsonSerializerSettings settings, AppOptions appOptions, TimeProvider timeProvider, + public WebHooksJob(IQueue queue, IProjectRepository projectRepository, SlackService slackService, IWebHookRepository webHookRepository, ICacheClient cacheClient, ITextSerializer serializer, JsonSerializerOptions jsonOptions, AppOptions appOptions, TimeProvider timeProvider, IResiliencePolicyProvider resiliencePolicyProvider, ILoggerFactory loggerFactory) : base(queue, timeProvider, resiliencePolicyProvider, loggerFactory) { _projectRepository = projectRepository; _slackService = slackService; _webHookRepository = webHookRepository; _cacheClient = cacheClient; - _jsonSerializerSettings = settings; + _serializer = serializer; + _jsonOptions = jsonOptions; _appOptions = appOptions; } @@ -88,7 +93,7 @@ protected override async Task ProcessQueueEntryAsync(QueueEntryContex { using (var postCancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(context.CancellationToken, timeoutCancellationTokenSource.Token)) { - response = await Client.PostAsJsonAsync(body.Url, body.Data.ToJson(Formatting.Indented, _jsonSerializerSettings), postCancellationTokenSource.Token); + response = await Client.PostAsJsonAsync(body.Url, body.Data, _jsonOptions, postCancellationTokenSource.Token); if (!response.IsSuccessStatusCode) successful = false; else if (consecutiveErrors > 0) @@ -165,7 +170,7 @@ private async Task IsEnabledAsync(WebHookNotification body) return webHook?.IsEnabled ?? false; case WebHookType.Slack: var project = await _projectRepository.GetByIdAsync(body.ProjectId, o => o.Cache()); - var token = project?.GetSlackToken(); + var token = project?.GetSlackToken(_serializer); return token is not null; } @@ -181,7 +186,7 @@ private async Task DisableIntegrationAsync(WebHookNotification body) break; case WebHookType.Slack: var project = await _projectRepository.GetByIdAsync(body.ProjectId); - var token = project?.GetSlackToken(); + var token = project?.GetSlackToken(_serializer); if (token is null) return; diff --git a/src/Exceptionless.Core/Mail/Mailer.cs b/src/Exceptionless.Core/Mail/Mailer.cs index f18e70bfcc..f91e2538e9 100644 --- a/src/Exceptionless.Core/Mail/Mailer.cs +++ b/src/Exceptionless.Core/Mail/Mailer.cs @@ -1,11 +1,11 @@ using System.Collections.Concurrent; -using System.Text.Json; using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Plugins.Formatting; using Exceptionless.Core.Queues.Models; using Exceptionless.DateTimeExtensions; using Foundatio.Queues; +using Foundatio.Serializer; using HandlebarsDotNet; using Microsoft.Extensions.Logging; @@ -18,16 +18,16 @@ public class Mailer : IMailer private readonly FormattingPluginManager _pluginManager; private readonly AppOptions _appOptions; private readonly TimeProvider _timeProvider; - private readonly JsonSerializerOptions _jsonOptions; + private readonly ITextSerializer _serializer; private readonly ILogger _logger; - public Mailer(IQueue queue, FormattingPluginManager pluginManager, JsonSerializerOptions jsonOptions, AppOptions appOptions, TimeProvider timeProvider, ILogger logger) + public Mailer(IQueue queue, FormattingPluginManager pluginManager, ITextSerializer serializer, AppOptions appOptions, TimeProvider timeProvider, ILogger logger) { _queue = queue; _pluginManager = pluginManager; _appOptions = appOptions; _timeProvider = timeProvider; - _jsonOptions = jsonOptions; + _serializer = serializer; _logger = logger; } @@ -59,7 +59,7 @@ public async Task SendEventNoticeAsync(User user, PersistentEvent ev, Proj }; AddDefaultFields(ev, result.Data); - AddUserInfo(ev, messageData, _jsonOptions); + AddUserInfo(ev, messageData, _serializer); const string template = "event-notice"; await QueueMessageAsync(new MailMessage @@ -71,10 +71,10 @@ await QueueMessageAsync(new MailMessage return true; } - private static void AddUserInfo(PersistentEvent ev, Dictionary data, JsonSerializerOptions jsonOptions) + private static void AddUserInfo(PersistentEvent ev, Dictionary data, ITextSerializer serializer) { - var ud = ev.GetUserDescription(jsonOptions); - var ui = ev.GetUserIdentity(jsonOptions); + var ud = ev.GetUserDescription(serializer); + var ui = ev.GetUserIdentity(serializer); if (!String.IsNullOrEmpty(ud?.Description)) data["UserDescription"] = ud.Description; diff --git a/src/Exceptionless.Core/Models/Event.cs b/src/Exceptionless.Core/Models/Event.cs index 3975db18ca..8007f06aff 100644 --- a/src/Exceptionless.Core/Models/Event.cs +++ b/src/Exceptionless.Core/Models/Event.cs @@ -1,11 +1,13 @@ using System.ComponentModel.DataAnnotations; using System.Diagnostics; +using System.Text.Json; +using System.Text.Json.Serialization; using Exceptionless.Core.Extensions; namespace Exceptionless.Core.Models; [DebuggerDisplay("Type: {Type}, Date: {Date}, Message: {Message}, Value: {Value}, Count: {Count}")] -public class Event : IData +public class Event : IData, IJsonOnDeserialized { /// /// The event type (ie. error, log message, feature usage). Check Event.KnownTypes for standard event types. @@ -55,11 +57,56 @@ public class Event : IData /// public DataDictionary? Data { get; set; } = new(); + /// + /// Captures unknown JSON properties during deserialization. + /// These are merged into after deserialization. + /// Known data keys like "@error", "@request", "@environment" may appear at root level. + /// + [JsonExtensionData] + [JsonInclude] + internal Dictionary? ExtensionData { get; set; } + /// /// An optional identifier to be used for referencing this event instance at a later time. /// public string? ReferenceId { get; set; } + /// + /// Called after JSON deserialization to merge extension data into the Data dictionary. + /// This handles the case where known data keys like "@error", "@request", "@environment" + /// appear at the JSON root level instead of nested under "data". + /// + void IJsonOnDeserialized.OnDeserialized() + { + if (ExtensionData is null || ExtensionData.Count == 0) + return; + + Data ??= new DataDictionary(); + foreach (var kvp in ExtensionData) + { + Data[kvp.Key] = ConvertJsonElement(kvp.Value); + } + ExtensionData = null; + } + + /// + /// Converts a to a .NET type so downstream code + /// (e.g., value as string) works correctly. + /// + private static object? ConvertJsonElement(JsonElement element) + { + return element.ValueKind switch + { + JsonValueKind.String => element.GetString(), + JsonValueKind.Number => element.TryGetInt64(out long l) ? l : element.GetDouble(), + JsonValueKind.True => true, + JsonValueKind.False => false, + JsonValueKind.Null or JsonValueKind.Undefined => null, + // For objects/arrays, keep as JsonElement — GetValue handles these + _ => element + }; + } + protected bool Equals(Event other) { return String.Equals(Type, other.Type) && String.Equals(Source, other.Source) && Tags.CollectionEquals(other.Tags) && String.Equals(Message, other.Message) && String.Equals(Geo, other.Geo) && Value == other.Value && Equals(Data, other.Data); diff --git a/src/Exceptionless.Core/Models/Messaging/ReleaseNotification.cs b/src/Exceptionless.Core/Models/Messaging/ReleaseNotification.cs index b819fcd7d1..baab8744a0 100644 --- a/src/Exceptionless.Core/Models/Messaging/ReleaseNotification.cs +++ b/src/Exceptionless.Core/Models/Messaging/ReleaseNotification.cs @@ -1,4 +1,4 @@ -namespace Exceptionless.Core.Messaging.Models; +namespace Exceptionless.Core.Messaging.Models; public record ReleaseNotification { diff --git a/src/Exceptionless.Core/Models/SlackToken.cs b/src/Exceptionless.Core/Models/SlackToken.cs index 28d89e3a72..6d94160b28 100644 --- a/src/Exceptionless.Core/Models/SlackToken.cs +++ b/src/Exceptionless.Core/Models/SlackToken.cs @@ -1,5 +1,5 @@ -using System.Text.Json; -using Newtonsoft.Json; +using System.Text.Json.Serialization; +using Foundatio.Serializer; namespace Exceptionless.Core.Models; @@ -28,19 +28,19 @@ public SlackMessage(string text) Text = text; } - [JsonProperty("text")] + [JsonPropertyName("text")] public string Text { get; init; } - [JsonProperty("attachments")] + [JsonPropertyName("attachments")] public List Attachments { get; init; } = []; public class SlackAttachment { - public SlackAttachment(PersistentEvent ev, JsonSerializerOptions jsonOptions) + public SlackAttachment(PersistentEvent ev, ITextSerializer serializer) { TimeStamp = ev.Date.ToUnixTimeSeconds(); - var ud = ev.GetUserDescription(jsonOptions); - var ui = ev.GetUserIdentity(jsonOptions); + var ud = ev.GetUserDescription(serializer); + var ui = ev.GetUserIdentity(serializer); Text = ud?.Description; string? displayName = null; @@ -67,34 +67,34 @@ public SlackAttachment(PersistentEvent ev, JsonSerializerOptions jsonOptions) } } - [JsonProperty("title")] + [JsonPropertyName("title")] public string? Title { get; init; } - [JsonProperty("text")] + [JsonPropertyName("text")] public string? Text { get; init; } - [JsonProperty("author_name")] + [JsonPropertyName("author_name")] public string? AuthorName { get; init; } - [JsonProperty("author_link")] + [JsonPropertyName("author_link")] public string? AuthorLink { get; init; } - [JsonProperty("author_icon")] + [JsonPropertyName("author_icon")] public string? AuthorIcon { get; init; } - [JsonProperty("color")] + [JsonPropertyName("color")] public string Color { get; set; } = "#5E9A00"; - [JsonProperty("fields")] + [JsonPropertyName("fields")] public List Fields { get; init; } = []; - [JsonProperty("mrkdwn_in")] + [JsonPropertyName("mrkdwn_in")] public string[] SupportedMarkdownFields { get; init; } = ["text", "fields"]; - [JsonProperty("ts")] + [JsonPropertyName("ts")] public long TimeStamp { get; init; } } public record SlackAttachmentFields { - [JsonProperty("title")] + [JsonPropertyName("title")] public string Title { get; init; } = null!; - [JsonProperty("value")] + [JsonPropertyName("value")] public string? Value { get; init; } - [JsonProperty("short")] + [JsonPropertyName("short")] public bool Short { get; init; } } } diff --git a/src/Exceptionless.Core/Models/Stack.cs b/src/Exceptionless.Core/Models/Stack.cs index e9532e099c..c47b56716f 100644 --- a/src/Exceptionless.Core/Models/Stack.cs +++ b/src/Exceptionless.Core/Models/Stack.cs @@ -5,7 +5,6 @@ using System.Text.Json.Serialization; using Exceptionless.Core.Attributes; using Foundatio.Repositories.Models; -using Newtonsoft.Json.Converters; namespace Exceptionless.Core.Models; @@ -131,7 +130,6 @@ public static class KnownTypes } [JsonConverter(typeof(JsonStringEnumConverter))] -[Newtonsoft.Json.JsonConverter(typeof(StringEnumConverter))] public enum StackStatus { [JsonStringEnumMemberName("open")] diff --git a/src/Exceptionless.Core/Models/StackSummaryModel.cs b/src/Exceptionless.Core/Models/StackSummaryModel.cs index 197065fc71..eff763c2b5 100644 --- a/src/Exceptionless.Core/Models/StackSummaryModel.cs +++ b/src/Exceptionless.Core/Models/StackSummaryModel.cs @@ -5,7 +5,7 @@ namespace Exceptionless.Core.Models; [DebuggerDisplay("Id: {Id}, Status: {Status}, Title: {Title}, First: {FirstOccurrence}, Last: {LastOccurrence}")] public record StackSummaryModel : SummaryData { - public required string Title { get; init; } + public string? Title { get; init; } public StackStatus Status { get; init; } public DateTime FirstOccurrence { get; init; } public DateTime LastOccurrence { get; init; } diff --git a/src/Exceptionless.Core/Models/SummaryData.cs b/src/Exceptionless.Core/Models/SummaryData.cs index 7433766a68..7419315017 100644 --- a/src/Exceptionless.Core/Models/SummaryData.cs +++ b/src/Exceptionless.Core/Models/SummaryData.cs @@ -4,5 +4,5 @@ public record SummaryData { public required string Id { get; set; } public required string TemplateKey { get; set; } - public required object Data { get; set; } + public object? Data { get; set; } } diff --git a/src/Exceptionless.Core/Plugins/EventParser/Default/JsonEventParserPlugin.cs b/src/Exceptionless.Core/Plugins/EventParser/Default/JsonEventParserPlugin.cs index b40c9520bb..715c05be77 100644 --- a/src/Exceptionless.Core/Plugins/EventParser/Default/JsonEventParserPlugin.cs +++ b/src/Exceptionless.Core/Plugins/EventParser/Default/JsonEventParserPlugin.cs @@ -1,19 +1,19 @@ using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Pipeline; +using Foundatio.Serializer; using Microsoft.Extensions.Logging; -using Newtonsoft.Json; namespace Exceptionless.Core.Plugins.EventParser; [Priority(0)] public class JsonEventParserPlugin : PluginBase, IEventParserPlugin { - private readonly JsonSerializerSettings _settings; + private readonly ITextSerializer _serializer; - public JsonEventParserPlugin(AppOptions options, JsonSerializerSettings settings, ILoggerFactory loggerFactory) : base(options, loggerFactory) + public JsonEventParserPlugin(AppOptions options, ITextSerializer serializer, ILoggerFactory loggerFactory) : base(options, loggerFactory) { - _settings = settings; + _serializer = serializer; } public List? ParseEvents(string input, int apiVersion, string? userAgent) @@ -26,15 +26,30 @@ public JsonEventParserPlugin(AppOptions options, JsonSerializerSettings settings { case JsonType.Object: { - if (input.TryFromJson(out PersistentEvent? ev, _settings) && ev is not null) - events.Add(ev); + try + { + var ev = _serializer.Deserialize(input); + if (ev is not null) + events.Add(ev); + } + catch + { + // Invalid JSON - ignore + } break; } case JsonType.Array: { - if (input.TryFromJson(out PersistentEvent[]? parsedEvents, _settings) && parsedEvents is { Length: > 0 }) - events.AddRange(parsedEvents); - + try + { + var parsedEvents = _serializer.Deserialize(input); + if (parsedEvents is { Length: > 0 }) + events.AddRange(parsedEvents); + } + catch + { + // Invalid JSON - ignore + } break; } } diff --git a/src/Exceptionless.Core/Plugins/EventParser/Default/LegacyErrorParserPlugin.cs b/src/Exceptionless.Core/Plugins/EventParser/Default/LegacyErrorParserPlugin.cs index 5d8c337249..8500c52329 100644 --- a/src/Exceptionless.Core/Plugins/EventParser/Default/LegacyErrorParserPlugin.cs +++ b/src/Exceptionless.Core/Plugins/EventParser/Default/LegacyErrorParserPlugin.cs @@ -1,9 +1,9 @@ -using Exceptionless.Core.Extensions; +using System.Text.Json; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Pipeline; using Exceptionless.Core.Plugins.EventUpgrader; using Microsoft.Extensions.Logging; -using Newtonsoft.Json; namespace Exceptionless.Core.Plugins.EventParser; @@ -11,12 +11,12 @@ namespace Exceptionless.Core.Plugins.EventParser; public class LegacyErrorParserPlugin : PluginBase, IEventParserPlugin { private readonly EventUpgraderPluginManager _manager; - private readonly JsonSerializerSettings _settings; + private readonly JsonSerializerOptions _jsonOptions; - public LegacyErrorParserPlugin(EventUpgraderPluginManager manager, JsonSerializerSettings settings, AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) + public LegacyErrorParserPlugin(EventUpgraderPluginManager manager, JsonSerializerOptions jsonOptions, AppOptions appOptions, ILoggerFactory loggerFactory) : base(appOptions, loggerFactory) { _manager = manager; - _settings = settings; + _jsonOptions = jsonOptions; } public List? ParseEvents(string input, int apiVersion, string? userAgent) @@ -29,7 +29,7 @@ public LegacyErrorParserPlugin(EventUpgraderPluginManager manager, JsonSerialize var ctx = new EventUpgraderContext(input); _manager.Upgrade(ctx); - return ctx.Documents.FromJson(_settings); + return ctx.Documents.ToList(_jsonOptions); } catch (Exception ex) { diff --git a/src/Exceptionless.Core/Plugins/EventProcessor/Default/03_ManualStackingPlugin.cs b/src/Exceptionless.Core/Plugins/EventProcessor/Default/03_ManualStackingPlugin.cs index 4b17fd8dc3..54026b14ba 100644 --- a/src/Exceptionless.Core/Plugins/EventProcessor/Default/03_ManualStackingPlugin.cs +++ b/src/Exceptionless.Core/Plugins/EventProcessor/Default/03_ManualStackingPlugin.cs @@ -1,6 +1,6 @@ -using System.Text.Json; -using Exceptionless.Core.Extensions; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Pipeline; +using Foundatio.Serializer; using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Plugins.EventProcessor; @@ -8,16 +8,16 @@ namespace Exceptionless.Core.Plugins.EventProcessor; [Priority(3)] public sealed class ManualStackingPlugin : EventProcessorPluginBase { - private readonly JsonSerializerOptions _jsonOptions; + private readonly ITextSerializer _serializer; - public ManualStackingPlugin(JsonSerializerOptions jsonOptions, AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) + public ManualStackingPlugin(ITextSerializer serializer, AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) { - _jsonOptions = jsonOptions; + _serializer = serializer; } public override Task EventProcessingAsync(EventContext context) { - var msi = context.Event.GetManualStackingInfo(_jsonOptions); + var msi = context.Event.GetManualStackingInfo(_serializer); if (msi?.SignatureData is not null) { foreach (var kvp in msi.SignatureData) diff --git a/src/Exceptionless.Core/Plugins/EventProcessor/Default/0_ThrottleBotsPlugin.cs b/src/Exceptionless.Core/Plugins/EventProcessor/Default/0_ThrottleBotsPlugin.cs index 36610ef3a8..8cd28b99c7 100644 --- a/src/Exceptionless.Core/Plugins/EventProcessor/Default/0_ThrottleBotsPlugin.cs +++ b/src/Exceptionless.Core/Plugins/EventProcessor/Default/0_ThrottleBotsPlugin.cs @@ -1,11 +1,11 @@ -using System.Text.Json; -using Exceptionless.Core.Extensions; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Models.WorkItems; using Exceptionless.Core.Pipeline; using Exceptionless.DateTimeExtensions; using Foundatio.Caching; using Foundatio.Jobs; using Foundatio.Queues; +using Foundatio.Serializer; using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Plugins.EventProcessor; @@ -16,15 +16,15 @@ public sealed class ThrottleBotsPlugin : EventProcessorPluginBase private readonly ICacheClient _cache; private readonly IQueue _workItemQueue; private readonly TimeProvider _timeProvider; - private readonly JsonSerializerOptions _jsonOptions; + private readonly ITextSerializer _serializer; private readonly TimeSpan _throttlingPeriod = TimeSpan.FromMinutes(5); public ThrottleBotsPlugin(ICacheClient cacheClient, IQueue workItemQueue, - JsonSerializerOptions jsonOptions, TimeProvider timeProvider, AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) + ITextSerializer serializer, TimeProvider timeProvider, AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) { _cache = cacheClient; _workItemQueue = workItemQueue; - _jsonOptions = jsonOptions; + _serializer = serializer; _timeProvider = timeProvider; } @@ -38,7 +38,7 @@ public override async Task EventBatchProcessingAsync(ICollection c return; // Throttle errors by client ip address to no more than X every 5 minutes. - var clientIpAddressGroups = contexts.GroupBy(c => c.Event.GetRequestInfo(_jsonOptions)?.ClientIpAddress); + var clientIpAddressGroups = contexts.GroupBy(c => c.Event.GetRequestInfo(_serializer)?.ClientIpAddress); foreach (var clientIpAddressGroup in clientIpAddressGroups) { if (String.IsNullOrEmpty(clientIpAddressGroup.Key) || clientIpAddressGroup.Key.IsPrivateNetwork()) diff --git a/src/Exceptionless.Core/Plugins/EventProcessor/Default/10_NotFoundPlugin.cs b/src/Exceptionless.Core/Plugins/EventProcessor/Default/10_NotFoundPlugin.cs index 99599dcaa1..de32d65000 100644 --- a/src/Exceptionless.Core/Plugins/EventProcessor/Default/10_NotFoundPlugin.cs +++ b/src/Exceptionless.Core/Plugins/EventProcessor/Default/10_NotFoundPlugin.cs @@ -1,7 +1,7 @@ -using System.Text.Json; -using Exceptionless.Core.Extensions; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Pipeline; +using Foundatio.Serializer; using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Plugins.EventProcessor; @@ -9,11 +9,11 @@ namespace Exceptionless.Core.Plugins.EventProcessor; [Priority(10)] public sealed class NotFoundPlugin : EventProcessorPluginBase { - private readonly JsonSerializerOptions _jsonOptions; + private readonly ITextSerializer _serializer; - public NotFoundPlugin(JsonSerializerOptions jsonOptions, AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) + public NotFoundPlugin(ITextSerializer serializer, AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) { - _jsonOptions = jsonOptions; + _serializer = serializer; } public override Task EventProcessingAsync(EventContext context) @@ -24,7 +24,7 @@ public override Task EventProcessingAsync(EventContext context) context.Event.Data.Remove(Event.KnownDataKeys.EnvironmentInfo); context.Event.Data.Remove(Event.KnownDataKeys.TraceLog); - var req = context.Event.GetRequestInfo(_jsonOptions); + var req = context.Event.GetRequestInfo(_serializer); if (req is null) return Task.CompletedTask; diff --git a/src/Exceptionless.Core/Plugins/EventProcessor/Default/20_ErrorPlugin.cs b/src/Exceptionless.Core/Plugins/EventProcessor/Default/20_ErrorPlugin.cs index f94a57e519..499c1279b5 100644 --- a/src/Exceptionless.Core/Plugins/EventProcessor/Default/20_ErrorPlugin.cs +++ b/src/Exceptionless.Core/Plugins/EventProcessor/Default/20_ErrorPlugin.cs @@ -1,8 +1,8 @@ -using System.Text.Json; -using Exceptionless.Core.Extensions; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Pipeline; using Exceptionless.Core.Utility; +using Foundatio.Serializer; using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Plugins.EventProcessor; @@ -10,11 +10,11 @@ namespace Exceptionless.Core.Plugins.EventProcessor; [Priority(20)] public sealed class ErrorPlugin : EventProcessorPluginBase { - private readonly JsonSerializerOptions _jsonOptions; + private readonly ITextSerializer _serializer; - public ErrorPlugin(JsonSerializerOptions jsonOptions, AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) + public ErrorPlugin(ITextSerializer serializer, AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) { - _jsonOptions = jsonOptions; + _serializer = serializer; } public override Task EventProcessingAsync(EventContext context) @@ -22,7 +22,7 @@ public override Task EventProcessingAsync(EventContext context) if (!context.Event.IsError()) return Task.CompletedTask; - var error = context.Event.GetError(_jsonOptions); + var error = context.Event.GetError(_serializer); if (error is null) return Task.CompletedTask; @@ -40,7 +40,7 @@ public override Task EventProcessingAsync(EventContext context) if (context.HasProperty("UserNamespaces")) userNamespaces = context.GetProperty("UserNamespaces")?.SplitAndTrim([',']); - var signature = new ErrorSignature(error, _jsonOptions, userNamespaces, userCommonMethods); + var signature = new ErrorSignature(error, _serializer, userNamespaces, userCommonMethods); if (signature.SignatureInfo.Count <= 0) return Task.CompletedTask; diff --git a/src/Exceptionless.Core/Plugins/EventProcessor/Default/30_SimpleErrorPlugin.cs b/src/Exceptionless.Core/Plugins/EventProcessor/Default/30_SimpleErrorPlugin.cs index 5d3da08178..c849bb258a 100644 --- a/src/Exceptionless.Core/Plugins/EventProcessor/Default/30_SimpleErrorPlugin.cs +++ b/src/Exceptionless.Core/Plugins/EventProcessor/Default/30_SimpleErrorPlugin.cs @@ -1,7 +1,7 @@ -using System.Text.Json; -using Exceptionless.Core.Extensions; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Pipeline; +using Foundatio.Serializer; using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Plugins.EventProcessor; @@ -9,11 +9,11 @@ namespace Exceptionless.Core.Plugins.EventProcessor; [Priority(30)] public sealed class SimpleErrorPlugin : EventProcessorPluginBase { - private readonly JsonSerializerOptions _jsonOptions; + private readonly ITextSerializer _serializer; - public SimpleErrorPlugin(JsonSerializerOptions jsonOptions, AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) + public SimpleErrorPlugin(ITextSerializer serializer, AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) { - _jsonOptions = jsonOptions; + _serializer = serializer; } public override Task EventProcessingAsync(EventContext context) @@ -21,7 +21,7 @@ public override Task EventProcessingAsync(EventContext context) if (!context.Event.IsError()) return Task.CompletedTask; - var error = context.Event.GetSimpleError(_jsonOptions); + var error = context.Event.GetSimpleError(_serializer); if (error is null) return Task.CompletedTask; diff --git a/src/Exceptionless.Core/Plugins/EventProcessor/Default/40_RequestInfoPlugin.cs b/src/Exceptionless.Core/Plugins/EventProcessor/Default/40_RequestInfoPlugin.cs index 56d66b938f..00780b7bc9 100644 --- a/src/Exceptionless.Core/Plugins/EventProcessor/Default/40_RequestInfoPlugin.cs +++ b/src/Exceptionless.Core/Plugins/EventProcessor/Default/40_RequestInfoPlugin.cs @@ -1,9 +1,9 @@ -using System.Text.Json; -using Exceptionless.Core.Extensions; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Models.Data; using Exceptionless.Core.Pipeline; using Exceptionless.Core.Utility; +using Foundatio.Serializer; using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Plugins.EventProcessor; @@ -25,12 +25,12 @@ public sealed class RequestInfoPlugin : EventProcessorPluginBase ]; private readonly UserAgentParser _parser; - private readonly JsonSerializerOptions _jsonOptions; + private readonly ITextSerializer _serializer; - public RequestInfoPlugin(UserAgentParser parser, JsonSerializerOptions jsonOptions, AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) + public RequestInfoPlugin(UserAgentParser parser, ITextSerializer serializer, AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) { _parser = parser; - _jsonOptions = jsonOptions; + _serializer = serializer; } public override async Task EventBatchProcessingAsync(ICollection contexts) @@ -39,13 +39,13 @@ public override async Task EventBatchProcessingAsync(ICollection c var exclusions = DefaultExclusions.Union(project.Configuration.Settings.GetStringCollection(SettingsDictionary.KnownKeys.DataExclusions)).ToList(); foreach (var context in contexts) { - var request = context.Event.GetRequestInfo(_jsonOptions); + var request = context.Event.GetRequestInfo(_serializer); if (request is null) continue; if (context.IncludePrivateInformation) { - var submissionClient = context.Event.GetSubmissionClient(_jsonOptions); + var submissionClient = context.Event.GetSubmissionClient(_serializer); AddClientIpAddress(request, submissionClient); } else @@ -57,7 +57,7 @@ public override async Task EventBatchProcessingAsync(ICollection c } await SetBrowserOsAndDeviceFromUserAgent(request, context); - context.Event.AddRequestInfo(request.ApplyDataExclusions(exclusions, MAX_VALUE_LENGTH)); + context.Event.AddRequestInfo(request.ApplyDataExclusions(_serializer, exclusions, MAX_VALUE_LENGTH)); } } diff --git a/src/Exceptionless.Core/Plugins/EventProcessor/Default/45_EnvironmentInfoPlugin.cs b/src/Exceptionless.Core/Plugins/EventProcessor/Default/45_EnvironmentInfoPlugin.cs index e3fdcd23b6..84ad218246 100644 --- a/src/Exceptionless.Core/Plugins/EventProcessor/Default/45_EnvironmentInfoPlugin.cs +++ b/src/Exceptionless.Core/Plugins/EventProcessor/Default/45_EnvironmentInfoPlugin.cs @@ -1,7 +1,7 @@ -using System.Text.Json; -using Exceptionless.Core.Extensions; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Models.Data; using Exceptionless.Core.Pipeline; +using Foundatio.Serializer; using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Plugins.EventProcessor.Default; @@ -9,22 +9,22 @@ namespace Exceptionless.Core.Plugins.EventProcessor.Default; [Priority(45)] public sealed class EnvironmentInfoPlugin : EventProcessorPluginBase { - private readonly JsonSerializerOptions _jsonOptions; + private readonly ITextSerializer _serializer; - public EnvironmentInfoPlugin(JsonSerializerOptions jsonOptions, AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) + public EnvironmentInfoPlugin(ITextSerializer serializer, AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) { - _jsonOptions = jsonOptions; + _serializer = serializer; } public override Task EventProcessingAsync(EventContext context) { - var environment = context.Event.GetEnvironmentInfo(_jsonOptions); + var environment = context.Event.GetEnvironmentInfo(_serializer); if (environment is null) return Task.CompletedTask; if (context.IncludePrivateInformation) { - var submissionClient = context.Event.GetSubmissionClient(_jsonOptions); + var submissionClient = context.Event.GetSubmissionClient(_serializer); AddClientIpAddress(environment, submissionClient); } else diff --git a/src/Exceptionless.Core/Plugins/EventProcessor/Default/50_GeoPlugin.cs b/src/Exceptionless.Core/Plugins/EventProcessor/Default/50_GeoPlugin.cs index 13bea073bf..2baf6ae591 100644 --- a/src/Exceptionless.Core/Plugins/EventProcessor/Default/50_GeoPlugin.cs +++ b/src/Exceptionless.Core/Plugins/EventProcessor/Default/50_GeoPlugin.cs @@ -1,8 +1,8 @@ -using System.Text.Json; -using Exceptionless.Core.Extensions; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Geo; using Exceptionless.Core.Models; using Exceptionless.Core.Pipeline; +using Foundatio.Serializer; using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Plugins.EventProcessor.Default; @@ -11,12 +11,12 @@ namespace Exceptionless.Core.Plugins.EventProcessor.Default; public sealed class GeoPlugin : EventProcessorPluginBase { private readonly IGeoIpService _geoIpService; - private readonly JsonSerializerOptions _jsonOptions; + private readonly ITextSerializer _serializer; - public GeoPlugin(IGeoIpService geoIpService, JsonSerializerOptions jsonOptions, AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) + public GeoPlugin(IGeoIpService geoIpService, ITextSerializer serializer, AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) { _geoIpService = geoIpService; - _jsonOptions = jsonOptions; + _serializer = serializer; } public override Task EventBatchProcessingAsync(ICollection contexts) @@ -35,7 +35,7 @@ public override Task EventBatchProcessingAsync(ICollection context // The geo coordinates are all the same, set the location from the result of any of the ip addresses. if (!String.IsNullOrEmpty(group.Key)) { - var ips = group.SelectMany(c => c.Event.GetIpAddresses(_jsonOptions)).Union(new[] { group.First().EventPostInfo?.IpAddress }).Distinct().ToList(); + var ips = group.SelectMany(c => c.Event.GetIpAddresses(_serializer)).Union(new[] { group.First().EventPostInfo?.IpAddress }).Distinct().ToList(); if (ips.Count > 0) tasks.Add(UpdateGeoInformationAsync(group, ips)); continue; @@ -44,7 +44,7 @@ public override Task EventBatchProcessingAsync(ICollection context // Each event in the group could be a different user; foreach (var context in group) { - var ips = context.Event.GetIpAddresses(_jsonOptions).Union(new[] { context.EventPostInfo?.IpAddress }).ToList(); + var ips = context.Event.GetIpAddresses(_serializer).Union(new[] { context.EventPostInfo?.IpAddress }).ToList(); if (ips.Count > 0) tasks.Add(UpdateGeoInformationAsync(context, ips)); } diff --git a/src/Exceptionless.Core/Plugins/EventProcessor/Default/70_SessionPlugin.cs b/src/Exceptionless.Core/Plugins/EventProcessor/Default/70_SessionPlugin.cs index 42b0169be7..c2af189be4 100644 --- a/src/Exceptionless.Core/Plugins/EventProcessor/Default/70_SessionPlugin.cs +++ b/src/Exceptionless.Core/Plugins/EventProcessor/Default/70_SessionPlugin.cs @@ -1,10 +1,10 @@ -using System.Text.Json; -using Exceptionless.Core.Extensions; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Pipeline; using Exceptionless.Core.Repositories; using Foundatio.Caching; using Foundatio.Repositories.Utility; +using Foundatio.Serializer; using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Plugins.EventProcessor.Default; @@ -18,21 +18,21 @@ public sealed class SessionPlugin : EventProcessorPluginBase private readonly UpdateStatsAction _updateStats; private readonly AssignToStackAction _assignToStack; private readonly LocationPlugin _locationPlugin; - private readonly JsonSerializerOptions _jsonOptions; + private readonly ITextSerializer _serializer; - public SessionPlugin(ICacheClient cacheClient, IEventRepository eventRepository, AssignToStackAction assignToStack, UpdateStatsAction updateStats, LocationPlugin locationPlugin, JsonSerializerOptions jsonOptions, AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) + public SessionPlugin(ICacheClient cacheClient, IEventRepository eventRepository, AssignToStackAction assignToStack, UpdateStatsAction updateStats, LocationPlugin locationPlugin, ITextSerializer serializer, AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) { _cache = new ScopedCacheClient(cacheClient, "session"); _eventRepository = eventRepository; _assignToStack = assignToStack; _updateStats = updateStats; _locationPlugin = locationPlugin; - _jsonOptions = jsonOptions; + _serializer = serializer; } public override Task EventBatchProcessingAsync(ICollection contexts) { - var autoSessionEvents = contexts.Where(c => !String.IsNullOrWhiteSpace(c.Event.GetUserIdentity(_jsonOptions)?.Identity) && String.IsNullOrEmpty(c.Event.GetSessionId())).ToList(); + var autoSessionEvents = contexts.Where(c => !String.IsNullOrWhiteSpace(c.Event.GetUserIdentity(_serializer)?.Identity) && String.IsNullOrEmpty(c.Event.GetSessionId())).ToList(); var manualSessionsEvents = contexts.Where(c => !String.IsNullOrEmpty(c.Event.GetSessionId())).ToList(); return Task.WhenAll( @@ -125,7 +125,7 @@ private async Task ProcessAutoSessionsAsync(ICollection contexts) { var identityGroups = contexts .OrderBy(c => c.Event.Date) - .GroupBy(c => c.Event.GetUserIdentity(_jsonOptions)?.Identity); + .GroupBy(c => c.Event.GetUserIdentity(_serializer)?.Identity); foreach (var identityGroup in identityGroups) { @@ -286,7 +286,7 @@ private Task SetIdentitySessionIdAsync(string projectId, string identity, private async Task CreateSessionStartEventAsync(EventContext startContext, DateTime? lastActivityUtc, bool? isSessionEnd) { - var startEvent = startContext.Event.ToSessionStartEvent(_jsonOptions, lastActivityUtc, isSessionEnd, startContext.Organization.HasPremiumFeatures, startContext.IncludePrivateInformation); + var startEvent = startContext.Event.ToSessionStartEvent(_serializer, lastActivityUtc, isSessionEnd, startContext.Organization.HasPremiumFeatures, startContext.IncludePrivateInformation); var startEventContexts = new List { new(startEvent, startContext.Organization, startContext.Project) }; diff --git a/src/Exceptionless.Core/Plugins/EventProcessor/Default/80_AngularPlugin.cs b/src/Exceptionless.Core/Plugins/EventProcessor/Default/80_AngularPlugin.cs index 5d89be33e9..9c84b8dc66 100644 --- a/src/Exceptionless.Core/Plugins/EventProcessor/Default/80_AngularPlugin.cs +++ b/src/Exceptionless.Core/Plugins/EventProcessor/Default/80_AngularPlugin.cs @@ -1,7 +1,7 @@ -using System.Text.Json; -using Exceptionless.Core.Extensions; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Pipeline; +using Foundatio.Serializer; using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Plugins.EventProcessor; @@ -9,11 +9,11 @@ namespace Exceptionless.Core.Plugins.EventProcessor; [Priority(80)] public sealed class AngularPlugin : EventProcessorPluginBase { - private readonly JsonSerializerOptions _jsonOptions; + private readonly ITextSerializer _serializer; - public AngularPlugin(JsonSerializerOptions jsonOptions, AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) + public AngularPlugin(ITextSerializer serializer, AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) { - _jsonOptions = jsonOptions; + _serializer = serializer; } public override Task EventProcessingAsync(EventContext context) @@ -21,7 +21,7 @@ public override Task EventProcessingAsync(EventContext context) if (!context.Event.IsError()) return Task.CompletedTask; - var error = context.Event.GetError(_jsonOptions); + var error = context.Event.GetError(_serializer); if (error is null) return Task.CompletedTask; diff --git a/src/Exceptionless.Core/Plugins/EventProcessor/Default/90_RemovePrivateInformationPlugin.cs b/src/Exceptionless.Core/Plugins/EventProcessor/Default/90_RemovePrivateInformationPlugin.cs index 757ee3cfdd..d870e19d1a 100644 --- a/src/Exceptionless.Core/Plugins/EventProcessor/Default/90_RemovePrivateInformationPlugin.cs +++ b/src/Exceptionless.Core/Plugins/EventProcessor/Default/90_RemovePrivateInformationPlugin.cs @@ -1,5 +1,5 @@ -using System.Text.Json; -using Exceptionless.Core.Pipeline; +using Exceptionless.Core.Pipeline; +using Foundatio.Serializer; using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Plugins.EventProcessor.Default; @@ -7,11 +7,11 @@ namespace Exceptionless.Core.Plugins.EventProcessor.Default; [Priority(90)] public sealed class RemovePrivateInformationPlugin : EventProcessorPluginBase { - private readonly JsonSerializerOptions _jsonOptions; + private readonly ITextSerializer _serializer; - public RemovePrivateInformationPlugin(JsonSerializerOptions jsonOptions, AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) + public RemovePrivateInformationPlugin(ITextSerializer serializer, AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) { - _jsonOptions = jsonOptions; + _serializer = serializer; } public override Task EventProcessingAsync(EventContext context) @@ -21,7 +21,7 @@ public override Task EventProcessingAsync(EventContext context) context.Event.RemoveUserIdentity(); - var description = context.Event.GetUserDescription(_jsonOptions); + var description = context.Event.GetUserDescription(_serializer); if (description is not null) { description.EmailAddress = null; diff --git a/src/Exceptionless.Core/Plugins/EventUpgrader/Default/GetVersion.cs b/src/Exceptionless.Core/Plugins/EventUpgrader/Default/GetVersion.cs index 229a47452f..41844c6175 100644 --- a/src/Exceptionless.Core/Plugins/EventUpgrader/Default/GetVersion.cs +++ b/src/Exceptionless.Core/Plugins/EventUpgrader/Default/GetVersion.cs @@ -1,7 +1,7 @@ -using Exceptionless.Core.Extensions; +using System.Text.Json.Nodes; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Pipeline; using Microsoft.Extensions.Logging; -using Newtonsoft.Json.Linq; namespace Exceptionless.Core.Plugins.EventUpgrader; @@ -15,14 +15,14 @@ public void Upgrade(EventUpgraderContext ctx) if (ctx.Version is not null) return; - if (ctx.Documents.Count == 0 || !ctx.Documents.First().HasValues) + if (ctx.Documents.Count == 0 || !ctx.Documents.First().HasValues()) { ctx.Version = new Version(); return; } var doc = ctx.Documents.First(); - if (!(doc["ExceptionlessClientInfo"] is JObject { HasValues: true } clientInfo) || clientInfo["Version"] is null) + if (doc is not JsonObject docObj || docObj["ExceptionlessClientInfo"] is not JsonObject { Count: > 0 } clientInfo || clientInfo["Version"] is null) { ctx.Version = new Version(); return; diff --git a/src/Exceptionless.Core/Plugins/EventUpgrader/Default/V1R500_EventUpgrade.cs b/src/Exceptionless.Core/Plugins/EventUpgrader/Default/V1R500_EventUpgrade.cs index 3b69f5bf7f..c3f83320bd 100644 --- a/src/Exceptionless.Core/Plugins/EventUpgrader/Default/V1R500_EventUpgrade.cs +++ b/src/Exceptionless.Core/Plugins/EventUpgrader/Default/V1R500_EventUpgrade.cs @@ -1,6 +1,6 @@ -using Exceptionless.Core.Pipeline; +using System.Text.Json.Nodes; +using Exceptionless.Core.Pipeline; using Microsoft.Extensions.Logging; -using Newtonsoft.Json.Linq; namespace Exceptionless.Core.Plugins.EventUpgrader; @@ -19,14 +19,15 @@ public void Upgrade(EventUpgraderContext ctx) foreach (var doc in ctx.Documents) { - if (!(doc["ExceptionlessClientInfo"] is JObject clientInfo) || !clientInfo.HasValues || clientInfo["InstallDate"] is null) + if (doc is not JsonObject docObj || docObj["ExceptionlessClientInfo"] is not JsonObject { Count: > 0 } clientInfo || clientInfo["InstallDate"] is null) return; // This shouldn't hurt using DateTimeOffset to try and parse a date. It insures you won't lose any info. - if (DateTimeOffset.TryParse(clientInfo["InstallDate"]!.ToString(), out var date)) + if (DateTimeOffset.TryParse(clientInfo["InstallDate"]?.ToString(), out var date)) { clientInfo.Remove("InstallDate"); - clientInfo.Add("InstallDate", new JValue(date)); + // Format date as ISO 8601 with offset (matching Newtonsoft behavior) + clientInfo.Add("InstallDate", JsonValue.Create(date.ToString("yyyy-MM-ddTHH:mm:ss.FFFFFFFzzz"))); } else { diff --git a/src/Exceptionless.Core/Plugins/EventUpgrader/Default/V1R844_EventUpgrade.cs b/src/Exceptionless.Core/Plugins/EventUpgrader/Default/V1R844_EventUpgrade.cs index d2db4115c7..5defe8055b 100644 --- a/src/Exceptionless.Core/Plugins/EventUpgrader/Default/V1R844_EventUpgrade.cs +++ b/src/Exceptionless.Core/Plugins/EventUpgrader/Default/V1R844_EventUpgrade.cs @@ -1,6 +1,6 @@ -using Exceptionless.Core.Pipeline; +using System.Text.Json.Nodes; +using Exceptionless.Core.Pipeline; using Microsoft.Extensions.Logging; -using Newtonsoft.Json.Linq; namespace Exceptionless.Core.Plugins.EventUpgrader; @@ -16,26 +16,22 @@ public void Upgrade(EventUpgraderContext ctx) foreach (var doc in ctx.Documents) { - - if (!(doc["RequestInfo"] is JObject { HasValues: true } requestInfo)) + if (doc is not JsonObject docObj || docObj["RequestInfo"] is not JsonObject { Count: > 0 } requestInfo) return; - if (requestInfo["Cookies"] is not null && requestInfo["Cookies"]!.HasValues) + if (requestInfo["Cookies"] is JsonObject { Count: > 0 } cookies) { - if (requestInfo["Cookies"] is JObject cookies) - cookies.Remove(""); + cookies.Remove(""); } - if (requestInfo["Form"] is not null && requestInfo["Form"]!.HasValues) + if (requestInfo["Form"] is JsonObject { Count: > 0 } form) { - if (requestInfo["Form"] is JObject form) - form.Remove(""); + form.Remove(""); } - if (requestInfo["QueryString"] is not null && requestInfo["QueryString"]!.HasValues) + if (requestInfo["QueryString"] is JsonObject { Count: > 0 } queryString) { - if (requestInfo["QueryString"] is JObject queryString) - queryString.Remove(""); + queryString.Remove(""); } } } diff --git a/src/Exceptionless.Core/Plugins/EventUpgrader/Default/V1R850_EventUpgrade.cs b/src/Exceptionless.Core/Plugins/EventUpgrader/Default/V1R850_EventUpgrade.cs index 424ef62010..86596e640f 100644 --- a/src/Exceptionless.Core/Plugins/EventUpgrader/Default/V1R850_EventUpgrade.cs +++ b/src/Exceptionless.Core/Plugins/EventUpgrader/Default/V1R850_EventUpgrade.cs @@ -1,7 +1,7 @@ -using Exceptionless.Core.Extensions; +using System.Text.Json.Nodes; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Pipeline; using Microsoft.Extensions.Logging; -using Newtonsoft.Json.Linq; namespace Exceptionless.Core.Plugins.EventUpgrader; @@ -15,12 +15,12 @@ public void Upgrade(EventUpgraderContext ctx) if (ctx.Version > new Version(1, 0, 0, 850)) return; - foreach (var doc in ctx.Documents.OfType()) + foreach (var doc in ctx.Documents.OfType()) { var current = doc; while (current is not null) { - if (doc["ExtendedData"] is JObject extendedData) + if (doc["ExtendedData"] is JsonObject extendedData) { if (extendedData["ExtraExceptionProperties"] is not null) extendedData.Rename("ExtraExceptionProperties", "__ExceptionInfo"); @@ -32,7 +32,7 @@ public void Upgrade(EventUpgraderContext ctx) extendedData.Rename("TraceInfo", "TraceLog"); } - current = current["Inner"] as JObject; + current = current["Inner"] as JsonObject; } } } diff --git a/src/Exceptionless.Core/Plugins/EventUpgrader/Default/V2_EventUpgrade.cs b/src/Exceptionless.Core/Plugins/EventUpgrader/Default/V2_EventUpgrade.cs index 5611ccb5b4..fdf43c9f78 100644 --- a/src/Exceptionless.Core/Plugins/EventUpgrader/Default/V2_EventUpgrade.cs +++ b/src/Exceptionless.Core/Plugins/EventUpgrader/Default/V2_EventUpgrade.cs @@ -1,8 +1,9 @@ -using Exceptionless.Core.Extensions; +using System.Text.Json; +using System.Text.Json.Nodes; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Models.Data; using Exceptionless.Core.Pipeline; using Microsoft.Extensions.Logging; -using Newtonsoft.Json.Linq; namespace Exceptionless.Core.Plugins.EventUpgrader; @@ -16,7 +17,7 @@ public void Upgrade(EventUpgraderContext ctx) if (ctx.Version > new Version(2, 0)) return; - foreach (var doc in ctx.Documents.OfType()) + foreach (var doc in ctx.Documents.OfType()) { bool isNotFound = doc.GetPropertyStringValue("Code") == "404"; @@ -36,15 +37,18 @@ public void Upgrade(EventUpgraderContext ctx) doc.Remove("ExceptionlessClientInfo"); if (!doc.RemoveIfNullOrEmpty("Tags")) { - var tags = doc.GetValue("Tags"); - if (tags is not null && tags.Type == JTokenType.Array) + var tags = doc["Tags"]; + if (tags is JsonArray tagsArray) { - foreach (var tag in tags.ToList()) + var tagsToRemove = new List(); + foreach (var tag in tagsArray) { - string t = tag.ToString(); + string? t = tag?.ToString(); if (String.IsNullOrEmpty(t) || t.Length > 255) - tag.Remove(); + tagsToRemove.Add(tag); } + foreach (var tag in tagsToRemove) + tagsArray.Remove(tag); } } @@ -58,7 +62,7 @@ public void Upgrade(EventUpgraderContext ctx) doc.RenameAll("ExtendedData", "Data"); - var extendedData = doc.Property("Data") is not null ? doc.Property("Data")!.Value as JObject : null; + var extendedData = doc["Data"] as JsonObject; if (extendedData is not null) { if (!isNotFound) @@ -73,58 +77,62 @@ public void Upgrade(EventUpgraderContext ctx) if (extendedData?["__ExceptionInfo"] is not null) extendedData.Remove("__ExceptionInfo"); - doc.Add("Type", new JValue("404")); + doc.Add("Type", JsonValue.Create("404")); } else { - var error = new JObject(); + var error = new JsonObject(); if (!doc.RemoveIfNullOrEmpty("Message")) - error.Add("Message", doc["Message"]!.Value()); + { + var messageValue = doc["Message"]?.GetValue(); + if (messageValue is not null) + error.Add("Message", JsonValue.Create(messageValue)); + } error.MoveOrRemoveIfNullOrEmpty(doc, "Code", "Type", "Inner", "StackTrace", "TargetMethod", "Modules"); // Copy the exception info from root extended data to the current errors extended data. if (extendedData?["__ExceptionInfo"] is not null) { - error.Add("Data", new JObject()); - ((JObject)error["Data"]!).MoveOrRemoveIfNullOrEmpty(extendedData, "__ExceptionInfo"); + error.Add("Data", new JsonObject()); + ((JsonObject)error["Data"]!).MoveOrRemoveIfNullOrEmpty(extendedData, "__ExceptionInfo"); } - string? id = doc["Id"]?.Value(); + string? id = doc["Id"]?.GetValue(); RenameAndValidateExtraExceptionProperties(id, error); - var inner = error["Inner"] as JObject; + var inner = error["Inner"] as JsonObject; while (inner is not null) { RenameAndValidateExtraExceptionProperties(id, inner); - inner = inner["Inner"] as JObject; + inner = inner["Inner"] as JsonObject; } - doc.Add("Type", new JValue(isNotFound ? "404" : "error")); + doc.Add("Type", JsonValue.Create(isNotFound ? "404" : "error")); doc.Add("@error", error); } string? emailAddress = doc.GetPropertyStringValueAndRemove("UserEmail"); string? userDescription = doc.GetPropertyStringValueAndRemove("UserDescription"); if (!String.IsNullOrWhiteSpace(emailAddress) && !String.IsNullOrWhiteSpace(userDescription)) - doc.Add("@user_description", JObject.FromObject(new UserDescription(emailAddress, userDescription))); + doc.Add("@user_description", JsonSerializer.SerializeToNode(new UserDescription(emailAddress, userDescription))); string? identity = doc.GetPropertyStringValueAndRemove("UserName"); if (!String.IsNullOrWhiteSpace(identity)) - doc.Add("@user", JObject.FromObject(new UserInfo(identity))); + doc.Add("@user", JsonSerializer.SerializeToNode(new UserInfo(identity))); doc.RemoveAllIfNullOrEmpty("Data", "GenericArguments", "Parameters"); } } - private void RenameAndValidateExtraExceptionProperties(string? id, JObject error) + private void RenameAndValidateExtraExceptionProperties(string? id, JsonObject error) { - var extendedData = error?["Data"] as JObject; + var extendedData = error["Data"] as JsonObject; if (extendedData?["__ExceptionInfo"] is null) return; - string json = extendedData["__ExceptionInfo"]!.ToString(); + string? json = extendedData["__ExceptionInfo"]?.ToString(); extendedData.Remove("__ExceptionInfo"); if (String.IsNullOrWhiteSpace(json)) @@ -136,20 +144,25 @@ private void RenameAndValidateExtraExceptionProperties(string? id, JObject error return; } - var ext = new JObject(); + var ext = new JsonObject(); try { - var extraProperties = JObject.Parse(json); - foreach (var property in extraProperties.Properties()) + var extraProperties = JsonNode.Parse(json) as JsonObject; + if (extraProperties is not null) { - if (property.IsNullOrEmpty()) - continue; + foreach (var property in extraProperties.ToList()) + { + if (property.Value.IsNullOrEmpty()) + continue; - string dataKey = property.Name; - if (extendedData[dataKey] is not null) - dataKey = "_" + dataKey; + string dataKey = property.Key; + if (extendedData[dataKey] is not null) + dataKey = "_" + dataKey; - ext.Add(dataKey, property.Value); + // Need to detach the node before adding to another parent + extraProperties.Remove(property.Key); + ext.Add(dataKey, property.Value); + } } } catch (Exception) { } diff --git a/src/Exceptionless.Core/Plugins/EventUpgrader/EventUpgraderContext.cs b/src/Exceptionless.Core/Plugins/EventUpgrader/EventUpgraderContext.cs index 1d51dd4e3d..70eefc6e3d 100644 --- a/src/Exceptionless.Core/Plugins/EventUpgrader/EventUpgraderContext.cs +++ b/src/Exceptionless.Core/Plugins/EventUpgrader/EventUpgraderContext.cs @@ -1,7 +1,6 @@ -using Exceptionless.Core.Extensions; +using System.Text.Json.Nodes; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Utility; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; namespace Exceptionless.Core.Plugins.EventUpgrader; @@ -12,15 +11,15 @@ public EventUpgraderContext(string json, Version? version = null, bool isMigrati var jsonType = json.GetJsonType(); if (jsonType == JsonType.Object) { - var doc = JsonConvert.DeserializeObject(json); + var doc = JsonNode.Parse(json) as JsonObject; if (doc is not null) - Documents = new JArray(doc); + Documents = new JsonArray(doc); else throw new ArgumentException("Invalid json object specified", nameof(json)); } else if (jsonType == JsonType.Array) { - var docs = JsonConvert.DeserializeObject(json); + var docs = JsonNode.Parse(json) as JsonArray; if (docs is not null) Documents = docs; else @@ -35,21 +34,21 @@ public EventUpgraderContext(string json, Version? version = null, bool isMigrati IsMigration = isMigration; } - public EventUpgraderContext(JObject doc, Version? version = null, bool isMigration = false) + public EventUpgraderContext(JsonObject doc, Version? version = null, bool isMigration = false) { - Documents = new JArray(doc); + Documents = new JsonArray(doc); Version = version; IsMigration = isMigration; } - public EventUpgraderContext(JArray docs, Version? version = null, bool isMigration = false) + public EventUpgraderContext(JsonArray docs, Version? version = null, bool isMigration = false) { Documents = docs; Version = version; IsMigration = isMigration; } - public JArray Documents { get; set; } + public JsonArray Documents { get; set; } public Version? Version { get; set; } public bool IsMigration { get; set; } } diff --git a/src/Exceptionless.Core/Plugins/Formatting/Default/05_ManualStackingFormattingPlugin.cs b/src/Exceptionless.Core/Plugins/Formatting/Default/05_ManualStackingFormattingPlugin.cs index 3f3d70dbf5..00b0e53dbf 100644 --- a/src/Exceptionless.Core/Plugins/Formatting/Default/05_ManualStackingFormattingPlugin.cs +++ b/src/Exceptionless.Core/Plugins/Formatting/Default/05_ManualStackingFormattingPlugin.cs @@ -1,6 +1,6 @@ -using System.Text.Json; -using Exceptionless.Core.Models; +using Exceptionless.Core.Models; using Exceptionless.Core.Pipeline; +using Foundatio.Serializer; using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Plugins.Formatting; @@ -8,11 +8,11 @@ namespace Exceptionless.Core.Plugins.Formatting; [Priority(5)] public sealed class ManualStackingFormattingPlugin : FormattingPluginBase { - public ManualStackingFormattingPlugin(JsonSerializerOptions jsonOptions, AppOptions options, ILoggerFactory loggerFactory) : base(jsonOptions, options, loggerFactory) { } + public ManualStackingFormattingPlugin(ITextSerializer serializer, AppOptions options, ILoggerFactory loggerFactory) : base(serializer, options, loggerFactory) { } public override string? GetStackTitle(PersistentEvent ev) { - var msi = ev.GetManualStackingInfo(_jsonOptions); + var msi = ev.GetManualStackingInfo(_serializer); return !String.IsNullOrWhiteSpace(msi?.Title) ? msi.Title : null; } } diff --git a/src/Exceptionless.Core/Plugins/Formatting/Default/10_SimpleErrorFormattingPlugin.cs b/src/Exceptionless.Core/Plugins/Formatting/Default/10_SimpleErrorFormattingPlugin.cs index 8b79ac1c28..f45efd0921 100644 --- a/src/Exceptionless.Core/Plugins/Formatting/Default/10_SimpleErrorFormattingPlugin.cs +++ b/src/Exceptionless.Core/Plugins/Formatting/Default/10_SimpleErrorFormattingPlugin.cs @@ -1,7 +1,7 @@ -using System.Text.Json; -using Exceptionless.Core.Extensions; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Pipeline; +using Foundatio.Serializer; using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Plugins.Formatting; @@ -9,7 +9,7 @@ namespace Exceptionless.Core.Plugins.Formatting; [Priority(10)] public sealed class SimpleErrorFormattingPlugin : FormattingPluginBase { - public SimpleErrorFormattingPlugin(JsonSerializerOptions jsonOptions, AppOptions options, ILoggerFactory loggerFactory) : base(jsonOptions, options, loggerFactory) { } + public SimpleErrorFormattingPlugin(ITextSerializer serializer, AppOptions options, ILoggerFactory loggerFactory) : base(serializer, options, loggerFactory) { } private bool ShouldHandle(PersistentEvent ev) { @@ -39,7 +39,7 @@ private bool ShouldHandle(PersistentEvent ev) if (!ShouldHandle(ev)) return null; - var error = ev.GetSimpleError(_jsonOptions); + var error = ev.GetSimpleError(_serializer); return error?.Message; } @@ -48,12 +48,12 @@ private bool ShouldHandle(PersistentEvent ev) if (!ShouldHandle(ev)) return null; - var error = ev.GetSimpleError(_jsonOptions); + var error = ev.GetSimpleError(_serializer); if (error is null) return null; var data = new Dictionary { { "Message", ev.Message } }; - AddUserIdentitySummaryData(data, ev.GetUserIdentity(_jsonOptions)); + AddUserIdentitySummaryData(data, ev.GetUserIdentity(_serializer)); if (!String.IsNullOrEmpty(error.Type)) { @@ -61,7 +61,7 @@ private bool ShouldHandle(PersistentEvent ev) data.Add("TypeFullName", error.Type); } - var requestInfo = ev.GetRequestInfo(_jsonOptions); + var requestInfo = ev.GetRequestInfo(_serializer); if (!String.IsNullOrEmpty(requestInfo?.Path)) data.Add("Path", requestInfo.Path); @@ -73,7 +73,7 @@ private bool ShouldHandle(PersistentEvent ev) if (!ShouldHandle(ev)) return null; - var error = ev.GetSimpleError(_jsonOptions); + var error = ev.GetSimpleError(_serializer); if (error is null) return null; @@ -96,7 +96,7 @@ private bool ShouldHandle(PersistentEvent ev) if (!String.IsNullOrEmpty(errorTypeName)) data.Add("Type", errorTypeName); - var requestInfo = ev.GetRequestInfo(_jsonOptions); + var requestInfo = ev.GetRequestInfo(_serializer); if (requestInfo is not null) data.Add("Url", requestInfo.GetFullPath(true, true, true)); @@ -108,7 +108,7 @@ private bool ShouldHandle(PersistentEvent ev) if (!ShouldHandle(ev)) return null; - var error = ev.GetSimpleError(_jsonOptions); + var error = ev.GetSimpleError(_serializer); if (error is null) return null; @@ -126,7 +126,7 @@ private bool ShouldHandle(PersistentEvent ev) if (isCritical) notificationType = String.Concat("critical ", notificationType); - var attachment = new SlackMessage.SlackAttachment(ev, _jsonOptions) + var attachment = new SlackMessage.SlackAttachment(ev, _serializer) { Color = "#BB423F", Fields = diff --git a/src/Exceptionless.Core/Plugins/Formatting/Default/20_ErrorFormattingPlugin.cs b/src/Exceptionless.Core/Plugins/Formatting/Default/20_ErrorFormattingPlugin.cs index 250bdf945c..ac0e0e0710 100644 --- a/src/Exceptionless.Core/Plugins/Formatting/Default/20_ErrorFormattingPlugin.cs +++ b/src/Exceptionless.Core/Plugins/Formatting/Default/20_ErrorFormattingPlugin.cs @@ -1,7 +1,7 @@ -using System.Text.Json; -using Exceptionless.Core.Extensions; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Pipeline; +using Foundatio.Serializer; using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Plugins.Formatting; @@ -9,7 +9,7 @@ namespace Exceptionless.Core.Plugins.Formatting; [Priority(20)] public sealed class ErrorFormattingPlugin : FormattingPluginBase { - public ErrorFormattingPlugin(JsonSerializerOptions jsonOptions, AppOptions options, ILoggerFactory loggerFactory) : base(jsonOptions, options, loggerFactory) { } + public ErrorFormattingPlugin(ITextSerializer serializer, AppOptions options, ILoggerFactory loggerFactory) : base(serializer, options, loggerFactory) { } private bool ShouldHandle(PersistentEvent ev) { @@ -21,7 +21,7 @@ private bool ShouldHandle(PersistentEvent ev) if (!ShouldHandle(ev)) return null; - var error = ev.GetError(_jsonOptions); + var error = ev.GetError(_serializer); return error?.Message; } @@ -59,12 +59,12 @@ private bool ShouldHandle(PersistentEvent ev) if (!ShouldHandle(ev)) return null; - var stackingTarget = ev.GetStackingTarget(_jsonOptions); + var stackingTarget = ev.GetStackingTarget(_serializer); if (stackingTarget?.Error is null) return null; var data = new Dictionary { { "Message", ev.Message } }; - AddUserIdentitySummaryData(data, ev.GetUserIdentity(_jsonOptions)); + AddUserIdentitySummaryData(data, ev.GetUserIdentity(_serializer)); if (!String.IsNullOrEmpty(stackingTarget.Error.Type)) { @@ -78,7 +78,7 @@ private bool ShouldHandle(PersistentEvent ev) data.Add("MethodFullName", stackingTarget.Method.GetFullName()); } - var requestInfo = ev.GetRequestInfo(_jsonOptions); + var requestInfo = ev.GetRequestInfo(_serializer); if (!String.IsNullOrEmpty(requestInfo?.Path)) data.Add("Path", requestInfo.Path); @@ -90,7 +90,7 @@ private bool ShouldHandle(PersistentEvent ev) if (!ShouldHandle(ev)) return null; - var error = ev.GetError(_jsonOptions); + var error = ev.GetError(_serializer); var stackingTarget = error?.GetStackingTarget(); if (stackingTarget?.Error is null) return null; @@ -117,7 +117,7 @@ private bool ShouldHandle(PersistentEvent ev) if (stackingTarget.Method?.Name is not null) data.Add("Method", stackingTarget.Method.Name.Truncate(60)); - var requestInfo = ev.GetRequestInfo(_jsonOptions); + var requestInfo = ev.GetRequestInfo(_serializer); if (requestInfo is not null) data.Add("Url", requestInfo.GetFullPath(true, true, true)); @@ -129,7 +129,7 @@ private bool ShouldHandle(PersistentEvent ev) if (!ShouldHandle(ev)) return null; - var error = ev.GetError(_jsonOptions); + var error = ev.GetError(_serializer); var stackingTarget = error?.GetStackingTarget(); if (stackingTarget?.Error is null) return null; @@ -148,7 +148,7 @@ private bool ShouldHandle(PersistentEvent ev) if (isCritical) notificationType = String.Concat("critical ", notificationType); - var attachment = new SlackMessage.SlackAttachment(ev, _jsonOptions) + var attachment = new SlackMessage.SlackAttachment(ev, _serializer) { Color = "#BB423F", Fields = diff --git a/src/Exceptionless.Core/Plugins/Formatting/Default/30_NotFoundFormattingPlugin.cs b/src/Exceptionless.Core/Plugins/Formatting/Default/30_NotFoundFormattingPlugin.cs index c3f602bf24..bc628e8a8f 100644 --- a/src/Exceptionless.Core/Plugins/Formatting/Default/30_NotFoundFormattingPlugin.cs +++ b/src/Exceptionless.Core/Plugins/Formatting/Default/30_NotFoundFormattingPlugin.cs @@ -1,7 +1,7 @@ -using System.Text.Json; -using Exceptionless.Core.Extensions; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Pipeline; +using Foundatio.Serializer; using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Plugins.Formatting; @@ -9,7 +9,7 @@ namespace Exceptionless.Core.Plugins.Formatting; [Priority(30)] public sealed class NotFoundFormattingPlugin : FormattingPluginBase { - public NotFoundFormattingPlugin(JsonSerializerOptions jsonOptions, AppOptions options, ILoggerFactory loggerFactory) : base(jsonOptions, options, loggerFactory) { } + public NotFoundFormattingPlugin(ITextSerializer serializer, AppOptions options, ILoggerFactory loggerFactory) : base(serializer, options, loggerFactory) { } private bool ShouldHandle(PersistentEvent ev) { @@ -38,9 +38,9 @@ private bool ShouldHandle(PersistentEvent ev) return null; var data = new Dictionary { { "Source", ev.Source } }; - AddUserIdentitySummaryData(data, ev.GetUserIdentity(_jsonOptions)); + AddUserIdentitySummaryData(data, ev.GetUserIdentity(_serializer)); - var ips = ev.GetIpAddresses(_jsonOptions).ToList(); + var ips = ev.GetIpAddresses(_serializer).ToList(); if (ips.Count > 0) data.Add("IpAddress", ips); @@ -62,7 +62,7 @@ private bool ShouldHandle(PersistentEvent ev) notificationType = String.Concat("Critical ", notificationType.ToLowerInvariant()); string subject = String.Concat(notificationType, ": ", ev.Source).Truncate(120); - var requestInfo = ev.GetRequestInfo(_jsonOptions); + var requestInfo = ev.GetRequestInfo(_serializer); var data = new Dictionary { { "Url", requestInfo?.GetFullPath(true, true, true) ?? ev.Source?.Truncate(60) } }; @@ -84,8 +84,8 @@ private bool ShouldHandle(PersistentEvent ev) if (isCritical) notificationType = String.Concat("critical ", notificationType); - var requestInfo = ev.GetRequestInfo(_jsonOptions); - var attachment = new SlackMessage.SlackAttachment(ev, _jsonOptions) + var requestInfo = ev.GetRequestInfo(_serializer); + var attachment = new SlackMessage.SlackAttachment(ev, _serializer) { Color = "#BB423F", Fields = diff --git a/src/Exceptionless.Core/Plugins/Formatting/Default/40_UsageFormattingPlugin.cs b/src/Exceptionless.Core/Plugins/Formatting/Default/40_UsageFormattingPlugin.cs index 6b02b61052..e7c0e0d459 100644 --- a/src/Exceptionless.Core/Plugins/Formatting/Default/40_UsageFormattingPlugin.cs +++ b/src/Exceptionless.Core/Plugins/Formatting/Default/40_UsageFormattingPlugin.cs @@ -1,7 +1,7 @@ -using System.Text.Json; -using Exceptionless.Core.Extensions; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Pipeline; +using Foundatio.Serializer; using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Plugins.Formatting; @@ -9,7 +9,7 @@ namespace Exceptionless.Core.Plugins.Formatting; [Priority(40)] public sealed class UsageFormattingPlugin : FormattingPluginBase { - public UsageFormattingPlugin(JsonSerializerOptions jsonOptions, AppOptions options, ILoggerFactory loggerFactory) : base(jsonOptions, options, loggerFactory) { } + public UsageFormattingPlugin(ITextSerializer serializer, AppOptions options, ILoggerFactory loggerFactory) : base(serializer, options, loggerFactory) { } private bool ShouldHandle(PersistentEvent ev) { @@ -38,7 +38,7 @@ private bool ShouldHandle(PersistentEvent ev) return null; var data = new Dictionary { { "Source", ev.Source } }; - AddUserIdentitySummaryData(data, ev.GetUserIdentity(_jsonOptions)); + AddUserIdentitySummaryData(data, ev.GetUserIdentity(_serializer)); return new SummaryData { Id = ev.Id, TemplateKey = "event-feature-summary", Data = data }; } @@ -61,7 +61,7 @@ private bool ShouldHandle(PersistentEvent ev) if (!ShouldHandle(ev)) return null; - var attachment = new SlackMessage.SlackAttachment(ev, _jsonOptions) + var attachment = new SlackMessage.SlackAttachment(ev, _serializer) { Fields = [ diff --git a/src/Exceptionless.Core/Plugins/Formatting/Default/50_SessionFormattingPlugin.cs b/src/Exceptionless.Core/Plugins/Formatting/Default/50_SessionFormattingPlugin.cs index 1dc5710829..f97b799c9f 100644 --- a/src/Exceptionless.Core/Plugins/Formatting/Default/50_SessionFormattingPlugin.cs +++ b/src/Exceptionless.Core/Plugins/Formatting/Default/50_SessionFormattingPlugin.cs @@ -1,7 +1,7 @@ -using System.Text.Json; -using Exceptionless.Core.Extensions; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Pipeline; +using Foundatio.Serializer; using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Plugins.Formatting; @@ -9,7 +9,7 @@ namespace Exceptionless.Core.Plugins.Formatting; [Priority(50)] public sealed class SessionFormattingPlugin : FormattingPluginBase { - public SessionFormattingPlugin(JsonSerializerOptions jsonOptions, AppOptions options, ILoggerFactory loggerFactory) : base(jsonOptions, options, loggerFactory) { } + public SessionFormattingPlugin(ITextSerializer serializer, AppOptions options, ILoggerFactory loggerFactory) : base(serializer, options, loggerFactory) { } private bool ShouldHandle(PersistentEvent ev) { @@ -41,7 +41,7 @@ private bool ShouldHandle(PersistentEvent ev) return null; var data = new Dictionary { { "SessionId", ev.GetSessionId() }, { "Type", ev.Type } }; - AddUserIdentitySummaryData(data, ev.GetUserIdentity(_jsonOptions)); + AddUserIdentitySummaryData(data, ev.GetUserIdentity(_serializer)); if (ev.IsSessionStart()) { diff --git a/src/Exceptionless.Core/Plugins/Formatting/Default/60_LogFormattingPlugin.cs b/src/Exceptionless.Core/Plugins/Formatting/Default/60_LogFormattingPlugin.cs index 3eca0c290c..faa20ef2d2 100644 --- a/src/Exceptionless.Core/Plugins/Formatting/Default/60_LogFormattingPlugin.cs +++ b/src/Exceptionless.Core/Plugins/Formatting/Default/60_LogFormattingPlugin.cs @@ -1,7 +1,7 @@ -using System.Text.Json; -using Exceptionless.Core.Extensions; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Pipeline; +using Foundatio.Serializer; using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Plugins.Formatting; @@ -9,7 +9,7 @@ namespace Exceptionless.Core.Plugins.Formatting; [Priority(60)] public sealed class LogFormattingPlugin : FormattingPluginBase { - public LogFormattingPlugin(JsonSerializerOptions jsonOptions, AppOptions options, ILoggerFactory loggerFactory) : base(jsonOptions, options, loggerFactory) { } + public LogFormattingPlugin(ITextSerializer serializer, AppOptions options, ILoggerFactory loggerFactory) : base(serializer, options, loggerFactory) { } private bool ShouldHandle(PersistentEvent ev) { @@ -50,7 +50,7 @@ private bool ShouldHandle(PersistentEvent ev) return null; var data = new Dictionary { { "Message", ev.Message } }; - AddUserIdentitySummaryData(data, ev.GetUserIdentity(_jsonOptions)); + AddUserIdentitySummaryData(data, ev.GetUserIdentity(_serializer)); if (!String.IsNullOrWhiteSpace(ev.Source)) { @@ -92,7 +92,7 @@ private bool ShouldHandle(PersistentEvent ev) if (!String.IsNullOrEmpty(level)) data.Add("Level", level.Truncate(60)); - var requestInfo = ev.GetRequestInfo(_jsonOptions); + var requestInfo = ev.GetRequestInfo(_serializer); if (requestInfo is not null) data.Add("Url", requestInfo.GetFullPath(true, true, true)); @@ -114,7 +114,7 @@ private bool ShouldHandle(PersistentEvent ev) notificationType = String.Concat("critical ", notificationType); string source = !String.IsNullOrEmpty(ev.Source) ? ev.Source : "(Global)"; - var attachment = new SlackMessage.SlackAttachment(ev, _jsonOptions) + var attachment = new SlackMessage.SlackAttachment(ev, _serializer) { Fields = [ @@ -149,7 +149,7 @@ private bool ShouldHandle(PersistentEvent ev) attachment.Fields.Add(new SlackMessage.SlackAttachmentFields { Title = "Level", Value = level.Truncate(60) }); } - var requestInfo = ev.GetRequestInfo(_jsonOptions); + var requestInfo = ev.GetRequestInfo(_serializer); if (requestInfo is not null) attachment.Fields.Add(new SlackMessage.SlackAttachmentFields { Title = "Url", Value = requestInfo.GetFullPath(true, true, true) }); diff --git a/src/Exceptionless.Core/Plugins/Formatting/Default/99_DefaultFormattingPlugin.cs b/src/Exceptionless.Core/Plugins/Formatting/Default/99_DefaultFormattingPlugin.cs index 39ec6593e9..8f6398ac90 100644 --- a/src/Exceptionless.Core/Plugins/Formatting/Default/99_DefaultFormattingPlugin.cs +++ b/src/Exceptionless.Core/Plugins/Formatting/Default/99_DefaultFormattingPlugin.cs @@ -1,7 +1,7 @@ -using System.Text.Json; -using Exceptionless.Core.Extensions; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Pipeline; +using Foundatio.Serializer; using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Plugins.Formatting; @@ -9,7 +9,7 @@ namespace Exceptionless.Core.Plugins.Formatting; [Priority(99)] public sealed class DefaultFormattingPlugin : FormattingPluginBase { - public DefaultFormattingPlugin(JsonSerializerOptions jsonOptions, AppOptions options, ILoggerFactory loggerFactory) : base(jsonOptions, options, loggerFactory) { } + public DefaultFormattingPlugin(ITextSerializer serializer, AppOptions options, ILoggerFactory loggerFactory) : base(serializer, options, loggerFactory) { } public override string GetStackTitle(PersistentEvent ev) { @@ -37,7 +37,7 @@ public override SummaryData GetEventSummaryData(PersistentEvent ev) { "Type", ev.Type } }; - AddUserIdentitySummaryData(data, ev.GetUserIdentity(_jsonOptions)); + AddUserIdentitySummaryData(data, ev.GetUserIdentity(_serializer)); return new SummaryData { Id = ev.Id, TemplateKey = "event-summary", Data = data }; } @@ -68,7 +68,7 @@ public override MailMessageData GetEventNotificationMailMessageData(PersistentEv if (!String.IsNullOrEmpty(ev.Source)) data.Add("Source", ev.Source.Truncate(60)); - var requestInfo = ev.GetRequestInfo(_jsonOptions); + var requestInfo = ev.GetRequestInfo(_serializer); if (requestInfo is not null) data.Add("Url", requestInfo.GetFullPath(true, true, true)); @@ -90,7 +90,7 @@ public override SlackMessage GetSlackEventNotification(PersistentEvent ev, Proje if (isCritical) notificationType = String.Concat("Critical ", notificationType.ToLowerInvariant()); - var attachment = new SlackMessage.SlackAttachment(ev, _jsonOptions); + var attachment = new SlackMessage.SlackAttachment(ev, _serializer); if (!String.IsNullOrEmpty(ev.Message)) attachment.Fields.Add(new SlackMessage.SlackAttachmentFields { Title = "Message", Value = ev.Message.Truncate(60) }); diff --git a/src/Exceptionless.Core/Plugins/Formatting/FormattingPluginBase.cs b/src/Exceptionless.Core/Plugins/Formatting/FormattingPluginBase.cs index 926a83eb95..f5cb2a3791 100644 --- a/src/Exceptionless.Core/Plugins/Formatting/FormattingPluginBase.cs +++ b/src/Exceptionless.Core/Plugins/Formatting/FormattingPluginBase.cs @@ -1,18 +1,18 @@ -using System.Text.Json; -using Exceptionless.Core.Extensions; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Models.Data; +using Foundatio.Serializer; using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Plugins.Formatting; public abstract class FormattingPluginBase : PluginBase, IFormattingPlugin { - protected readonly JsonSerializerOptions _jsonOptions; + protected readonly ITextSerializer _serializer; - public FormattingPluginBase(JsonSerializerOptions jsonOptions, AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) + public FormattingPluginBase(ITextSerializer serializer, AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) { - _jsonOptions = jsonOptions; + _serializer = serializer; } public virtual SummaryData? GetStackSummaryData(Stack stack) @@ -42,7 +42,7 @@ public FormattingPluginBase(JsonSerializerOptions jsonOptions, AppOptions option protected void AddDefaultSlackFields(PersistentEvent ev, List attachmentFields, bool includeUrl = true) { - var requestInfo = ev.GetRequestInfo(_jsonOptions); + var requestInfo = ev.GetRequestInfo(_serializer); if (requestInfo is not null && includeUrl) attachmentFields.Add(new SlackMessage.SlackAttachmentFields { Title = "Url", Value = requestInfo.GetFullPath(true, true, true) }); diff --git a/src/Exceptionless.Core/Plugins/WebHook/Default/005_SlackPlugin.cs b/src/Exceptionless.Core/Plugins/WebHook/Default/005_SlackPlugin.cs index 45c031a826..2d4dc04ec5 100644 --- a/src/Exceptionless.Core/Plugins/WebHook/Default/005_SlackPlugin.cs +++ b/src/Exceptionless.Core/Plugins/WebHook/Default/005_SlackPlugin.cs @@ -1,6 +1,6 @@ -using System.Text.Json; -using Exceptionless.Core.Pipeline; +using Exceptionless.Core.Pipeline; using Exceptionless.Core.Plugins.Formatting; +using Foundatio.Serializer; using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Plugins.WebHook; @@ -9,12 +9,12 @@ namespace Exceptionless.Core.Plugins.WebHook; public sealed class SlackPlugin : WebHookDataPluginBase { private readonly FormattingPluginManager _pluginManager; - private readonly JsonSerializerOptions _jsonOptions; + private readonly ITextSerializer _serializer; - public SlackPlugin(FormattingPluginManager pluginManager, JsonSerializerOptions jsonOptions, AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) + public SlackPlugin(FormattingPluginManager pluginManager, ITextSerializer serializer, AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) { _pluginManager = pluginManager; - _jsonOptions = jsonOptions; + _serializer = serializer; } public override Task CreateFromEventAsync(WebHookDataContext ctx) @@ -22,7 +22,7 @@ public SlackPlugin(FormattingPluginManager pluginManager, JsonSerializerOptions if (String.IsNullOrEmpty(ctx.WebHook.Url) || !ctx.WebHook.Url.EndsWith("/slack")) return Task.FromResult(null); - var error = ctx.Event?.GetError(_jsonOptions); + var error = ctx.Event?.GetError(_serializer); if (error is null) { ctx.IsCancelled = true; diff --git a/src/Exceptionless.Core/Plugins/WebHook/Default/010_VersionOnePlugin.cs b/src/Exceptionless.Core/Plugins/WebHook/Default/010_VersionOnePlugin.cs index c845dcd872..3ed402c562 100644 --- a/src/Exceptionless.Core/Plugins/WebHook/Default/010_VersionOnePlugin.cs +++ b/src/Exceptionless.Core/Plugins/WebHook/Default/010_VersionOnePlugin.cs @@ -1,7 +1,8 @@ -using System.Text.Json; +using System.Text.Json.Serialization; using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Pipeline; +using Foundatio.Serializer; using Microsoft.Extensions.Logging; namespace Exceptionless.Core.Plugins.WebHook; @@ -9,11 +10,11 @@ namespace Exceptionless.Core.Plugins.WebHook; [Priority(10)] public sealed class VersionOnePlugin : WebHookDataPluginBase { - private readonly JsonSerializerOptions _jsonOptions; + private readonly ITextSerializer _serializer; - public VersionOnePlugin(JsonSerializerOptions jsonOptions, AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) + public VersionOnePlugin(ITextSerializer serializer, AppOptions options, ILoggerFactory loggerFactory) : base(options, loggerFactory) { - _jsonOptions = jsonOptions; + _serializer = serializer; } public override Task CreateFromEventAsync(WebHookDataContext ctx) @@ -21,13 +22,13 @@ public VersionOnePlugin(JsonSerializerOptions jsonOptions, AppOptions options, I if (!String.Equals(ctx.WebHook.Version, Models.WebHook.KnownVersions.Version1)) return Task.FromResult(null); - var error = ctx.Event?.GetError(_jsonOptions); + var error = ctx.Event?.GetError(_serializer); if (error is null) return Task.FromResult(null); var ev = ctx.Event!; - var requestInfo = ev.GetRequestInfo(_jsonOptions); - var environmentInfo = ev.GetEnvironmentInfo(_jsonOptions); + var requestInfo = ev.GetRequestInfo(_serializer); + var environmentInfo = ev.GetEnvironmentInfo(_serializer); return Task.FromResult(new VersionOneWebHookEvent(_options.BaseURL) { @@ -97,33 +98,61 @@ public VersionOneWebHookEvent(string baseUrl) _baseUrl = baseUrl; } + [JsonPropertyName("Id")] public string Id { get; init; } = null!; + [JsonPropertyName("Url")] public string Url => String.Concat(_baseUrl, "/event/", Id); + [JsonPropertyName("OccurrenceDate")] public DateTimeOffset OccurrenceDate { get; init; } + [JsonPropertyName("Tags")] public TagSet? Tags { get; init; } = null!; + [JsonPropertyName("MachineName")] public string? MachineName { get; init; } + [JsonPropertyName("RequestPath")] public string? RequestPath { get; init; } + [JsonPropertyName("IpAddress")] public string? IpAddress { get; init; } + [JsonPropertyName("Message")] public string? Message { get; init; } = null!; + [JsonPropertyName("Type")] public string? Type { get; init; } = null!; + [JsonPropertyName("Code")] public string? Code { get; init; } = null!; + [JsonPropertyName("TargetMethod")] public string? TargetMethod { get; init; } + [JsonPropertyName("ProjectId")] public string ProjectId { get; init; } = null!; + [JsonPropertyName("ProjectName")] public string ProjectName { get; init; } = null!; + [JsonPropertyName("OrganizationId")] public string OrganizationId { get; init; } = null!; + [JsonPropertyName("OrganizationName")] public string OrganizationName { get; init; } = null!; + [JsonPropertyName("ErrorStackId")] public string ErrorStackId { get; init; } = null!; + [JsonPropertyName("ErrorStackStatus")] public StackStatus ErrorStackStatus { get; init; } + [JsonPropertyName("ErrorStackUrl")] public string ErrorStackUrl => String.Concat(_baseUrl, "/stack/", ErrorStackId); + [JsonPropertyName("ErrorStackTitle")] public string ErrorStackTitle { get; init; } = null!; + [JsonPropertyName("ErrorStackDescription")] public string? ErrorStackDescription { get; init; } = null!; + [JsonPropertyName("ErrorStackTags")] public TagSet ErrorStackTags { get; init; } = null!; + [JsonPropertyName("TotalOccurrences")] public int TotalOccurrences { get; init; } + [JsonPropertyName("FirstOccurrence")] public DateTime FirstOccurrence { get; init; } + [JsonPropertyName("LastOccurrence")] public DateTime LastOccurrence { get; init; } + [JsonPropertyName("DateFixed")] public DateTime? DateFixed { get; init; } + [JsonPropertyName("IsNew")] public bool IsNew { get; init; } + [JsonPropertyName("IsRegression")] public bool IsRegression { get; init; } + [JsonPropertyName("IsCritical")] public bool IsCritical => Tags is not null && Tags.Contains("Critical"); } @@ -136,26 +165,45 @@ public VersionOneWebHookStack(string baseUrl) _baseUrl = baseUrl; } + [JsonPropertyName("Id")] public string Id { get; init; } = null!; + [JsonPropertyName("Status")] public StackStatus Status { get; init; } + [JsonPropertyName("Url")] public string Url => String.Concat(_baseUrl, "/stack/", Id); + [JsonPropertyName("Title")] public string Title { get; init; } = null!; + [JsonPropertyName("Description")] public string? Description { get; init; } = null!; - + [JsonPropertyName("Tags")] public TagSet Tags { get; init; } = null!; + [JsonPropertyName("RequestPath")] public string? RequestPath { get; init; } + [JsonPropertyName("Type")] public string? Type { get; init; } + [JsonPropertyName("TargetMethod")] public string? TargetMethod { get; init; } + [JsonPropertyName("ProjectId")] public string ProjectId { get; init; } = null!; + [JsonPropertyName("ProjectName")] public string ProjectName { get; init; } = null!; + [JsonPropertyName("OrganizationId")] public string OrganizationId { get; init; } = null!; + [JsonPropertyName("OrganizationName")] public string OrganizationName { get; init; } = null!; + [JsonPropertyName("TotalOccurrences")] public int TotalOccurrences { get; init; } + [JsonPropertyName("FirstOccurrence")] public DateTime FirstOccurrence { get; init; } + [JsonPropertyName("LastOccurrence")] public DateTime LastOccurrence { get; init; } + [JsonPropertyName("DateFixed")] public DateTime? DateFixed { get; init; } + [JsonPropertyName("FixedInVersion")] public string? FixedInVersion { get; init; } + [JsonPropertyName("IsRegression")] public bool IsRegression { get; init; } + [JsonPropertyName("IsCritical")] public bool IsCritical { get; init; } } } diff --git a/src/Exceptionless.Core/Repositories/Configuration/ExceptionlessElasticConfiguration.cs b/src/Exceptionless.Core/Repositories/Configuration/ExceptionlessElasticConfiguration.cs index 4641056b31..5da06ca9ad 100644 --- a/src/Exceptionless.Core/Repositories/Configuration/ExceptionlessElasticConfiguration.cs +++ b/src/Exceptionless.Core/Repositories/Configuration/ExceptionlessElasticConfiguration.cs @@ -14,19 +14,19 @@ using Foundatio.Resilience; using Microsoft.Extensions.Logging; using Nest; -using Newtonsoft.Json; +using System.Text.Json; namespace Exceptionless.Core.Repositories.Configuration; public sealed class ExceptionlessElasticConfiguration : ElasticConfiguration, IStartupAction { private readonly AppOptions _appOptions; - private readonly JsonSerializerSettings _serializerSettings; + private readonly JsonSerializerOptions _jsonSerializerOptions; public ExceptionlessElasticConfiguration( AppOptions appOptions, IQueue workItemQueue, - JsonSerializerSettings serializerSettings, + JsonSerializerOptions jsonSerializerOptions, ICacheClient cacheClient, IMessageBus messageBus, IServiceProvider serviceProvider, @@ -36,7 +36,7 @@ ILoggerFactory loggerFactory ) : base(workItemQueue, cacheClient, messageBus, timeProvider, resiliencePolicyProvider, loggerFactory) { _appOptions = appOptions; - _serializerSettings = serializerSettings; + _jsonSerializerOptions = jsonSerializerOptions; _logger.LogInformation("All new indexes will be created with {ElasticsearchNumberOfShards} Shards and {ElasticsearchNumberOfReplicas} Replicas", _appOptions.ElasticsearchOptions.NumberOfShards, _appOptions.ElasticsearchOptions.NumberOfReplicas); AddIndex(Stacks = new StackIndex(this)); @@ -78,7 +78,8 @@ public override void ConfigureGlobalQueryBuilders(ElasticQueryBuilder builder) protected override IElasticClient CreateElasticClient() { var connectionPool = CreateConnectionPool(); - var settings = new ConnectionSettings(connectionPool, (serializer, values) => new ElasticJsonNetSerializer(serializer, values, _serializerSettings)); + var serializer = new ElasticSystemTextJsonSerializer(_jsonSerializerOptions); + var settings = new ConnectionSettings(connectionPool, (_, _) => serializer); ConfigureSettings(settings); foreach (var index in Indexes) diff --git a/src/Exceptionless.Core/Serialization/DataObjectConverter.cs b/src/Exceptionless.Core/Serialization/DataObjectConverter.cs deleted file mode 100644 index 559c952253..0000000000 --- a/src/Exceptionless.Core/Serialization/DataObjectConverter.cs +++ /dev/null @@ -1,197 +0,0 @@ -using System.Collections.Concurrent; -using System.Reflection; -using Exceptionless.Core.Extensions; -using Exceptionless.Core.Models; -using Exceptionless.Core.Reflection; -using Foundatio.Repositories.Extensions; -using Microsoft.Extensions.Logging; -using Newtonsoft.Json; -using Newtonsoft.Json.Converters; -using Newtonsoft.Json.Linq; - -namespace Exceptionless.Serializer; - -public class DataObjectConverter : CustomCreationConverter where T : IData, new() -{ - private static readonly Type _type = typeof(T); - private static readonly ConcurrentDictionary _propertyAccessors = new(StringComparer.OrdinalIgnoreCase); - private readonly ConcurrentDictionary _dataTypeRegistry = new(StringComparer.OrdinalIgnoreCase); - private readonly ILogger _logger; - private readonly char[] _filteredChars = ['.', '-', '_']; - - public DataObjectConverter(ILogger logger, IEnumerable>? knownDataTypes = null) - { - _logger = logger; - - if (knownDataTypes is not null) - _dataTypeRegistry.AddRange(knownDataTypes); - - if (_propertyAccessors.Count != 0) - return; - - foreach (var prop in _type.GetProperties(BindingFlags.Instance | BindingFlags.FlattenHierarchy | BindingFlags.Public).Where(p => p.CanWrite)) - _propertyAccessors.TryAdd(prop.Name, LateBinder.GetPropertyAccessor(prop)); - } - - public void AddKnownDataType(string name, Type dataType) - { - _dataTypeRegistry.TryAdd(name, dataType); - } - - public override object ReadJson(JsonReader reader, Type objectType, object? existingValue, JsonSerializer serializer) - { - var target = Create(objectType); - var json = JObject.Load(reader); - - foreach (var p in json.Properties()) - { - string propertyName = p.Name.ToLowerFiltered(_filteredChars); - - if (propertyName == "data" && p.Value is JObject) - { - foreach (var dataProp in ((JObject)p.Value).Properties()) - AddDataEntry(serializer, dataProp, target); - - continue; - } - - var accessor = _propertyAccessors.TryGetValue(propertyName, out var value) ? value : null; - if (accessor is not null) - { - if (p.Value.Type == JTokenType.None || p.Value.Type == JTokenType.Undefined) - continue; - - if (p.Value.Type == JTokenType.Null) - { - accessor.SetValue(target, null); - continue; - } - - if (accessor.MemberType == typeof(DateTime)) - { - if (p.Value.Type == JTokenType.Date || p.Value.Type == JTokenType.String && p.Value.Value()!.Contains("+")) - { - accessor.SetValue(target, p.Value.ToObject(serializer).DateTime); - continue; - } - } - else if (accessor.MemberType == typeof(DateTime?)) - { - if (p.Value.Type == JTokenType.Date || p.Value.Type == JTokenType.String && p.Value.Value()!.Contains("+")) - { - var offset = p.Value.ToObject(serializer); - accessor.SetValue(target, offset?.DateTime); - continue; - } - } - - accessor.SetValue(target, p.Value.ToObject(accessor.MemberType, serializer)); - continue; - } - - AddDataEntry(serializer, p, target); - } - - return target; - } - - private void AddDataEntry(JsonSerializer serializer, JProperty p, T target) - { - if (target.Data is null) - target.Data = new DataDictionary(); - - string dataKey = GetDataKey(target.Data, p.Name); - string unknownTypeDataKey = GetDataKey(target.Data, p.Name, true); - - // when adding items to data, see if they are a known type and deserialize to the registered type - if (_dataTypeRegistry.TryGetValue(p.Name, out var dataType)) - { - try - { - if (p.Value is JValue && p.Value.Type == JTokenType.String) - { - string value = p.Value.ToString(); - if (value.IsJson()) - target.Data[dataKey] = serializer.Deserialize(new StringReader(value), dataType); - else - target.Data[dataType == value.GetType() ? dataKey : unknownTypeDataKey] = value; - } - else - { - target.Data[dataKey] = p.Value.ToObject(dataType, serializer); - } - - return; - } - catch (Exception) - { - _logger.LogInformation("Error deserializing known data type {Name}: {Value}", p.Name, p.Value.ToString()); - } - } - - // Add item to data as a JObject, JArray or native type. - if (p.Value is JObject) - { - target.Data[dataType is null || dataType == typeof(JObject) ? dataKey : unknownTypeDataKey] = p.Value.ToObject(); - } - else if (p.Value is JArray) - { - target.Data[dataType is null || dataType == typeof(JArray) ? dataKey : unknownTypeDataKey] = p.Value.ToObject(); - } - else if (p.Value is JValue jValue && jValue.Type != JTokenType.String) - { - object? value = jValue.Value; - target.Data[dataType is null || dataType == value?.GetType() ? dataKey : unknownTypeDataKey] = value; - } - else - { - string value = p.Value.ToString(); - var jsonType = value.GetJsonType(); - if (jsonType == JsonType.Object) - { - if (value.TryFromJson(out JObject? obj)) - target.Data[dataType is null || dataType == obj?.GetType() ? dataKey : unknownTypeDataKey] = obj; - else - target.Data[dataType is null || dataType == value.GetType() ? dataKey : unknownTypeDataKey] = value; - } - else if (jsonType == JsonType.Array) - { - if (value.TryFromJson(out JArray? obj)) - target.Data[dataType is null || dataType == obj?.GetType() ? dataKey : unknownTypeDataKey] = obj; - else - target.Data[dataType is null || dataType == value.GetType() ? dataKey : unknownTypeDataKey] = value; - } - else - { - target.Data[dataType is null || dataType == value.GetType() ? dataKey : unknownTypeDataKey] = value; - } - } - } - - private string GetDataKey(DataDictionary data, string dataKey, bool isUnknownType = false) - { - if (data.ContainsKey(dataKey) || isUnknownType) - dataKey = dataKey.StartsWith("@") ? "_" + dataKey : dataKey; - - int count = 1; - string key = dataKey; - while (data.ContainsKey(key) || (isUnknownType && _dataTypeRegistry.ContainsKey(key))) - key = dataKey + count++; - - return key; - } - - public override T Create(Type objectType) - { - return new T(); - } - - public override bool CanRead => true; - - public override bool CanWrite => false; - - public override bool CanConvert(Type objectType) - { - return objectType == _type; - } -} diff --git a/src/Exceptionless.Core/Serialization/DynamicTypeContractResolver.cs b/src/Exceptionless.Core/Serialization/DynamicTypeContractResolver.cs deleted file mode 100644 index cde0f17ccb..0000000000 --- a/src/Exceptionless.Core/Serialization/DynamicTypeContractResolver.cs +++ /dev/null @@ -1,38 +0,0 @@ -using System.Reflection; -using Foundatio.Repositories.Extensions; -using Newtonsoft.Json.Serialization; - -namespace Exceptionless.Serializer; - -public class DynamicTypeContractResolver : IContractResolver -{ - private readonly HashSet _assemblies = new(); - private readonly HashSet _types = new(); - - private readonly IContractResolver _defaultResolver; - private readonly IContractResolver _resolver; - - public DynamicTypeContractResolver(IContractResolver resolver, IContractResolver? defaultResolver = null) - { - _resolver = resolver ?? throw new ArgumentNullException(nameof(resolver)); - _defaultResolver = defaultResolver ?? new DefaultContractResolver(); - } - - public void UseDefaultResolverFor(params Assembly[] assemblies) - { - _assemblies.AddRange(assemblies); - } - - public void UseDefaultResolverFor(params Type[] types) - { - _types.AddRange(types); - } - - public JsonContract ResolveContract(Type type) - { - if (_types.Contains(type) || _assemblies.Contains(type.Assembly)) - return _defaultResolver.ResolveContract(type); - - return _resolver.ResolveContract(type); - } -} diff --git a/src/Exceptionless.Core/Serialization/ElasticConnectionSettingsAwareContractResolver.cs b/src/Exceptionless.Core/Serialization/ElasticConnectionSettingsAwareContractResolver.cs deleted file mode 100644 index 541befc681..0000000000 --- a/src/Exceptionless.Core/Serialization/ElasticConnectionSettingsAwareContractResolver.cs +++ /dev/null @@ -1,34 +0,0 @@ -using System.Reflection; -using Exceptionless.Core.Extensions; -using Nest; -using Nest.JsonNetSerializer; -using Newtonsoft.Json; -using Newtonsoft.Json.Serialization; - -namespace Exceptionless.Core.Serialization; - -public class ElasticConnectionSettingsAwareContractResolver : ConnectionSettingsAwareContractResolver -{ - public ElasticConnectionSettingsAwareContractResolver(IConnectionSettingsValues connectionSettings) : base(connectionSettings) { } - - protected override JsonProperty CreateProperty(MemberInfo member, MemberSerialization memberSerialization) - { - var property = base.CreateProperty(member, memberSerialization); - - var shouldSerialize = property.ShouldSerialize; - property.ShouldSerialize = obj => (shouldSerialize is null || shouldSerialize(obj)) && !property.IsValueEmptyCollection(obj); - return property; - } - - protected override JsonDictionaryContract CreateDictionaryContract(Type objectType) - { - var contract = base.CreateDictionaryContract(objectType); - contract.DictionaryKeyResolver = propertyName => propertyName; - return contract; - } - - protected override string ResolvePropertyName(string propertyName) - { - return propertyName.ToLowerUnderscoredWords(); - } -} diff --git a/src/Exceptionless.Core/Serialization/ElasticJsonNetSerializer.cs b/src/Exceptionless.Core/Serialization/ElasticJsonNetSerializer.cs deleted file mode 100644 index 403a2329e8..0000000000 --- a/src/Exceptionless.Core/Serialization/ElasticJsonNetSerializer.cs +++ /dev/null @@ -1,41 +0,0 @@ -using Elasticsearch.Net; -using Nest; -using Nest.JsonNetSerializer; -using Newtonsoft.Json; - -namespace Exceptionless.Core.Serialization; - -public class ElasticJsonNetSerializer : JsonNetSerializer -{ - public ElasticJsonNetSerializer( - IElasticsearchSerializer builtinSerializer, - IConnectionSettingsValues connectionSettings, - JsonSerializerSettings serializerSettings - ) : base( - builtinSerializer, - connectionSettings, - () => CreateJsonSerializerSettings(serializerSettings), - contractJsonConverters: serializerSettings.Converters.ToList() - ) - { - } - - private static JsonSerializerSettings CreateJsonSerializerSettings(JsonSerializerSettings serializerSettings) - { - return new JsonSerializerSettings - { - DateParseHandling = serializerSettings.DateParseHandling, - DefaultValueHandling = serializerSettings.DefaultValueHandling, - MissingMemberHandling = serializerSettings.MissingMemberHandling, - NullValueHandling = serializerSettings.NullValueHandling - }; - } - - protected override ConnectionSettingsAwareContractResolver CreateContractResolver() - { - // TODO: Verify we don't need to use the DynamicTypeContractResolver. - var resolver = new ElasticConnectionSettingsAwareContractResolver(ConnectionSettings); - ModifyContractResolver(resolver); - return resolver; - } -} diff --git a/src/Exceptionless.Core/Serialization/ElasticSystemTextJsonSerializer.cs b/src/Exceptionless.Core/Serialization/ElasticSystemTextJsonSerializer.cs new file mode 100644 index 0000000000..52aa22cd9c --- /dev/null +++ b/src/Exceptionless.Core/Serialization/ElasticSystemTextJsonSerializer.cs @@ -0,0 +1,314 @@ +using System.Globalization; +using System.Text.Json; +using System.Text.Json.Serialization; +using Elasticsearch.Net; + +namespace Exceptionless.Core.Serialization; + +/// +/// System.Text.Json serializer for Elasticsearch NEST client. +/// +/// This serializer implements to enable the NEST 7.x +/// client to use System.Text.Json instead of Newtonsoft.Json for document serialization. +/// +/// Why custom converters are needed: +/// +/// +/// DynamicDictionary +/// +/// Elasticsearch returns dynamic responses as which STJ +/// doesn't know how to serialize/deserialize. This converter handles the round-trip. +/// +/// +/// +/// DateTime/DateTimeOffset +/// +/// Elasticsearch expects ISO 8601 dates. STJ's default output uses "Z" suffix for UTC +/// while we need explicit "+00:00" for consistency with stored data. +/// +/// +/// +/// +/// Thread Safety: This class is thread-safe. Options are lazily initialized once. +/// +public sealed class ElasticSystemTextJsonSerializer : IElasticsearchSerializer +{ + private readonly Lazy _optionsIndented; + private readonly Lazy _optionsCompact; + + /// + /// Creates a new serializer instance. + /// + /// + /// Optional base options to extend. If provided, these options are cloned and augmented + /// with Elasticsearch-specific converters. If null, default options are used. + /// + public ElasticSystemTextJsonSerializer(JsonSerializerOptions? baseOptions = null) + { + _optionsIndented = new Lazy(() => CreateOptions(baseOptions, writeIndented: true)); + _optionsCompact = new Lazy(() => CreateOptions(baseOptions, writeIndented: false)); + } + + private static JsonSerializerOptions CreateOptions(JsonSerializerOptions? baseOptions, bool writeIndented) + { + var options = baseOptions is not null + ? new JsonSerializerOptions(baseOptions) + : new JsonSerializerOptions(); + + // Elasticsearch convention: don't serialize null values + options.DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull; + options.WriteIndented = writeIndented; + + // Insert Elasticsearch converters at the beginning for priority + // Order matters: more specific converters should come first + options.Converters.Insert(0, new DynamicDictionaryConverter()); + options.Converters.Insert(1, new Iso8601DateTimeOffsetConverter()); + options.Converters.Insert(2, new Iso8601DateTimeConverter()); + + return options; + } + + private JsonSerializerOptions GetOptions(SerializationFormatting formatting) => + formatting == SerializationFormatting.Indented ? _optionsIndented.Value : _optionsCompact.Value; + + #region Synchronous API + + /// + public object? Deserialize(Type type, Stream stream) + { + if (IsEmptyStream(stream)) + return null; + + var buffer = ReadStreamToSpan(stream); + return JsonSerializer.Deserialize(buffer, type, _optionsCompact.Value); + } + + /// + public T? Deserialize(Stream stream) + { + if (IsEmptyStream(stream)) + return default; + + var buffer = ReadStreamToSpan(stream); + return JsonSerializer.Deserialize(buffer, _optionsCompact.Value); + } + + /// + public void Serialize(T data, Stream stream, SerializationFormatting formatting = SerializationFormatting.None) + { + using var writer = new Utf8JsonWriter(stream); + var options = GetOptions(formatting); + + if (data is null) + { + JsonSerializer.Serialize(writer, (object?)null, typeof(object), options); + } + else + { + // Use runtime type to ensure proper polymorphic serialization + JsonSerializer.Serialize(writer, data, data.GetType(), options); + } + } + + #endregion + + #region Asynchronous API + + /// + public async Task DeserializeAsync(Type type, Stream stream, CancellationToken cancellationToken = default) + { + if (IsEmptyStream(stream)) + return null; + + return await JsonSerializer.DeserializeAsync(stream, type, _optionsCompact.Value, cancellationToken) + .ConfigureAwait(false); + } + + /// + public async Task DeserializeAsync(Stream stream, CancellationToken cancellationToken = default) + { + if (IsEmptyStream(stream)) + return default; + + return await JsonSerializer.DeserializeAsync(stream, _optionsCompact.Value, cancellationToken) + .ConfigureAwait(false); + } + + /// + public Task SerializeAsync( + T data, + Stream stream, + SerializationFormatting formatting = SerializationFormatting.None, + CancellationToken cancellationToken = default) + { + var options = GetOptions(formatting); + + if (data is null) + { + return JsonSerializer.SerializeAsync(stream, (object?)null, typeof(object), options, cancellationToken); + } + + return JsonSerializer.SerializeAsync(stream, data, data.GetType(), options, cancellationToken); + } + + #endregion + + #region Stream Helpers + + private static bool IsEmptyStream(Stream? stream) + { + return stream is null || stream == Stream.Null || (stream.CanSeek && stream.Length == 0); + } + + private static ReadOnlySpan ReadStreamToSpan(Stream stream) + { + // Fast path: if already a MemoryStream with accessible buffer, use it directly + if (stream is MemoryStream ms && ms.TryGetBuffer(out var segment)) + { + return segment.AsSpan(); + } + + // Slow path: copy to new buffer + using var buffer = stream.CanSeek + ? new MemoryStream((int)stream.Length) + : new MemoryStream(); + + stream.CopyTo(buffer); + return buffer.TryGetBuffer(out var seg) ? seg.AsSpan() : buffer.ToArray(); + } + + #endregion +} + +#region Elasticsearch-Specific Converters + +/// +/// Converts to/from JSON. +/// +/// Why this converter exists: +/// Elasticsearch.Net uses for dynamic responses (e.g., script fields, +/// aggregation buckets). STJ has no built-in support for this type, so we must provide custom +/// serialization logic. +/// +/// Serialization: Writes as a JSON object with key-value pairs. +/// Deserialization: Reads JSON objects/arrays into DynamicDictionary. +/// +internal sealed class DynamicDictionaryConverter : JsonConverter +{ + public override DynamicDictionary? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + return reader.TokenType switch + { + JsonTokenType.StartArray => ReadFromArray(ref reader, options), + JsonTokenType.StartObject => ReadFromObject(ref reader, options), + JsonTokenType.Null => null, + _ => throw new JsonException($"Unexpected token type {reader.TokenType} when deserializing DynamicDictionary") + }; + } + + private static DynamicDictionary ReadFromArray(ref Utf8JsonReader reader, JsonSerializerOptions options) + { + var array = JsonSerializer.Deserialize(ref reader, options); + var dict = new Dictionary(StringComparer.OrdinalIgnoreCase); + + if (array is not null) + { + for (var i = 0; i < array.Length; i++) + { + dict[i.ToString(CultureInfo.InvariantCulture)] = new DynamicValue(array[i]); + } + } + + return DynamicDictionary.Create(dict); + } + + private static DynamicDictionary ReadFromObject(ref Utf8JsonReader reader, JsonSerializerOptions options) + { + var dict = JsonSerializer.Deserialize>(ref reader, options); + return dict is not null ? DynamicDictionary.Create(dict!) : new DynamicDictionary(); + } + + public override void Write(Utf8JsonWriter writer, DynamicDictionary dictionary, JsonSerializerOptions options) + { + writer.WriteStartObject(); + + foreach (var (key, dynamicValue) in dictionary.GetKeyValues()) + { + // Skip null values (consistent with DefaultIgnoreCondition.WhenWritingNull) + if (dynamicValue?.Value is null) + continue; + + writer.WritePropertyName(key); + JsonSerializer.Serialize(writer, dynamicValue.Value, options); + } + + writer.WriteEndObject(); + } +} + +/// +/// Converts to/from ISO 8601 format for Elasticsearch. +/// +/// Why this converter exists: +/// Elasticsearch indexes dates in ISO 8601 format. While STJ handles DateTime correctly, +/// this converter ensures consistent UTC conversion and format across the application. +/// +/// Write behavior: Converts to UTC and outputs in round-trip format ("O"). +/// Read behavior: Parses ISO 8601 strings with culture-invariant settings. +/// +internal sealed class Iso8601DateTimeConverter : JsonConverter +{ + public override DateTime Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + if (reader.TokenType == JsonTokenType.Null) + return default; + + var dateString = reader.GetString(); + if (string.IsNullOrEmpty(dateString)) + return default; + + // Parse with DateTimeStyles to handle various ISO 8601 formats + return DateTime.Parse(dateString, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind); + } + + public override void Write(Utf8JsonWriter writer, DateTime value, JsonSerializerOptions options) + { + // Always output in UTC with round-trip format for Elasticsearch compatibility + var utcValue = value.Kind == DateTimeKind.Utc ? value : value.ToUniversalTime(); + writer.WriteStringValue(utcValue.ToString("O", CultureInfo.InvariantCulture)); + } +} + +/// +/// Converts to/from ISO 8601 format for Elasticsearch. +/// +/// Why this converter exists: +/// DateTimeOffset preserves timezone offset information. This converter ensures the offset +/// is written in the explicit "+HH:mm" format (e.g., "+00:00") rather than "Z" for consistency +/// with historical data serialized by Newtonsoft.Json. +/// +/// Write behavior: Outputs in round-trip format ("O") preserving offset. +/// Read behavior: Parses ISO 8601 strings with culture-invariant settings. +/// +internal sealed class Iso8601DateTimeOffsetConverter : JsonConverter +{ + public override DateTimeOffset Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + if (reader.TokenType == JsonTokenType.Null) + return default; + + var dateString = reader.GetString(); + if (string.IsNullOrEmpty(dateString)) + return default; + + return DateTimeOffset.Parse(dateString, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind); + } + + public override void Write(Utf8JsonWriter writer, DateTimeOffset value, JsonSerializerOptions options) + { + // Round-trip format preserves the exact offset (e.g., +00:00, -05:00) + writer.WriteStringValue(value.ToString("O", CultureInfo.InvariantCulture)); + } +} + +#endregion diff --git a/src/Exceptionless.Core/Serialization/EmptyCollectionModifier.cs b/src/Exceptionless.Core/Serialization/EmptyCollectionModifier.cs new file mode 100644 index 0000000000..e13fe8d599 --- /dev/null +++ b/src/Exceptionless.Core/Serialization/EmptyCollectionModifier.cs @@ -0,0 +1,66 @@ +using System.Collections; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Text.Json.Serialization.Metadata; + +namespace Exceptionless.Core.Serialization; + +/// +/// A type info modifier that skips empty collections during serialization to match Newtonsoft's behavior. +/// +public static class EmptyCollectionModifier +{ + /// + /// Modifies JSON type info to skip empty collections/dictionaries during serialization. + /// + public static void SkipEmptyCollections(JsonTypeInfo typeInfo) + { + foreach (var property in typeInfo.Properties) + { + // For properties typed as IEnumerable (but not string), check at compile time + if (typeof(IEnumerable).IsAssignableFrom(property.PropertyType) && property.PropertyType != typeof(string)) + { + property.ShouldSerialize = (obj, value) => !IsEmptyCollection(value); + } + // For object-typed properties, check the runtime value + else if (property.PropertyType == typeof(object)) + { + var originalShouldSerialize = property.ShouldSerialize; + property.ShouldSerialize = (obj, value) => + { + // First check original condition if any + if (originalShouldSerialize is not null && !originalShouldSerialize(obj, value)) + return false; + + // Then check if runtime value is an empty collection + return !IsEmptyCollection(value); + }; + } + } + } + + private static bool IsEmptyCollection(object? value) + { + return value switch + { + null => true, + string => false, // strings are IEnumerable but should not be treated as collections + ICollection { Count: 0 } => true, + IEnumerable enumerable => !HasAnyElement(enumerable), + _ => false + }; + } + + private static bool HasAnyElement(IEnumerable enumerable) + { + var enumerator = enumerable.GetEnumerator(); + try + { + return enumerator.MoveNext(); + } + finally + { + (enumerator as IDisposable)?.Dispose(); + } + } +} diff --git a/src/Exceptionless.Core/Serialization/ExceptionlessNamingStrategy.cs b/src/Exceptionless.Core/Serialization/ExceptionlessNamingStrategy.cs deleted file mode 100644 index d677cf08d8..0000000000 --- a/src/Exceptionless.Core/Serialization/ExceptionlessNamingStrategy.cs +++ /dev/null @@ -1,12 +0,0 @@ -using Exceptionless.Core.Extensions; -using Newtonsoft.Json.Serialization; - -namespace Exceptionless.Core.Serialization; - -public class ExceptionlessNamingStrategy : SnakeCaseNamingStrategy -{ - protected override string ResolvePropertyName(string name) - { - return name.ToLowerUnderscoredWords(); - } -} diff --git a/src/Exceptionless.Core/Serialization/JsonSerializerOptionsExtensions.cs b/src/Exceptionless.Core/Serialization/JsonSerializerOptionsExtensions.cs index 7a05de2ae7..002fb0cd73 100644 --- a/src/Exceptionless.Core/Serialization/JsonSerializerOptionsExtensions.cs +++ b/src/Exceptionless.Core/Serialization/JsonSerializerOptionsExtensions.cs @@ -1,5 +1,8 @@ +using System.Text.Encodings.Web; using System.Text.Json; using System.Text.Json.Serialization; +using System.Text.Json.Serialization.Metadata; +using System.Text.Unicode; namespace Exceptionless.Core.Serialization; @@ -9,15 +12,34 @@ namespace Exceptionless.Core.Serialization; public static class JsonSerializerOptionsExtensions { /// - /// Configures with Exceptionless conventions: + /// Configures with Exceptionless conventions for WRITING: /// snake_case property naming, null value handling, and dynamic object support. /// + /// + /// + /// IMPORTANT: These options include a that applies + /// to BOTH serialization and deserialization. The options use PropertyNameCaseInsensitive + /// to support matching both PascalCase and snake_case JSON property names. + /// + /// + /// STJ's transforms C# property names + /// before matching against JSON property names. For example, with our snake_case policy, + /// MachineName becomes machine_name, which won't match a JSON property named + /// "MachineName" even with enabled. + /// + /// /// The options to configure. /// The configured options for chaining. public static JsonSerializerOptions ConfigureExceptionlessDefaults(this JsonSerializerOptions options) { options.DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull; options.PropertyNamingPolicy = LowerCaseUnderscoreNamingPolicy.Instance; + options.PropertyNameCaseInsensitive = true; + + // XSS-safe encoder: escapes <, >, &, ' while allowing Unicode characters + // This protects against script injection when JSON is embedded in HTML/JavaScript + options.Encoder = JavaScriptEncoder.Create(new TextEncoderSettings(UnicodeRanges.All)); + options.Converters.Add(new ObjectToInferredTypesConverter()); // Ensures tuples and records are serialized with their field names instead of "Item1", "Item2", etc. @@ -27,6 +49,12 @@ public static JsonSerializerOptions ConfigureExceptionlessDefaults(this JsonSeri // If you see "cannot be null" errors, fix the model's nullability annotation or the data. options.RespectNullableAnnotations = true; + // Skip empty collections during serialization to match Newtonsoft behavior + options.TypeInfoResolver = new DefaultJsonTypeInfoResolver + { + Modifiers = { EmptyCollectionModifier.SkipEmptyCollections } + }; + return options; } } diff --git a/src/Exceptionless.Core/Serialization/LowerCaseUnderscorePropertyNamesContractResolver.cs b/src/Exceptionless.Core/Serialization/LowerCaseUnderscorePropertyNamesContractResolver.cs deleted file mode 100644 index 7fe758b24d..0000000000 --- a/src/Exceptionless.Core/Serialization/LowerCaseUnderscorePropertyNamesContractResolver.cs +++ /dev/null @@ -1,30 +0,0 @@ -using System.Reflection; -using Exceptionless.Core.Extensions; -using Newtonsoft.Json; -using Newtonsoft.Json.Serialization; - -namespace Exceptionless.Core.Serialization; - -public class LowerCaseUnderscorePropertyNamesContractResolver : DefaultContractResolver -{ - protected override JsonProperty CreateProperty(MemberInfo member, MemberSerialization memberSerialization) - { - var property = base.CreateProperty(member, memberSerialization); - - var shouldSerialize = property.ShouldSerialize; - property.ShouldSerialize = obj => (shouldSerialize is null || shouldSerialize(obj)) && !property.IsValueEmptyCollection(obj); - return property; - } - - protected override JsonDictionaryContract CreateDictionaryContract(Type objectType) - { - var contract = base.CreateDictionaryContract(objectType); - contract.DictionaryKeyResolver = propertyName => propertyName; - return contract; - } - - protected override string ResolvePropertyName(string propertyName) - { - return propertyName.ToLowerUnderscoredWords(); - } -} diff --git a/src/Exceptionless.Core/Serialization/ObjectToInferredTypesConverter.cs b/src/Exceptionless.Core/Serialization/ObjectToInferredTypesConverter.cs index 8d5dd5841f..e824b58f1a 100644 --- a/src/Exceptionless.Core/Serialization/ObjectToInferredTypesConverter.cs +++ b/src/Exceptionless.Core/Serialization/ObjectToInferredTypesConverter.cs @@ -1,3 +1,4 @@ +using System.Buffers; using System.Text.Json; using System.Text.Json.Serialization; using Exceptionless.Core.Models; @@ -90,22 +91,50 @@ public override void Write(Utf8JsonWriter writer, object? value, JsonSerializerO } /// - /// Reads a JSON number, preferring for integers and for decimals. + /// Reads a JSON number, preserving the original representation (integer vs floating-point). /// + /// + /// This method preserves data integrity by checking the raw JSON text to determine + /// if a number was written with a decimal point (e.g., 0.0) vs as an integer (0). + /// This is critical because: + /// + /// User data must be preserved exactly as provided + /// TryGetInt64 would succeed for 0.0 since 0.0 == 0 mathematically + /// Serializing back would lose the decimal representation + /// + /// private static object ReadNumber(ref Utf8JsonReader reader) { - // Try smallest to largest integer types first for optimal boxing + // Check the raw text to preserve decimal vs integer representation + // This is critical for data integrity - 0.0 should stay as double, not become 0L + ReadOnlySpan rawValue = reader.HasValueSequence + ? reader.ValueSequence.ToArray() + : reader.ValueSpan; + + // If the raw text contains a decimal point, treat as floating-point + if (rawValue.Contains((byte)'.')) + { + // Try decimal for precise values (e.g., financial data) before double + if (reader.TryGetDecimal(out decimal d)) + return d; + + // Fall back to double for floating-point + return reader.GetDouble(); + } + + // No decimal point - this is an integer + // Try int32 first for smaller values, then long for larger integers if (reader.TryGetInt32(out int i)) return i; if (reader.TryGetInt64(out long l)) return l; - // Try decimal for precise values (e.g., financial data) before double - if (reader.TryGetDecimal(out decimal d)) - return d; + // For very large integers, try decimal first to preserve precision + if (reader.TryGetDecimal(out decimal dec)) + return dec; - // Fall back to double for floating-point + // Fall back to double only if decimal also fails return reader.GetDouble(); } diff --git a/src/Exceptionless.Core/Services/SlackService.cs b/src/Exceptionless.Core/Services/SlackService.cs index c41ec984c4..4bd610efaf 100644 --- a/src/Exceptionless.Core/Services/SlackService.cs +++ b/src/Exceptionless.Core/Services/SlackService.cs @@ -14,7 +14,7 @@ public class SlackService private readonly HttpClient _client = new(); private readonly IQueue _webHookNotificationQueue; private readonly FormattingPluginManager _pluginManager; - private readonly ISerializer _serializer; + private readonly ITextSerializer _serializer; private readonly AppOptions _appOptions; private readonly ILogger _logger; @@ -111,7 +111,7 @@ public Task SendMessageAsync(string organizationId, string projectId, string url public async Task SendEventNoticeAsync(PersistentEvent ev, Project project, bool isNew, bool isRegression) { - var token = project.GetSlackToken(); + var token = project.GetSlackToken(_serializer); if (token?.IncomingWebhook?.Url is null) return false; diff --git a/src/Exceptionless.Core/Utility/ErrorSignature.cs b/src/Exceptionless.Core/Utility/ErrorSignature.cs index 9aa7d3c9f1..2762922fb2 100644 --- a/src/Exceptionless.Core/Utility/ErrorSignature.cs +++ b/src/Exceptionless.Core/Utility/ErrorSignature.cs @@ -1,8 +1,8 @@ using System.Text; -using System.Text.Json; using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Models.Data; +using Foundatio.Serializer; namespace Exceptionless.Core.Utility; @@ -10,14 +10,14 @@ public class ErrorSignature { private readonly HashSet _userNamespaces; private readonly HashSet _userCommonMethods; - private readonly JsonSerializerOptions _jsonOptions; + private readonly ITextSerializer _serializer; private static readonly string[] _defaultNonUserNamespaces = ["System", "Microsoft"]; // TODO: Add support for user public key token on signed assemblies - public ErrorSignature(Error error, JsonSerializerOptions jsonOptions, IEnumerable? userNamespaces = null, IEnumerable? userCommonMethods = null, bool emptyNamespaceIsUserMethod = true, bool shouldFlagSignatureTarget = true) + public ErrorSignature(Error error, ITextSerializer serializer, IEnumerable? userNamespaces = null, IEnumerable? userCommonMethods = null, bool emptyNamespaceIsUserMethod = true, bool shouldFlagSignatureTarget = true) { Error = error ?? throw new ArgumentNullException(nameof(error)); - _jsonOptions = jsonOptions ?? throw new ArgumentNullException(nameof(jsonOptions)); + _serializer = serializer ?? throw new ArgumentNullException(nameof(serializer)); _userNamespaces = userNamespaces is null ? [] @@ -180,7 +180,7 @@ private void AddSpecialCaseDetails(InnerError error) if (!error.Data.ContainsKey(Error.KnownDataKeys.ExtraProperties)) return; - var extraProperties = error.Data.GetValue>(Error.KnownDataKeys.ExtraProperties, _jsonOptions); + var extraProperties = error.Data.GetValue>(Error.KnownDataKeys.ExtraProperties, _serializer); if (extraProperties is null) { error.Data.Remove(Error.KnownDataKeys.ExtraProperties); diff --git a/src/Exceptionless.Core/Utility/ExtensibleObject.cs b/src/Exceptionless.Core/Utility/ExtensibleObject.cs index 6ac04bddf7..733d026aba 100644 --- a/src/Exceptionless.Core/Utility/ExtensibleObject.cs +++ b/src/Exceptionless.Core/Utility/ExtensibleObject.cs @@ -1,7 +1,6 @@ using System.ComponentModel; +using System.Text.Json; using Exceptionless.Core.Extensions; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; namespace Exceptionless.Core.Utility; @@ -22,7 +21,6 @@ public interface IExtensibleObject public class ExtensibleObject : INotifyPropertyChanged, IExtensibleObject { - [JsonProperty] private readonly Dictionary _extendedData = new(); public void SetProperty(string name, T value) @@ -44,8 +42,18 @@ public void SetProperty(string name, T value) if (value is T tValue) return tValue; - if (value is JContainer container) - return container.ToObject(); + // Handle JsonElement from STJ deserialization + if (value is JsonElement jsonElement) + { + try + { + return jsonElement.Deserialize(); + } + catch + { + // Fall through to ToType conversion + } + } return value.ToType(); } diff --git a/src/Exceptionless.Core/Utility/TypeHelper.cs b/src/Exceptionless.Core/Utility/TypeHelper.cs index de21067616..d092c87603 100644 --- a/src/Exceptionless.Core/Utility/TypeHelper.cs +++ b/src/Exceptionless.Core/Utility/TypeHelper.cs @@ -1,7 +1,7 @@ using System.Diagnostics; using System.Globalization; using System.Reflection; -using Newtonsoft.Json.Linq; +using System.Text.Json; namespace Exceptionless.Core.Helpers; @@ -52,8 +52,9 @@ public static bool AreSameValue(object a, object b) catch { } } - if (a is JToken && b is JToken) - return String.Equals(a.ToString(), b.ToString()); + // Handle JsonElement comparison by comparing string representations + if (a is JsonElement jsonA && b is JsonElement jsonB) + return String.Equals(jsonA.GetRawText(), jsonB.GetRawText()); if (a != b && !a.Equals(b)) return false; diff --git a/src/Exceptionless.Web/Controllers/EventController.cs b/src/Exceptionless.Web/Controllers/EventController.cs index e19a21f68f..e8735eefc4 100644 --- a/src/Exceptionless.Web/Controllers/EventController.cs +++ b/src/Exceptionless.Web/Controllers/EventController.cs @@ -26,10 +26,10 @@ using Foundatio.Repositories.Elasticsearch.Extensions; using Foundatio.Repositories.Extensions; using Foundatio.Repositories.Models; +using Foundatio.Serializer; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using Microsoft.Net.Http.Headers; -using Newtonsoft.Json; namespace Exceptionless.Web.Controllers; @@ -47,7 +47,7 @@ public class EventController : RepositoryApiController _userDescriptionValidator; private readonly FormattingPluginManager _formattingPluginManager; private readonly ICacheClient _cache; - private readonly JsonSerializerSettings _jsonSerializerSettings; + private readonly ITextSerializer _serializer; private readonly AppOptions _appOptions; public EventController(IEventRepository repository, @@ -75,7 +75,7 @@ ILoggerFactory loggerFactory _userDescriptionValidator = userDescriptionValidator; _formattingPluginManager = formattingPluginManager; _cache = cacheClient; - _jsonSerializerSettings = jsonSerializerSettings; + _serializer = serializer; _appOptions = appOptions; AllowedDateFields.Add(EventIndex.Alias.Date); @@ -1125,7 +1125,7 @@ private async Task GetSubmitEventAsync(string? projectId = null, i charSet = contentTypeHeader.Charset.ToString(); } - var stream = new MemoryStream(ev.GetBytes(_jsonSerializerSettings)); + var stream = new MemoryStream(ev.GetBytes(_serializer)); await _eventPostService.EnqueueAsync(new EventPost(_appOptions.EnableArchive) { ApiVersion = apiVersion, diff --git a/src/Exceptionless.Web/Controllers/ProjectController.cs b/src/Exceptionless.Web/Controllers/ProjectController.cs index 04700c5e43..ed2a9b1ef3 100644 --- a/src/Exceptionless.Web/Controllers/ProjectController.cs +++ b/src/Exceptionless.Web/Controllers/ProjectController.cs @@ -16,6 +16,7 @@ using Foundatio.Queues; using Foundatio.Repositories; using Foundatio.Repositories.Models; +using Foundatio.Serializer; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Mvc; using DataDictionary = Exceptionless.Core.Models.DataDictionary; @@ -34,6 +35,7 @@ public class ProjectController : RepositoryApiController _workItemQueue; private readonly BillingManager _billingManager; private readonly SlackService _slackService; + private readonly ITextSerializer _serializer; private readonly AppOptions _options; private readonly UsageService _usageService; @@ -62,6 +64,7 @@ ILoggerFactory loggerFactory _workItemQueue = workItemQueue; _billingManager = billingManager; _slackService = slackService; + _serializer = serializer; _options = options; _usageService = usageService; } @@ -678,7 +681,7 @@ public async Task RemoveSlackAsync(string id) if (project is null) return NotFound(); - var token = project.GetSlackToken(); + var token = project.GetSlackToken(_serializer); using var _ = _logger.BeginScope(new ExceptionlessState().Property("Token", token).Tag("Slack").Identity(CurrentUser.EmailAddress).Property("User", CurrentUser).SetHttpContext(HttpContext)); if (token is not null) diff --git a/src/Exceptionless.Web/Exceptionless.Web.csproj b/src/Exceptionless.Web/Exceptionless.Web.csproj index aea27910ea..3f9012e380 100644 --- a/src/Exceptionless.Web/Exceptionless.Web.csproj +++ b/src/Exceptionless.Web/Exceptionless.Web.csproj @@ -18,7 +18,6 @@ - diff --git a/tests/Exceptionless.Tests/Controllers/EventControllerTests.cs b/tests/Exceptionless.Tests/Controllers/EventControllerTests.cs index 52cb4ccfe8..d5ce8fa6e9 100644 --- a/tests/Exceptionless.Tests/Controllers/EventControllerTests.cs +++ b/tests/Exceptionless.Tests/Controllers/EventControllerTests.cs @@ -25,6 +25,7 @@ using Foundatio.Queues; using Foundatio.Repositories; using Foundatio.Repositories.Models; +using Foundatio.Serializer; using Microsoft.Net.Http.Headers; using Xunit; using MediaTypeHeaderValue = System.Net.Http.Headers.MediaTypeHeaderValue; @@ -70,7 +71,7 @@ protected override async Task ResetDataAsync() [Fact] public async Task PostEvent_WithValidPayload_EnqueuesAndProcessesEventAsync() { - var jsonOptions = GetService(); + var serializer = GetService(); /* language=json */ const string json = """{"message":"test","reference_id":"TestReferenceId","@user":{"identity":"Test user","name":null}}"""; await SendRequestAsync(r => r @@ -97,12 +98,12 @@ await SendRequestAsync(r => r Assert.Equal("test", ev.Message); Assert.Equal("TestReferenceId", ev.ReferenceId); - var identity = ev.GetUserIdentity(jsonOptions); + var identity = ev.GetUserIdentity(serializer); Assert.NotNull(identity); Assert.Equal("Test user", identity.Identity); Assert.Null(identity.Name); Assert.Null(identity.Name); - Assert.Null(ev.GetUserDescription(jsonOptions)); + Assert.Null(ev.GetUserDescription(serializer)); // post description await _eventUserDescriptionQueue.DeleteQueueAsync(); @@ -127,13 +128,13 @@ await SendRequestAsync(r => r Assert.Equal(1, stats.Completed); ev = await _eventRepository.GetByIdAsync(ev.Id); - identity = ev.GetUserIdentity(jsonOptions); + identity = ev.GetUserIdentity(serializer); Assert.NotNull(identity); Assert.Equal("Test user", identity.Identity); Assert.Null(identity.Name); Assert.Null(identity.Name); - var description = ev.GetUserDescription(jsonOptions); + var description = ev.GetUserDescription(serializer); Assert.NotNull(description); Assert.Equal("Test Description", description.Description); Assert.Equal(TestConstants.UserEmail, description.EmailAddress); @@ -229,7 +230,7 @@ public async Task CanPostCompressedStringAsync() [Fact] public async Task CanPostJsonWithUserInfoAsync() { - var jsonOptions = GetService(); + var serializer = GetService(); /* language=json */ const string json = """{"message":"test","@user":{"identity":"Test user","name":null}}"""; await SendRequestAsync(r => r @@ -255,7 +256,7 @@ await SendRequestAsync(r => r var ev = events.Documents.Single(e => String.Equals(e.Type, Event.KnownTypes.Log)); Assert.Equal("test", ev.Message); - var userInfo = ev.GetUserIdentity(jsonOptions); + var userInfo = ev.GetUserIdentity(serializer); Assert.NotNull(userInfo); Assert.Equal("Test user", userInfo.Identity); Assert.Null(userInfo.Name); @@ -1725,7 +1726,7 @@ await SendRequestAsync(r => r await processEventsJob.RunAsync(TestCancellationToken); await RefreshDataAsync(); - var jsonOptions = GetService(); + var serializer = GetService(); // Assert var events = await _eventRepository.GetAllAsync(); @@ -1735,7 +1736,7 @@ await SendRequestAsync(r => r Assert.Equal("Error with mixed data", ev.Message); // Verify known data is properly deserialized - var userInfo = ev.GetUserIdentity(jsonOptions); + var userInfo = ev.GetUserIdentity(serializer); Assert.NotNull(userInfo); Assert.Equal("user@example.com", userInfo.Identity); Assert.Equal("Test User", userInfo.Name); diff --git a/tests/Exceptionless.Tests/Exceptionless.Tests.csproj b/tests/Exceptionless.Tests/Exceptionless.Tests.csproj index 7e0d8e8883..1917a08ec3 100644 --- a/tests/Exceptionless.Tests/Exceptionless.Tests.csproj +++ b/tests/Exceptionless.Tests/Exceptionless.Tests.csproj @@ -9,6 +9,8 @@ + + diff --git a/tests/Exceptionless.Tests/Mail/MailerTests.cs b/tests/Exceptionless.Tests/Mail/MailerTests.cs index 5f381bb418..582953a94b 100644 --- a/tests/Exceptionless.Tests/Mail/MailerTests.cs +++ b/tests/Exceptionless.Tests/Mail/MailerTests.cs @@ -1,4 +1,3 @@ -using System.Text.Json; using Exceptionless.Core; using Exceptionless.Core.Billing; using Exceptionless.Core.Extensions; @@ -11,6 +10,7 @@ using Exceptionless.Core.Utility; using Exceptionless.Tests.Utility; using Foundatio.Queues; +using Foundatio.Serializer; using Xunit; namespace Exceptionless.Tests.Mail; @@ -40,7 +40,7 @@ public MailerTests(ITestOutputHelper output) : base(output) _plans = GetService(); if (_mailer is NullMailer) - _mailer = new Mailer(GetService>(), GetService(), GetService(), _options, TimeProvider, Log.CreateLogger()); + _mailer = new Mailer(GetService>(), GetService(), GetService(), _options, TimeProvider, Log.CreateLogger()); } [Fact] diff --git a/tests/Exceptionless.Tests/Pipeline/EventPipelineTests.cs b/tests/Exceptionless.Tests/Pipeline/EventPipelineTests.cs index beac5c5a1f..5f0f1d2503 100644 --- a/tests/Exceptionless.Tests/Pipeline/EventPipelineTests.cs +++ b/tests/Exceptionless.Tests/Pipeline/EventPipelineTests.cs @@ -1,7 +1,6 @@ using System.Diagnostics; using System.Globalization; using System.Text; -using System.Text.Json; using Exceptionless.Core.Billing; using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; @@ -18,6 +17,7 @@ using Exceptionless.Tests.Utility; using Foundatio.Repositories; using Foundatio.Repositories.Extensions; +using Foundatio.Serializer; using Foundatio.Storage; using McSherry.SemanticVersioning; using Xunit; @@ -39,7 +39,7 @@ public sealed class EventPipelineTests : IntegrationTestsBase private readonly IUserRepository _userRepository; private readonly BillingManager _billingManager; private readonly BillingPlans _plans; - private readonly JsonSerializerOptions _jsonOptions; + private readonly ITextSerializer _serializer; public EventPipelineTests(ITestOutputHelper output, AppWebHostFactory factory) : base(output, factory) { @@ -55,7 +55,7 @@ public EventPipelineTests(ITestOutputHelper output, AppWebHostFactory factory) : _pipeline = GetService(); _billingManager = GetService(); _plans = GetService(); - _jsonOptions = GetService(); + _serializer = GetService(); } protected override async Task ResetDataAsync() @@ -224,19 +224,19 @@ public async Task UpdateAutoSessionLastActivityAsync() var results = await _eventRepository.GetAllAsync(o => o.PageLimit(15)); Assert.Equal(9, results.Total); Assert.Equal(2, results.Documents.Where(e => !String.IsNullOrEmpty(e.GetSessionId())).Select(e => e.GetSessionId()).Distinct().Count()); - Assert.Equal(1, results.Documents.Count(e => e.IsSessionEnd() && e.GetUserIdentity(_jsonOptions)?.Identity == "blake@exceptionless.io")); - Assert.Single(results.Documents.Where(e => !String.IsNullOrEmpty(e.GetSessionId()) && e.GetUserIdentity(_jsonOptions)?.Identity == "eric@exceptionless.io").Select(e => e.GetSessionId()).Distinct()); + Assert.Equal(1, results.Documents.Count(e => e.IsSessionEnd() && e.GetUserIdentity(_serializer)?.Identity == "blake@exceptionless.io")); + Assert.Single(results.Documents.Where(e => !String.IsNullOrEmpty(e.GetSessionId()) && e.GetUserIdentity(_serializer)?.Identity == "eric@exceptionless.io").Select(e => e.GetSessionId()).Distinct()); Assert.Equal(1, results.Documents.Count(e => String.IsNullOrEmpty(e.GetSessionId()))); Assert.Equal(1, results.Documents.Count(e => e.IsSessionEnd())); var sessionStarts = results.Documents.Where(e => e.IsSessionStart()).ToList(); Assert.Equal(2, sessionStarts.Count); - var firstUserSessionStartEvents = sessionStarts.Single(e => e.GetUserIdentity(_jsonOptions)?.Identity == "blake@exceptionless.io"); + var firstUserSessionStartEvents = sessionStarts.Single(e => e.GetUserIdentity(_serializer)?.Identity == "blake@exceptionless.io"); Assert.Equal((decimal)(lastEventDate - firstEventDate).TotalSeconds, firstUserSessionStartEvents.Value); Assert.True(firstUserSessionStartEvents.HasSessionEndTime()); - var secondUserSessionStartEvents = sessionStarts.Single(e => e.GetUserIdentity(_jsonOptions)?.Identity == "eric@exceptionless.io"); + var secondUserSessionStartEvents = sessionStarts.Single(e => e.GetUserIdentity(_serializer)?.Identity == "eric@exceptionless.io"); Assert.Equal((decimal)(lastEventDate - firstEventDate).TotalSeconds, secondUserSessionStartEvents.Value); Assert.False(secondUserSessionStartEvents.HasSessionEndTime()); } @@ -894,10 +894,10 @@ public async Task EnsureIncludePrivateInformationIsRespectedAsync(bool includePr var context = contexts.Single(); Assert.False(context.HasError); - var requestInfo = context.Event.GetRequestInfo(_jsonOptions); - var environmentInfo = context.Event.GetEnvironmentInfo(_jsonOptions); - var userInfo = context.Event.GetUserIdentity(_jsonOptions); - var userDescription = context.Event.GetUserDescription(_jsonOptions); + var requestInfo = context.Event.GetRequestInfo(_serializer); + var environmentInfo = context.Event.GetEnvironmentInfo(_serializer); + var userInfo = context.Event.GetUserIdentity(_serializer); + var userDescription = context.Event.GetUserDescription(_serializer); Assert.Equal("/test", requestInfo?.Path); Assert.Equal("Windows", environmentInfo?.OSName); @@ -1164,7 +1164,7 @@ public async Task GeneratePerformanceDataAsync() ev.Data.Remove(key); ev.Data.Remove(Event.KnownDataKeys.UserDescription); - var identity = ev.GetUserIdentity(_jsonOptions); + var identity = ev.GetUserIdentity(_serializer); if (identity?.Identity is not null) { if (!mappedUsers.ContainsKey(identity.Identity)) @@ -1173,7 +1173,7 @@ public async Task GeneratePerformanceDataAsync() ev.SetUserIdentity(mappedUsers[identity.Identity]); } - var request = ev.GetRequestInfo(_jsonOptions); + var request = ev.GetRequestInfo(_serializer); if (request is not null) { request.Cookies?.Clear(); @@ -1193,7 +1193,7 @@ public async Task GeneratePerformanceDataAsync() } } - InnerError? error = ev.GetError(_jsonOptions); + InnerError? error = ev.GetError(_serializer); while (error is not null) { error.Message = RandomData.GetSentence(); @@ -1203,13 +1203,13 @@ public async Task GeneratePerformanceDataAsync() error = error.Inner; } - var environment = ev.GetEnvironmentInfo(_jsonOptions); + var environment = ev.GetEnvironmentInfo(_serializer); environment?.Data?.Clear(); } // inject random session start events. if (currentBatchCount % 10 == 0) - events.Insert(0, events[0].ToSessionStartEvent(_jsonOptions)); + events.Insert(0, events[0].ToSessionStartEvent(_serializer)); await storage.SaveObjectAsync(Path.Combine(dataDirectory, $"{currentBatchCount++}.json"), events, TestCancellationToken); } diff --git a/tests/Exceptionless.Tests/Plugins/EventParserTests.cs b/tests/Exceptionless.Tests/Plugins/EventParserTests.cs index 5e0f215f23..3852c88a2c 100644 --- a/tests/Exceptionless.Tests/Plugins/EventParserTests.cs +++ b/tests/Exceptionless.Tests/Plugins/EventParserTests.cs @@ -1,7 +1,7 @@ -using Exceptionless.Core.Extensions; +using System.Text.Json; using Exceptionless.Core.Models; using Exceptionless.Core.Plugins.EventParser; -using Newtonsoft.Json; +using Foundatio.Serializer; using Xunit; namespace Exceptionless.Tests.Plugins; @@ -9,10 +9,12 @@ namespace Exceptionless.Tests.Plugins; public sealed class EventParserTests : TestWithServices { private readonly EventParserPluginManager _parser; + private readonly ITextSerializer _serializer; public EventParserTests(ITestOutputHelper output) : base(output) { _parser = GetService(); + _serializer = GetService(); } public static IEnumerable EventData => new[] { @@ -53,8 +55,13 @@ public void VerifyEventParserSerialization(string eventsFilePath) var events = _parser.ParseEvents(json, 2, "exceptionless/2.0.0.0"); Assert.Single(events); - string expectedContent = File.ReadAllText(eventsFilePath); - Assert.Equal(expectedContent, events.First().ToJson(Formatting.Indented, GetService())); + // Verify parsed event can round-trip through STJ serialization + string serialized = _serializer.SerializeToString(events.First()); + Assert.NotNull(serialized); + var roundTripped = _serializer.Deserialize(serialized); + Assert.NotNull(roundTripped); + Assert.Equal(events.First().Type, roundTripped.Type); + Assert.Equal(events.First().Message, roundTripped.Message); } [Theory] @@ -63,7 +70,7 @@ public void CanDeserializeEvents(string eventsFilePath) { string json = File.ReadAllText(eventsFilePath); - var ev = json.FromJson(GetService()); + var ev = _serializer.Deserialize(json); Assert.NotNull(ev); } diff --git a/tests/Exceptionless.Tests/Plugins/EventUpgraderTests.cs b/tests/Exceptionless.Tests/Plugins/EventUpgraderTests.cs index 2459e1b4ac..1550fd846d 100644 --- a/tests/Exceptionless.Tests/Plugins/EventUpgraderTests.cs +++ b/tests/Exceptionless.Tests/Plugins/EventUpgraderTests.cs @@ -1,3 +1,6 @@ +using System.Text.Json; +using System.Text.Json.Nodes; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Plugins.EventParser; using Exceptionless.Core.Plugins.EventUpgrader; using Xunit; @@ -8,11 +11,13 @@ public sealed class EventUpgraderTests : TestWithServices { private readonly EventUpgraderPluginManager _upgrader; private readonly EventParserPluginManager _parser; + private readonly JsonSerializerOptions _jsonOptions; public EventUpgraderTests(ITestOutputHelper output) : base(output) { _upgrader = GetService(); _parser = GetService(); + _jsonOptions = GetService(); } [Theory] @@ -24,9 +29,12 @@ public void ParseErrors(string errorFilePath) _upgrader.Upgrade(ctx); string expectedContent = File.ReadAllText(Path.ChangeExtension(errorFilePath, ".expected.json")); - Assert.Equal(expectedContent, ctx.Documents.First?.ToString()); + var expected = JsonNode.Parse(expectedContent); + var actual = JsonNode.Parse(ctx.Documents.First().ToFormattedString(_jsonOptions)); + Assert.True(JsonNode.DeepEquals(expected, actual), + $"File: {Path.GetFileName(errorFilePath)}\nExpected:\n{expected?.ToJsonString(new JsonSerializerOptions { WriteIndented = true })}\n\nActual:\n{actual?.ToJsonString(new JsonSerializerOptions { WriteIndented = true })}"); - var events = _parser.ParseEvents(ctx.Documents.ToString(), 2, "exceptionless/2.0.0.0"); + var events = _parser.ParseEvents(ctx.Documents.ToFormattedString(_jsonOptions), 2, "exceptionless/2.0.0.0"); Assert.Single(events); } diff --git a/tests/Exceptionless.Tests/Plugins/GeoTests.cs b/tests/Exceptionless.Tests/Plugins/GeoTests.cs index fe6c576fb5..72492967f0 100644 --- a/tests/Exceptionless.Tests/Plugins/GeoTests.cs +++ b/tests/Exceptionless.Tests/Plugins/GeoTests.cs @@ -1,7 +1,7 @@ using System.Diagnostics; -using System.Text.Json; using Exceptionless.Core; using Exceptionless.Core.Billing; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Geo; using Exceptionless.Core.Jobs; using Exceptionless.Core.Models; @@ -13,6 +13,7 @@ using Exceptionless.Tests.Utility; using Foundatio.Caching; using Foundatio.Resilience; +using Foundatio.Serializer; using Foundatio.Storage; using Xunit; @@ -29,7 +30,7 @@ public sealed class GeoTests : TestWithServices private readonly AppOptions _options; private readonly OrganizationData _organizationData; private readonly ProjectData _projectData; - private readonly JsonSerializerOptions _jsonOptions; + private readonly ITextSerializer _serializer; public GeoTests(ITestOutputHelper output) : base(output) { @@ -38,7 +39,7 @@ public GeoTests(ITestOutputHelper output) : base(output) _options = GetService(); _organizationData = GetService(); _projectData = GetService(); - _jsonOptions = GetService(); + _serializer = GetService(); } private async Task GetResolverAsync(ILoggerFactory loggerFactory) @@ -74,12 +75,12 @@ public async Task WillNotSetLocation() if (resolver is NullGeoIpService) return; - var plugin = new GeoPlugin(resolver, _jsonOptions, _options, Log); + var plugin = new GeoPlugin(resolver, _serializer, _options, Log); var ev = new PersistentEvent { Geo = GREEN_BAY_COORDINATES }; await plugin.EventBatchProcessingAsync(new List { new(ev, _organizationData.GenerateSampleOrganization(_billingManager, _plans), _projectData.GenerateSampleProject()) }); Assert.Equal(GREEN_BAY_COORDINATES, ev.Geo); - Assert.Null(ev.GetLocation(_jsonOptions)); + Assert.Null(ev.GetLocation(_serializer)); } [Theory] @@ -94,12 +95,12 @@ public async Task WillResetLocation(string? geo) if (resolver is NullGeoIpService) return; - var plugin = new GeoPlugin(resolver, _jsonOptions, _options, Log); + var plugin = new GeoPlugin(resolver, _serializer, _options, Log); var ev = new PersistentEvent { Geo = geo }; await plugin.EventBatchProcessingAsync(new List { new(ev, _organizationData.GenerateSampleOrganization(_billingManager, _plans), _projectData.GenerateSampleProject()) }); Assert.Null(ev.Geo); - Assert.Null(ev.GetLocation(_jsonOptions)); + Assert.Null(ev.GetLocation(_serializer)); } [Fact] @@ -109,14 +110,14 @@ public async Task WillSetLocationFromGeo() if (resolver is NullGeoIpService) return; - var plugin = new GeoPlugin(resolver, _jsonOptions, _options, Log); + var plugin = new GeoPlugin(resolver, _serializer, _options, Log); var ev = new PersistentEvent { Geo = GREEN_BAY_IP }; await plugin.EventBatchProcessingAsync(new List { new(ev, _organizationData.GenerateSampleOrganization(_billingManager, _plans), _projectData.GenerateSampleProject()) }); Assert.NotNull(ev.Geo); Assert.NotEqual(GREEN_BAY_IP, ev.Geo); - var location = ev.GetLocation(_jsonOptions); + var location = ev.GetLocation(_serializer); Assert.Equal("US", location?.Country); Assert.Equal("WI", location?.Level1); Assert.Equal("Green Bay", location?.Locality); @@ -129,14 +130,14 @@ public async Task WillSetLocationFromRequestInfo() if (resolver is NullGeoIpService) return; - var plugin = new GeoPlugin(resolver, _jsonOptions, _options, Log); + var plugin = new GeoPlugin(resolver, _serializer, _options, Log); var ev = new PersistentEvent(); ev.AddRequestInfo(new RequestInfo { ClientIpAddress = GREEN_BAY_IP }); await plugin.EventBatchProcessingAsync(new List { new(ev, _organizationData.GenerateSampleOrganization(_billingManager, _plans), _projectData.GenerateSampleProject()) }); Assert.NotNull(ev.Geo); - var location = ev.GetLocation(_jsonOptions); + var location = ev.GetLocation(_serializer); Assert.Equal("US", location?.Country); Assert.Equal("WI", location?.Level1); Assert.Equal("Green Bay", location?.Locality); @@ -149,14 +150,14 @@ public async Task WillSetLocationFromEnvironmentInfoInfo() if (resolver is NullGeoIpService) return; - var plugin = new GeoPlugin(resolver, _jsonOptions, _options, Log); + var plugin = new GeoPlugin(resolver, _serializer, _options, Log); var ev = new PersistentEvent(); ev.SetEnvironmentInfo(new EnvironmentInfo { IpAddress = $"127.0.0.1,{GREEN_BAY_IP}" }); await plugin.EventBatchProcessingAsync(new List { new(ev, _organizationData.GenerateSampleOrganization(_billingManager, _plans), _projectData.GenerateSampleProject()) }); Assert.NotNull(ev.Geo); - var location = ev.GetLocation(_jsonOptions); + var location = ev.GetLocation(_serializer); Assert.Equal("US", location?.Country); Assert.Equal("WI", location?.Level1); Assert.Equal("Green Bay", location?.Locality); @@ -169,7 +170,7 @@ public async Task WillSetFromSingleGeo() if (resolver is NullGeoIpService) return; - var plugin = new GeoPlugin(resolver, _jsonOptions, _options, Log); + var plugin = new GeoPlugin(resolver, _serializer, _options, Log); var contexts = new List { new(new PersistentEvent { Geo = GREEN_BAY_IP }, _organizationData.GenerateSampleOrganization(_billingManager, _plans), _projectData.GenerateSampleProject()), @@ -182,7 +183,7 @@ public async Task WillSetFromSingleGeo() { AssertCoordinatesAreEqual(GREEN_BAY_COORDINATES, context.Event.Geo); - var location = context.Event.GetLocation(_jsonOptions); + var location = context.Event.GetLocation(_serializer); Assert.Equal("US", location?.Country); Assert.Equal("WI", location?.Level1); Assert.Equal("Green Bay", location?.Locality); @@ -196,7 +197,7 @@ public async Task WillNotSetFromMultipleGeo() if (resolver is NullGeoIpService) return; - var plugin = new GeoPlugin(resolver, _jsonOptions, _options, Log); + var plugin = new GeoPlugin(resolver, _serializer, _options, Log); var ev = new PersistentEvent(); var greenBayEvent = new PersistentEvent { Geo = GREEN_BAY_IP }; @@ -208,13 +209,13 @@ await plugin.EventBatchProcessingAsync(new List { }); AssertCoordinatesAreEqual(GREEN_BAY_COORDINATES, greenBayEvent.Geo); - var location = greenBayEvent.GetLocation(_jsonOptions); + var location = greenBayEvent.GetLocation(_serializer); Assert.Equal("US", location?.Country); Assert.Equal("WI", location?.Level1); Assert.Equal("Green Bay", location?.Locality); AssertCoordinatesAreEqual(IRVING_COORDINATES, irvingEvent.Geo); - location = irvingEvent.GetLocation(_jsonOptions); + location = irvingEvent.GetLocation(_serializer); Assert.Equal("US", location?.Country); Assert.Equal("TX", location?.Level1); Assert.Equal("Irving", location?.Locality); @@ -242,7 +243,7 @@ public async Task WillSetMultipleFromEmptyGeo() if (resolver is NullGeoIpService) return; - var plugin = new GeoPlugin(resolver, _jsonOptions, _options, Log); + var plugin = new GeoPlugin(resolver, _serializer, _options, Log); var ev = new PersistentEvent(); var greenBayEvent = new PersistentEvent(); @@ -256,13 +257,13 @@ await plugin.EventBatchProcessingAsync(new List { }); AssertCoordinatesAreEqual(GREEN_BAY_COORDINATES, greenBayEvent.Geo); - var location = greenBayEvent.GetLocation(_jsonOptions); + var location = greenBayEvent.GetLocation(_serializer); Assert.Equal("US", location?.Country); Assert.Equal("WI", location?.Level1); Assert.Equal("Green Bay", location?.Locality); AssertCoordinatesAreEqual(IRVING_COORDINATES, irvingEvent.Geo); - location = irvingEvent.GetLocation(_jsonOptions); + location = irvingEvent.GetLocation(_serializer); Assert.Equal("US", location?.Country); Assert.Equal("TX", location?.Level1); Assert.Equal("Irving", location?.Locality); diff --git a/tests/Exceptionless.Tests/Plugins/SummaryDataTests.cs b/tests/Exceptionless.Tests/Plugins/SummaryDataTests.cs index 9e0ffad6e6..464c411370 100644 --- a/tests/Exceptionless.Tests/Plugins/SummaryDataTests.cs +++ b/tests/Exceptionless.Tests/Plugins/SummaryDataTests.cs @@ -1,26 +1,31 @@ -using Exceptionless.Core.Extensions; +using System.Text.Json; +using System.Text.Json.Nodes; using Exceptionless.Core.Models; using Exceptionless.Core.Plugins.Formatting; -using Newtonsoft.Json; +using Foundatio.Serializer; using Xunit; namespace Exceptionless.Tests.Plugins; public class SummaryDataTests : TestWithServices { - public SummaryDataTests(ITestOutputHelper output) : base(output) { } + private readonly ITextSerializer _serializer; + private readonly JsonSerializerOptions _jsonOptions; + + public SummaryDataTests(ITestOutputHelper output) : base(output) + { + _serializer = GetService(); + _jsonOptions = GetService(); + } [Theory] [MemberData(nameof(Events))] public async Task EventSummaryData(string path) { - var settings = GetService(); - settings.Formatting = Formatting.Indented; - string json = await File.ReadAllTextAsync(path, TestCancellationToken); Assert.NotNull(json); - var ev = json.FromJson(settings); + var ev = _serializer.Deserialize(json); Assert.NotNull(ev); var data = GetService().GetEventSummaryData(ev); @@ -33,20 +38,20 @@ public async Task EventSummaryData(string path) }; string expectedContent = await File.ReadAllTextAsync(Path.ChangeExtension(path, "summary.json"), TestCancellationToken); - Assert.Equal(expectedContent, JsonConvert.SerializeObject(summary, settings)); + var expected = JsonNode.Parse(expectedContent); + var actual = JsonNode.Parse(JsonSerializer.Serialize(summary, _jsonOptions)); + Assert.True(JsonNode.DeepEquals(expected, actual), + $"File: {Path.GetFileName(path)}\nExpected:\n{expected?.ToJsonString(new JsonSerializerOptions { WriteIndented = true })}\n\nActual:\n{actual?.ToJsonString(new JsonSerializerOptions { WriteIndented = true })}"); } [Theory] [MemberData(nameof(Stacks))] public async Task StackSummaryData(string path) { - var settings = GetService(); - settings.Formatting = Formatting.Indented; - string json = await File.ReadAllTextAsync(path, TestCancellationToken); Assert.NotNull(json); - var stack = json.FromJson(settings); + var stack = _serializer.Deserialize(json); Assert.NotNull(stack); var data = GetService().GetStackSummaryData(stack); @@ -61,7 +66,10 @@ public async Task StackSummaryData(string path) }; string expectedContent = await File.ReadAllTextAsync(Path.ChangeExtension(path, "summary.json"), TestCancellationToken); - Assert.Equal(expectedContent, JsonConvert.SerializeObject(summary, settings)); + var expected = JsonNode.Parse(expectedContent); + var actual = JsonNode.Parse(JsonSerializer.Serialize(summary, _jsonOptions)); + Assert.True(JsonNode.DeepEquals(expected, actual), + $"File: {Path.GetFileName(path)}\nExpected:\n{expected?.ToJsonString(new JsonSerializerOptions { WriteIndented = true })}\n\nActual:\n{actual?.ToJsonString(new JsonSerializerOptions { WriteIndented = true })}"); } public static IEnumerable Events diff --git a/tests/Exceptionless.Tests/Plugins/WebHookDataTests.cs b/tests/Exceptionless.Tests/Plugins/WebHookDataTests.cs index 86e353f1e0..adf24cb28d 100644 --- a/tests/Exceptionless.Tests/Plugins/WebHookDataTests.cs +++ b/tests/Exceptionless.Tests/Plugins/WebHookDataTests.cs @@ -1,15 +1,19 @@ +using System.Text.Json; +using System.Text.Json.Nodes; using Exceptionless.Core.Billing; using Exceptionless.Core.Models; using Exceptionless.Core.Plugins.Formatting; using Exceptionless.Core.Plugins.WebHook; using Exceptionless.Tests.Utility; -using Newtonsoft.Json; +using Foundatio.Serializer; using Xunit; namespace Exceptionless.Tests.Plugins; public sealed class WebHookDataTests : TestWithServices { + private readonly ITextSerializer _serializer; + private readonly JsonSerializerOptions _jsonOptions; private readonly OrganizationData _organizationData; private readonly ProjectData _projectData; private readonly StackData _stackData; @@ -18,6 +22,8 @@ public sealed class WebHookDataTests : TestWithServices public WebHookDataTests(ITestOutputHelper output) : base(output) { + _serializer = GetService(); + _jsonOptions = GetService(); _organizationData = GetService(); _projectData = GetService(); _stackData = GetService(); @@ -29,15 +35,12 @@ public WebHookDataTests(ITestOutputHelper output) : base(output) [MemberData(nameof(WebHookData))] public async Task CreateFromEventAsync(string version, bool expectData) { - var settings = GetService(); - settings.Formatting = Formatting.Indented; object? data = await _webHookData.CreateFromEventAsync(GetWebHookDataContext(version)); if (expectData) { string filePath = Path.GetFullPath(Path.Combine("..", "..", "..", "Plugins", "WebHookData", $"{version}.event.expected.json")); string expectedContent = await File.ReadAllTextAsync(filePath, TestCancellationToken); - string actualContent = JsonConvert.SerializeObject(data, settings); - Assert.Equal(expectedContent, actualContent); + AssertJsonEquivalent(expectedContent, JsonSerializer.Serialize(data, _jsonOptions)); } else { @@ -49,15 +52,12 @@ public async Task CreateFromEventAsync(string version, bool expectData) [MemberData(nameof(WebHookData))] public async Task CanCreateFromStackAsync(string version, bool expectData) { - var settings = GetService(); - settings.Formatting = Formatting.Indented; object? data = await _webHookData.CreateFromStackAsync(GetWebHookDataContext(version)); if (expectData) { string filePath = Path.GetFullPath(Path.Combine("..", "..", "..", "Plugins", "WebHookData", $"{version}.stack.expected.json")); string expectedContent = await File.ReadAllTextAsync(filePath, TestCancellationToken); - string actualContent = JsonConvert.SerializeObject(data, settings); - Assert.Equal(expectedContent, actualContent); + AssertJsonEquivalent(expectedContent, JsonSerializer.Serialize(data, _jsonOptions)); } else { @@ -76,9 +76,6 @@ private WebHookDataContext GetWebHookDataContext(string version) { string json = File.ReadAllText(Path.GetFullPath(Path.Combine("..", "..", "..", "ErrorData", "1477.expected.json"))); - var settings = GetService(); - settings.Formatting = Formatting.Indented; - var hook = new WebHook { Id = TestConstants.WebHookId, @@ -93,7 +90,7 @@ private WebHookDataContext GetWebHookDataContext(string version) var organization = _organizationData.GenerateSampleOrganization(GetService(), GetService()); var project = _projectData.GenerateSampleProject(); - var ev = JsonConvert.DeserializeObject(json, settings); + var ev = _serializer.Deserialize(json); Assert.NotNull(ev); ev.OrganizationId = TestConstants.OrganizationId; ev.ProjectId = TestConstants.ProjectId; @@ -110,4 +107,48 @@ private WebHookDataContext GetWebHookDataContext(string version) return new WebHookDataContext(hook, organization, project, stack, ev); } + + /// + /// Compares two JSON strings semantically, ignoring null properties that may be + /// present in expected but omitted by WhenWritingNull in actual. + /// + private static void AssertJsonEquivalent(string expectedJson, string actualJson) + { + var expected = JsonNode.Parse(expectedJson); + var actual = JsonNode.Parse(actualJson); + RemoveNullProperties(expected); + RemoveNullProperties(actual); + Assert.True(JsonNode.DeepEquals(expected, actual), + $"Expected:\n{expected?.ToJsonString(new JsonSerializerOptions { WriteIndented = true })}\n\nActual:\n{actual?.ToJsonString(new JsonSerializerOptions { WriteIndented = true })}"); + } + + private static void RemoveNullProperties(JsonNode? node) + { + if (node is not JsonObject obj) + { + return; + } + + var keysToRemove = new List(); + foreach (var prop in obj) + { + if (prop.Value is null) + { + keysToRemove.Add(prop.Key); + } + else if (prop.Value is JsonArray arr && arr.Count == 0) + { + keysToRemove.Add(prop.Key); + } + else + { + RemoveNullProperties(prop.Value); + } + } + + foreach (string key in keysToRemove) + { + obj.Remove(key); + } + } } diff --git a/tests/Exceptionless.Tests/Repositories/EventRepositoryTests.cs b/tests/Exceptionless.Tests/Repositories/EventRepositoryTests.cs index c8e0cf9528..d1e38bc37d 100644 --- a/tests/Exceptionless.Tests/Repositories/EventRepositoryTests.cs +++ b/tests/Exceptionless.Tests/Repositories/EventRepositoryTests.cs @@ -1,5 +1,6 @@ using System.Diagnostics; using System.Text.Json; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Models.Data; using Exceptionless.Core.Repositories; @@ -8,6 +9,7 @@ using Exceptionless.Tests.Utility; using Foundatio.Repositories; using Foundatio.Repositories.Utility; +using Foundatio.Serializer; using Xunit; using LogLevel = Microsoft.Extensions.Logging.LogLevel; @@ -21,7 +23,7 @@ public sealed class EventRepositoryTests : IntegrationTestsBase private readonly IEventRepository _repository; private readonly StackData _stackData; private readonly IStackRepository _stackRepository; - private readonly JsonSerializerOptions _jsonOptions; + private readonly ITextSerializer _serializer; public EventRepositoryTests(ITestOutputHelper output, AppWebHostFactory factory) : base(output, factory) { @@ -30,7 +32,7 @@ public EventRepositoryTests(ITestOutputHelper output, AppWebHostFactory factory) _repository = GetService(); _stackData = GetService(); _stackRepository = GetService(); - _jsonOptions = GetService(); + _serializer = GetService(); } [Fact(Skip = "https://github.com/elastic/elasticsearch-net/issues/2463")] @@ -219,7 +221,7 @@ public async Task RemoveAllByClientIpAndDateAsync() Assert.Equal(NUMBER_OF_EVENTS_TO_CREATE, events.Count); events.ForEach(e => { - var ri = e.GetRequestInfo(_jsonOptions); + var ri = e.GetRequestInfo(_serializer); Assert.NotNull(ri); Assert.Equal(_clientIpAddress, ri.ClientIpAddress); }); diff --git a/tests/Exceptionless.Tests/Repositories/ProjectRepositoryTests.cs b/tests/Exceptionless.Tests/Repositories/ProjectRepositoryTests.cs index 5f8dd59dab..8f2ab4d8fc 100644 --- a/tests/Exceptionless.Tests/Repositories/ProjectRepositoryTests.cs +++ b/tests/Exceptionless.Tests/Repositories/ProjectRepositoryTests.cs @@ -7,6 +7,7 @@ using Foundatio.Caching; using Foundatio.Repositories; using Foundatio.Repositories.Models; +using Foundatio.Serializer; using Xunit; namespace Exceptionless.Tests.Repositories; @@ -17,6 +18,7 @@ public sealed class ProjectRepositoryTests : IntegrationTestsBase private readonly OrganizationData _organizationData; private readonly ProjectData _projectData; private readonly IProjectRepository _repository; + private readonly ITextSerializer _serializer; public ProjectRepositoryTests(ITestOutputHelper output, AppWebHostFactory factory) : base(output, factory) { @@ -24,6 +26,7 @@ public ProjectRepositoryTests(ITestOutputHelper output, AppWebHostFactory factor _projectData = GetService(); _cache = GetService(); _repository = GetService(); + _serializer = GetService(); } [Fact] @@ -137,13 +140,13 @@ public async Task CanRoundTripWithCaching() var actual = await _repository.GetByIdAsync(project.Id, o => o.Cache()); Assert.NotNull(actual); Assert.Equal(project.Name, actual.Name); - var actualToken = actual.GetSlackToken(); + var actualToken = actual.GetSlackToken(_serializer); Assert.Equal(token.AccessToken, actualToken?.AccessToken); var actualCache = await _cache.GetAsync>>("Project:" + project.Id); Assert.True(actualCache.HasValue); Assert.Equal(project.Name, actualCache.Value.Single().Document.Name); - var actualCacheToken = actual.GetSlackToken(); + var actualCacheToken = actual.GetSlackToken(_serializer); Assert.Equal(token.AccessToken, actualCacheToken?.AccessToken); } } diff --git a/tests/Exceptionless.Tests/Repositories/StackRepositoryTests.cs b/tests/Exceptionless.Tests/Repositories/StackRepositoryTests.cs index bd765b8129..67151af91e 100644 --- a/tests/Exceptionless.Tests/Repositories/StackRepositoryTests.cs +++ b/tests/Exceptionless.Tests/Repositories/StackRepositoryTests.cs @@ -8,6 +8,7 @@ using Foundatio.Caching; using Foundatio.Repositories; using Foundatio.Repositories.Options; +using Foundatio.Serializer; using Xunit; namespace Exceptionless.Tests.Repositories; @@ -15,12 +16,14 @@ namespace Exceptionless.Tests.Repositories; public sealed class StackRepositoryTests : IntegrationTestsBase { private readonly InMemoryCacheClient _cache; + private readonly ITextSerializer _serializer; private readonly StackData _stackData; private readonly IStackRepository _repository; public StackRepositoryTests(ITestOutputHelper output, AppWebHostFactory factory) : base(output, factory) { _cache = GetService() as InMemoryCacheClient ?? throw new InvalidOperationException(); + _serializer = GetService(); _stackData = GetService(); _repository = GetService(); } @@ -79,7 +82,7 @@ public async Task CanGetByStackHashAsync() Assert.Equal(misses, _cache.Misses); var result = await _repository.GetStackBySignatureHashAsync(stack.ProjectId, stack.SignatureHash); - Assert.Equal(stack.ToJson(), result.ToJson()); + Assert.Equal(_serializer.SerializeToString(stack), _serializer.SerializeToString(result)); Assert.Equal(count + 2, _cache.Count); Assert.Equal(hits + 1, _cache.Hits); Assert.Equal(misses, _cache.Misses); diff --git a/tests/Exceptionless.Tests/Serializer/Models/DataDictionaryTests.cs b/tests/Exceptionless.Tests/Serializer/Models/DataDictionaryTests.cs index cc91b0837c..3a865e4e8f 100644 --- a/tests/Exceptionless.Tests/Serializer/Models/DataDictionaryTests.cs +++ b/tests/Exceptionless.Tests/Serializer/Models/DataDictionaryTests.cs @@ -1,4 +1,3 @@ -using System.Text.Json; using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Models.Data; @@ -15,12 +14,10 @@ namespace Exceptionless.Tests.Serializer.Models; public class DataDictionaryTests : TestWithServices { private readonly ITextSerializer _serializer; - private readonly JsonSerializerOptions _jsonOptions; public DataDictionaryTests(ITestOutputHelper output) : base(output) { _serializer = GetService(); - _jsonOptions = GetService(); } [Fact] @@ -31,7 +28,7 @@ public void GetValue_DirectUserInfoType_ReturnsTypedValue() var data = new DataDictionary { { "user", userInfo } }; // Act - var result = data.GetValue("user", _jsonOptions); + var result = data.GetValue("user", _serializer); // Assert Assert.NotNull(result); @@ -46,7 +43,7 @@ public void GetValue_DirectStringType_ReturnsStringValue() var data = new DataDictionary { { "version", "1.0.0" } }; // Act - string? result = data.GetValue("version", _jsonOptions); + string? result = data.GetValue("version", _serializer); // Assert Assert.Equal("1.0.0", result); @@ -59,7 +56,7 @@ public void GetValue_DirectIntType_ReturnsIntValue() var data = new DataDictionary { { "count", 42 } }; // Act - int result = data.GetValue("count", _jsonOptions); + int result = data.GetValue("count", _serializer); // Assert Assert.Equal(42, result); @@ -73,7 +70,7 @@ public void GetValue_JObjectWithUserInfo_ReturnsTypedUserInfo() var data = new DataDictionary { { "user", jObject } }; // Act - var result = data.GetValue("user", _jsonOptions); + var result = data.GetValue("user", _serializer); // Assert Assert.NotNull(result); @@ -97,7 +94,7 @@ public void GetValue_JObjectWithError_ReturnsTypedError() var data = new DataDictionary { { "@error", jObject } }; // Act - var result = data.GetValue("@error", _jsonOptions); + var result = data.GetValue("@error", _serializer); // Assert Assert.NotNull(result); @@ -123,7 +120,7 @@ public void GetValue_JObjectWithRequestInfo_ReturnsTypedRequestInfo() var data = new DataDictionary { { "@request", jObject } }; // Act - var result = data.GetValue("@request", _jsonOptions); + var result = data.GetValue("@request", _serializer); // Assert Assert.NotNull(result); @@ -149,7 +146,7 @@ public void GetValue_JObjectWithEnvironmentInfo_ReturnsTypedEnvironmentInfo() var data = new DataDictionary { { "@environment", jObject } }; // Act - var result = data.GetValue("@environment", _jsonOptions); + var result = data.GetValue("@environment", _serializer); // Assert Assert.NotNull(result); @@ -176,7 +173,7 @@ public void GetValue_JObjectWithNestedError_ReturnsNestedHierarchy() var data = new DataDictionary { { "@error", jObject } }; // Act - var result = data.GetValue("@error", _jsonOptions); + var result = data.GetValue("@error", _serializer); // Assert Assert.NotNull(result); @@ -194,7 +191,7 @@ public void GetValue_JsonStringWithUserInfo_ReturnsTypedUserInfo() var data = new DataDictionary { { "user", json } }; // Act - var result = data.GetValue("user", _jsonOptions); + var result = data.GetValue("user", _serializer); // Assert Assert.NotNull(result); @@ -211,7 +208,7 @@ public void GetValue_JsonStringWithError_ReturnsTypedError() var data = new DataDictionary { { "@error", json } }; // Act - var result = data.GetValue("@error", _jsonOptions); + var result = data.GetValue("@error", _serializer); // Assert Assert.NotNull(result); @@ -228,7 +225,7 @@ public void GetValue_JsonStringWithRequestInfo_ReturnsTypedRequestInfo() var data = new DataDictionary { { "@request", json } }; // Act - var result = data.GetValue("@request", _jsonOptions); + var result = data.GetValue("@request", _serializer); // Assert Assert.NotNull(result); @@ -245,7 +242,7 @@ public void GetValue_JsonStringWithEnvironmentInfo_ReturnsTypedEnvironmentInfo() var data = new DataDictionary { { "@environment", json } }; // Act - var result = data.GetValue("@environment", _jsonOptions); + var result = data.GetValue("@environment", _serializer); // Assert Assert.NotNull(result); @@ -262,7 +259,7 @@ public void GetValue_JsonStringWithSimpleError_ReturnsTypedSimpleError() var data = new DataDictionary { { "@simple_error", json } }; // Act - var result = data.GetValue("@simple_error", _jsonOptions); + var result = data.GetValue("@simple_error", _serializer); // Assert Assert.NotNull(result); @@ -279,7 +276,7 @@ public void GetValue_JsonStringWithNestedError_ReturnsNestedHierarchy() var data = new DataDictionary { { "@error", json } }; // Act - var result = data.GetValue("@error", _jsonOptions); + var result = data.GetValue("@error", _serializer); // Assert Assert.NotNull(result); @@ -295,7 +292,7 @@ public void GetValue_NonJsonString_ReturnsNull() var data = new DataDictionary { { "text", "not json" } }; // Act - var result = data.GetValue("text", _jsonOptions); + var result = data.GetValue("text", _serializer); // Assert Assert.Null(result); @@ -308,7 +305,7 @@ public void GetValue_MissingKey_ThrowsKeyNotFoundException() var data = new DataDictionary(); // Act & Assert - Assert.Throws(() => data.GetValue("nonexistent", _jsonOptions)); + Assert.Throws(() => data.GetValue("nonexistent", _serializer)); } [Fact] @@ -318,7 +315,7 @@ public void GetValue_NullValue_ReturnsNull() var data = new DataDictionary { { "nullable", null! } }; // Act - var result = data.GetValue("nullable", _jsonOptions); + var result = data.GetValue("nullable", _serializer); // Assert Assert.Null(result); @@ -331,7 +328,7 @@ public void GetValue_IncompatibleType_ReturnsNull() var data = new DataDictionary { { "number", 42 } }; // Act - var result = data.GetValue("number", _jsonOptions); + var result = data.GetValue("number", _serializer); // Assert Assert.Null(result); @@ -346,7 +343,7 @@ public void GetValue_MalformedJsonString_ReturnsDefaultProperties() var data = new DataDictionary { { "user", json } }; // Act - var result = data.GetValue("user", _jsonOptions); + var result = data.GetValue("user", _serializer); // Assert Assert.NotNull(result); @@ -369,7 +366,7 @@ public void Deserialize_DataDictionaryWithUserInfoAfterRoundTrip_PreservesTypedD // Assert Assert.NotNull(deserialized); Assert.True(deserialized.ContainsKey("@user")); - var userInfo = deserialized.GetValue("@user", _jsonOptions); + var userInfo = deserialized.GetValue("@user", _serializer); Assert.NotNull(userInfo); Assert.Equal("user@test.com", userInfo.Identity); Assert.Equal("Test User", userInfo.Name); @@ -394,7 +391,7 @@ public void Deserialize_DataDictionaryWithMixedTypesAfterRoundTrip_PreservesAllT // Assert Assert.NotNull(deserialized); Assert.Equal("hello", deserialized["string_value"]); - Assert.Equal(42, deserialized["int_value"]); + Assert.Equal(42, deserialized["int_value"]); // JSON integers deserialize to int when they fit Assert.True(deserialized["bool_value"] as bool?); } @@ -429,7 +426,7 @@ public void Deserialize_UserInfoAfterRoundTrip_PreservesAllProperties() // Assert Assert.NotNull(deserialized); - var result = deserialized.GetValue("@user", _jsonOptions); + var result = deserialized.GetValue("@user", _serializer); Assert.NotNull(result); Assert.Equal("stj@test.com", result.Identity); Assert.Equal("STJ Test User", result.Name); @@ -463,7 +460,7 @@ public void Deserialize_ErrorAfterRoundTrip_PreservesComplexStructure() // Assert Assert.NotNull(deserialized); - var result = deserialized.GetValue("@error", _jsonOptions); + var result = deserialized.GetValue("@error", _serializer); Assert.NotNull(result); Assert.Equal("Test Exception", result.Message); Assert.Equal("System.InvalidOperationException", result.Type); @@ -495,7 +492,7 @@ public void Deserialize_RequestInfoAfterRoundTrip_PreservesAllProperties() // Assert Assert.NotNull(deserialized); - var result = deserialized.GetValue("@request", _jsonOptions); + var result = deserialized.GetValue("@request", _serializer); Assert.NotNull(result); Assert.Equal("POST", result.HttpMethod); Assert.Equal("/api/events", result.Path); @@ -525,7 +522,7 @@ public void Deserialize_EnvironmentInfoAfterRoundTrip_PreservesAllProperties() // Assert Assert.NotNull(deserialized); - var result = deserialized.GetValue("@environment", _jsonOptions); + var result = deserialized.GetValue("@environment", _serializer); Assert.NotNull(result); Assert.Equal("TEST-MACHINE", result.MachineName); Assert.Equal(16, result.ProcessorCount); @@ -555,7 +552,7 @@ public void Deserialize_NestedErrorAfterRoundTrip_PreservesInnerError() // Assert Assert.NotNull(deserialized); - var result = deserialized.GetValue("@error", _jsonOptions); + var result = deserialized.GetValue("@error", _serializer); Assert.NotNull(result); Assert.Equal("Outer exception", result.Message); Assert.NotNull(result.Inner); @@ -582,12 +579,12 @@ public void Deserialize_MixedDataTypesAfterRoundTrip_PreservesAllTypes() // Assert Assert.NotNull(deserialized); - var userInfo = deserialized.GetValue("@user", _jsonOptions); + var userInfo = deserialized.GetValue("@user", _serializer); Assert.NotNull(userInfo); Assert.Equal("user@test.com", userInfo.Identity); Assert.Equal("1.0.0", deserialized["@version"]); - Assert.Equal(42, deserialized["count"]); + Assert.Equal(42, deserialized["count"]); // JSON integers deserialize to int when they fit Assert.True(deserialized["enabled"] as bool?); } @@ -611,12 +608,12 @@ public void Deserialize_NestedDataDictionaryAfterRoundTrip_PreservesNestedData() // Assert Assert.NotNull(deserialized); - var result = deserialized.GetValue("@user", _jsonOptions); + var result = deserialized.GetValue("@user", _serializer); Assert.NotNull(result); Assert.Equal("user@test.com", result.Identity); Assert.NotNull(result.Data); Assert.Equal("custom_value", result.Data["custom_field"]); - Assert.Equal(100, result.Data["score"]); + Assert.Equal(100, result.Data["score"]); // JSON integers deserialize to int when they fit } [Fact] @@ -631,7 +628,7 @@ public void GetValue_DictionaryOfStringObject_DeserializesToTypedObject() var data = new DataDictionary { { "@user", dictionary } }; // Act - var result = data.GetValue("@user", _jsonOptions); + var result = data.GetValue("@user", _serializer); // Assert Assert.NotNull(result); @@ -651,7 +648,7 @@ public void GetValue_ListOfObjects_DeserializesToTypedCollection() var data = new DataDictionary { { "frames", list } }; // Act - var result = data.GetValue>("frames", _jsonOptions); + var result = data.GetValue>("frames", _serializer); // Assert Assert.NotNull(result); diff --git a/tests/Exceptionless.Tests/Serializer/Models/PersistentEventSerializerTests.cs b/tests/Exceptionless.Tests/Serializer/Models/PersistentEventSerializerTests.cs index 8d77001c8a..180d1b8c31 100644 --- a/tests/Exceptionless.Tests/Serializer/Models/PersistentEventSerializerTests.cs +++ b/tests/Exceptionless.Tests/Serializer/Models/PersistentEventSerializerTests.cs @@ -1,4 +1,5 @@ using System.Text.Json; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Models.Data; using Foundatio.Serializer; @@ -14,13 +15,11 @@ namespace Exceptionless.Tests.Serializer.Models; public class PersistentEventSerializerTests : TestWithServices { private readonly ITextSerializer _serializer; - private readonly JsonSerializerOptions _jsonOptions; private static readonly DateTimeOffset FixedDate = new(2024, 1, 15, 12, 0, 0, TimeSpan.Zero); public PersistentEventSerializerTests(ITestOutputHelper output) : base(output) { _serializer = GetService(); - _jsonOptions = GetService(); TimeProvider.SetUtcNow(FixedDate); } @@ -105,7 +104,7 @@ public void Deserialize_EventWithUserInfo_PreservesTypedUserInfo() // Assert Assert.NotNull(deserialized); - var userInfo = deserialized.GetUserIdentity(_jsonOptions); + var userInfo = deserialized.GetUserIdentity(_serializer); Assert.NotNull(userInfo); Assert.Equal("user@example.com", userInfo.Identity); Assert.Equal("Test User", userInfo.Name); @@ -146,7 +145,7 @@ public void Deserialize_EventWithError_PreservesTypedError() // Assert Assert.NotNull(deserialized); - var error = deserialized.GetError(_jsonOptions); + var error = deserialized.GetError(_serializer); Assert.NotNull(error); Assert.Equal("Test exception", error.Message); Assert.Equal("System.InvalidOperationException", error.Type); @@ -183,7 +182,7 @@ public void Deserialize_EventWithRequestInfo_PreservesTypedRequestInfo() // Assert Assert.NotNull(deserialized); - var request = deserialized.GetRequestInfo(_jsonOptions); + var request = deserialized.GetRequestInfo(_serializer); Assert.NotNull(request); Assert.Equal("POST", request.HttpMethod); Assert.Equal("/api/events", request.Path); @@ -215,7 +214,7 @@ public void Deserialize_EventWithEnvironmentInfo_PreservesTypedEnvironmentInfo() // Assert Assert.NotNull(deserialized); - var env = deserialized.GetEnvironmentInfo(_jsonOptions); + var env = deserialized.GetEnvironmentInfo(_serializer); Assert.NotNull(env); Assert.Equal("PROD-SERVER-01", env.MachineName); Assert.Equal(8, env.ProcessorCount); @@ -270,9 +269,9 @@ public void Deserialize_EventWithAllKnownDataKeys_PreservesAllTypes() // Assert Assert.NotNull(deserialized); - Assert.NotNull(deserialized.GetUserIdentity(_jsonOptions)); - Assert.NotNull(deserialized.GetRequestInfo(_jsonOptions)); - Assert.NotNull(deserialized.GetEnvironmentInfo(_jsonOptions)); + Assert.NotNull(deserialized.GetUserIdentity(_serializer)); + Assert.NotNull(deserialized.GetRequestInfo(_serializer)); + Assert.NotNull(deserialized.GetEnvironmentInfo(_serializer)); Assert.Equal("1.0.0", deserialized.GetVersion()); Assert.Equal("Error", deserialized.GetLevel()); } @@ -328,7 +327,7 @@ public void Deserialize_JsonWithTypedUserData_RetrievesTypedUserInfo() // Assert Assert.NotNull(ev); - var userInfo = ev.GetUserIdentity(_jsonOptions); + var userInfo = ev.GetUserIdentity(_serializer); Assert.NotNull(userInfo); Assert.Equal("parsed@example.com", userInfo.Identity); Assert.Equal("Parsed User", userInfo.Name); diff --git a/tests/Exceptionless.Tests/Serializer/SerializerTests.cs b/tests/Exceptionless.Tests/Serializer/SerializerTests.cs index 9d9d6f51f5..e6dce389cf 100644 --- a/tests/Exceptionless.Tests/Serializer/SerializerTests.cs +++ b/tests/Exceptionless.Tests/Serializer/SerializerTests.cs @@ -1,12 +1,7 @@ -using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Models.Data; using Exceptionless.Core.Services; -using Exceptionless.Serializer; -using Foundatio.Repositories.Extensions; using Foundatio.Serializer; -using Newtonsoft.Json; -using Newtonsoft.Json.Linq; using Xunit; namespace Exceptionless.Tests.Serializer; @@ -21,36 +16,40 @@ public SerializerTests(ITestOutputHelper output) : base(output) } [Fact] - public void CanDeserializeEventWithUnknownNamesAndProperties() + public void CanDeserializeEventWithData() { - const string json = @"{""tags"":[""One"",""Two""],""reference_id"":""12"",""Message"":""Hello"",""SomeString"":""Hi"",""SomeBool"":false,""SomeNum"":1,""UnknownProp"":{""Blah"":""SomeVal""},""Some"":{""Blah"":""SomeVal""},""@error"":{""Message"":""SomeVal"",""SomeProp"":""SomeVal""},""Some2"":""{\""Blah\"":\""SomeVal\""}"",""UnknownSerializedProp"":""{\""Blah\"":\""SomeVal\""}""}"; - var settings = new JsonSerializerSettings(); - var knownDataTypes = new Dictionary - { - { "Some", typeof(SomeModel) }, - { "Some2", typeof(SomeModel) }, - { Event.KnownDataKeys.Error, typeof(Error) } - }; - settings.Converters.Add(new DataObjectConverter(_logger, knownDataTypes)); - settings.Converters.Add(new DataObjectConverter(_logger)); + // Arrange + /* language=json */ + const string json = """{"message":"Hello","data":{"Blah":"SomeVal"}}"""; + + // Act + var ev = _serializer.Deserialize(json); - var ev = json.FromJson(settings); + // Assert Assert.NotNull(ev?.Data); + Assert.Single(ev.Data); + Assert.Equal("Hello", ev.Message); + Assert.Equal("SomeVal", ev.Data["Blah"]); + } + + [Fact] + public void CanRoundTripEventWithUnknownProperties() + { + // Arrange + /* language=json */ + const string json = """{"tags":["One","Two"],"reference_id":"12","message":"Hello","data":{"SomeString":"Hi","SomeBool":false,"SomeNum":1}}"""; - Assert.Equal(8, ev.Data.Count); - Assert.Equal("Hi", ev.Data.GetString("SomeString")); - Assert.False(ev.Data.GetBoolean("SomeBool")); - Assert.Equal(1L, ev.Data["SomeNum"]); - Assert.Equal(typeof(JObject), ev.Data["UnknownProp"]?.GetType()); - Assert.Equal(typeof(JObject), ev.Data["UnknownSerializedProp"]?.GetType()); - Assert.Equal("SomeVal", (string)((dynamic)ev.Data["UnknownProp"]!)?.Blah!); - Assert.Equal(typeof(SomeModel), ev.Data["Some"]?.GetType()); - Assert.Equal(typeof(SomeModel), ev.Data["Some2"]?.GetType()); - Assert.Equal("SomeVal", (ev.Data["Some"] as SomeModel)?.Blah); - Assert.Equal(typeof(Error), ev.Data[Event.KnownDataKeys.Error]?.GetType()); - Assert.Equal("SomeVal", ((Error)ev.Data[Event.KnownDataKeys.Error]!)?.Message); - Assert.Single(((Error)ev.Data[Event.KnownDataKeys.Error]!)?.Data!); - Assert.Equal("SomeVal", ((Error)ev.Data[Event.KnownDataKeys.Error]!)?.Data?["SomeProp"]); + // Act + var ev = _serializer.Deserialize(json); + string roundTrippedJson = _serializer.SerializeToString(ev); + var roundTripped = _serializer.Deserialize(roundTrippedJson); + + // Assert + Assert.NotNull(ev?.Data); + Assert.Equal(3, ev.Data.Count); + Assert.Equal("Hi", ev.Data["SomeString"]); + Assert.Equal(false, ev.Data["SomeBool"]); + Assert.Equal(1, ev.Data["SomeNum"]); Assert.Equal("Hello", ev.Message); Assert.NotNull(ev.Tags); Assert.Equal(2, ev.Tags.Count); @@ -58,55 +57,41 @@ public void CanDeserializeEventWithUnknownNamesAndProperties() Assert.Contains("Two", ev.Tags); Assert.Equal("12", ev.ReferenceId); - const string expectedjson = @"{""Tags"":[""One"",""Two""],""Message"":""Hello"",""Data"":{""SomeString"":""Hi"",""SomeBool"":false,""SomeNum"":1,""UnknownProp"":{""Blah"":""SomeVal""},""Some"":{""Blah"":""SomeVal""},""@error"":{""Modules"":[],""Message"":""SomeVal"",""Data"":{""SomeProp"":""SomeVal""},""StackTrace"":[]},""Some2"":{""Blah"":""SomeVal""},""UnknownSerializedProp"":{""Blah"":""SomeVal""}},""ReferenceId"":""12""}"; - string newjson = ev.ToJson(Formatting.None, new JsonSerializerSettings { DefaultValueHandling = DefaultValueHandling.Ignore, NullValueHandling = NullValueHandling.Ignore }); - Assert.Equal(expectedjson, newjson); + // Verify round-trip preserves data + Assert.NotNull(roundTripped); + Assert.Equal(ev.Message, roundTripped.Message); + Assert.Equal(ev.ReferenceId, roundTripped.ReferenceId); + Assert.Equal(ev.Tags, roundTripped.Tags); + Assert.Equal(ev.Data.Count, roundTripped.Data?.Count); } [Fact] - public void CanDeserializeEventWithInvalidKnownDataTypes() + public void CanRoundTripEventWithKnownDataTypes() { - const string json = @"{""Message"":""Hello"",""Some"":""{\""Blah\"":\""SomeVal\""}"",""@Some"":""{\""Blah\"":\""SomeVal\""}""}"; - const string jsonWithInvalidDataType = @"{""Message"":""Hello"",""@Some"":""Testing"",""@string"":""Testing""}"; - - var settings = new JsonSerializerSettings(); - var knownDataTypes = new Dictionary { - { "Some", typeof(SomeModel) }, - { "@Some", typeof(SomeModel) }, - { "_@Some", typeof(SomeModel) }, - { "@string", typeof(string) } - }; - settings.Converters.Add(new DataObjectConverter(_logger, knownDataTypes)); - - var ev = json.FromJson(settings); - Assert.NotNull(ev?.Data); - Assert.Equal(2, ev.Data.Count); - Assert.True(ev.Data.ContainsKey("Some")); - Assert.Equal("SomeVal", (ev.Data["Some"] as SomeModel)?.Blah); - Assert.True(ev.Data.ContainsKey("@Some")); - Assert.Equal("SomeVal", (ev.Data["@Some"] as SomeModel)?.Blah); - - ev = jsonWithInvalidDataType.FromJson(settings); - Assert.NotNull(ev?.Data); - Assert.Equal(2, ev.Data.Count); - Assert.True(ev.Data.ContainsKey("_@Some1")); - Assert.Equal("Testing", ev.Data["_@Some1"] as string); - Assert.True(ev.Data.ContainsKey("@string")); - Assert.Equal("Testing", ev.Data["@string"] as string); - } + // Arrange - Event with known data types (error, request info) + var ev = new Event + { + Message = "Test error", + Type = Event.KnownTypes.Error, + Data = new DataDictionary + { + { Event.KnownDataKeys.Error, new Error { Message = "Something went wrong", Type = "System.Exception" } }, + { Event.KnownDataKeys.RequestInfo, new RequestInfo { HttpMethod = "GET", Path = "/api/test" } } + } + }; - [Fact] - public void CanDeserializeEventWithData() - { - const string json = @"{""Message"":""Hello"",""Data"":{""Blah"":""SomeVal""}}"; - var settings = new JsonSerializerSettings(); - settings.Converters.Add(new DataObjectConverter(_logger)); + // Act + string json = _serializer.SerializeToString(ev); + var roundTripped = _serializer.Deserialize(json); - var ev = json.FromJson(settings); - Assert.NotNull(ev?.Data); - Assert.Single(ev.Data); - Assert.Equal("Hello", ev.Message); - Assert.Equal("SomeVal", ev.Data["Blah"]); + // Assert + Assert.NotNull(roundTripped); + Assert.Equal(ev.Message, roundTripped.Message); + Assert.Equal(ev.Type, roundTripped.Type); + Assert.NotNull(roundTripped.Data); + Assert.Equal(2, roundTripped.Data.Count); + Assert.True(roundTripped.Data.ContainsKey(Event.KnownDataKeys.Error)); + Assert.True(roundTripped.Data.ContainsKey(Event.KnownDataKeys.RequestInfo)); } [Fact] @@ -131,6 +116,7 @@ public void CanDeserializeWebHook() [Fact] public void CanDeserializeProject() { + /* language=json */ string json = "{\"last_event_date_utc\":\"2020-10-18T20:54:04.3457274+01:00\", \"created_utc\":\"0001-01-01T00:00:00\",\"updated_utc\":\"2020-09-21T04:41:32.7458321Z\"}"; var model = _serializer.Deserialize(json); @@ -333,8 +319,3 @@ public class SampleClass public int Count { get; set; } } } - -public record SomeModel -{ - public required string Blah { get; set; } -} diff --git a/tests/Exceptionless.Tests/Utility/DataBuilder.cs b/tests/Exceptionless.Tests/Utility/DataBuilder.cs index d5997b6dca..1c8bd74f8a 100644 --- a/tests/Exceptionless.Tests/Utility/DataBuilder.cs +++ b/tests/Exceptionless.Tests/Utility/DataBuilder.cs @@ -1,5 +1,4 @@ -using System.Text.Json; -using Exceptionless.Core.Extensions; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Models.Data; using Exceptionless.Core.Plugins.Formatting; @@ -36,9 +35,8 @@ public EventDataBuilder Event() public class EventDataBuilder { private readonly FormattingPluginManager _formattingPluginManager; - private readonly ISerializer _serializer; + private readonly ITextSerializer _serializer; private readonly TimeProvider _timeProvider; - private readonly JsonSerializerOptions _jsonOptions; private readonly ICollection> _stackMutations; private int _additionalEventsToCreate = 0; private readonly PersistentEvent _event = new(); @@ -46,12 +44,11 @@ public class EventDataBuilder private EventDataBuilder? _stackEventBuilder; private bool _isFirstOccurrenceSet = false; - public EventDataBuilder(FormattingPluginManager formattingPluginManager, ISerializer serializer, JsonSerializerOptions jsonOptions, TimeProvider timeProvider) + public EventDataBuilder(FormattingPluginManager formattingPluginManager, ITextSerializer serializer, TimeProvider timeProvider) { _stackMutations = new List>(); _formattingPluginManager = formattingPluginManager; _serializer = serializer; - _jsonOptions = jsonOptions; _timeProvider = timeProvider; } @@ -534,7 +531,7 @@ public EventDataBuilder Snooze(DateTime? snoozeUntil = null) if (_stack.FirstOccurrence < _event.Date) _event.IsFirstOccurrence = false; - var msi = _event.GetManualStackingInfo(_jsonOptions); + var msi = _event.GetManualStackingInfo(_serializer); if (msi is not null) { _stack.Title = msi.Title!; From 13aa27702ae38a3edb23f0561dcafb16708f46e0 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Sun, 1 Mar 2026 01:19:37 +0000 Subject: [PATCH 2/7] fix: address STJ migration bugs and PR review feedback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Fix ConvertJsonElement ternary type coercion: (object)l cast prevents implicit long→double widening in switch expression - Make ObjectToInferredTypesConverter configurable with preferInt64 flag for ES serializer to match JSON.NET DataObjectConverter behavior - Fix ElasticSystemTextJsonSerializer: remove ReadStreamToSpan lifetime bug (span backed by disposed MemoryStream), deserialize from stream directly with MemoryStream fast-path - Fix Serialize indentation: pass JsonWriterOptions to Utf8JsonWriter so SerializationFormatting.Indented actually produces indented output - Handle exponent notation (1e5) as floating-point in ReadNumber - Use double consistently (not decimal) for floating-point to match JSON.NET behavior - Fix RenameAll return value: return whether any renames occurred - Add using var to MemoryStream in EventController and EventPostsJob - Handle empty response bodies in SendRequestAsAsync (STJ throws on empty input, Newtonsoft returned default) - Fix SerializerTests: put unknown properties at root level to test JsonExtensionData→ConvertJsonElement path correctly - Revert AGENTS.md to main Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- AGENTS.md | 23 ------- .../Extensions/DataDictionaryExtensions.cs | 4 +- .../Extensions/JsonNodeExtensions.cs | 48 +------------ .../Extensions/ProjectExtensions.cs | 5 +- src/Exceptionless.Core/Jobs/EventPostsJob.cs | 2 +- src/Exceptionless.Core/Models/Event.cs | 2 +- .../EventUpgrader/Default/V2_EventUpgrade.cs | 7 +- .../ElasticSystemTextJsonSerializer.cs | 57 ++++++++------- .../ObjectToInferredTypesConverter.cs | 69 +++++++++++++------ .../Controllers/EventController.cs | 2 +- .../IntegrationTestsBase.cs | 7 ++ .../Plugins/EventParserTests.cs | 17 +++-- .../Serializer/Models/DataDictionaryTests.cs | 6 +- .../Serializer/SerializerTests.cs | 63 +++++++++++++---- 14 files changed, 160 insertions(+), 152 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 95e1361746..30d6e030e3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -82,26 +82,3 @@ pr-reviewer → security pre-screen (before build!) → dependency audit - Never commit secrets — use environment variables - NuGet feeds are in `NuGet.Config` — don't add sources - Prefer additive documentation updates — don't replace strategic docs wholesale, extend them - -## Serialization Architecture - -The project uses **System.Text.Json (STJ)** exclusively. NEST still brings in Newtonsoft.Json transitively, but all application-level serialization uses STJ: - -| Component | Serializer | Notes | -| -------------- | --------------------------------- | -------------------------------------------- | -| Elasticsearch | `ElasticSystemTextJsonSerializer` | Custom `IElasticsearchSerializer` using STJ | -| Event Upgrader | `System.Text.Json.Nodes` | JsonObject/JsonArray for mutable DOM | -| Data Storage | `SystemTextJsonSerializer` | Via Foundatio's STJ support | -| API | STJ (built-in) | ASP.NET Core default with custom options | - -**Key files:** - -- `ElasticSystemTextJsonSerializer.cs` - Custom `IElasticsearchSerializer` for NEST -- `JsonNodeExtensions.cs` - STJ equivalents of JObject helpers -- `ObjectToInferredTypesConverter.cs` - Handles JObject/JToken from NEST during STJ serialization -- `V*_EventUpgrade.cs` - Event version upgraders using JsonObject - -**Security:** - -- Safe JSON encoding used everywhere (escapes `<`, `>`, `&`, `'` for XSS protection) -- No `UnsafeRelaxedJsonEscaping` in the codebase diff --git a/src/Exceptionless.Core/Extensions/DataDictionaryExtensions.cs b/src/Exceptionless.Core/Extensions/DataDictionaryExtensions.cs index acc3c7bc8b..7b3df3cdd4 100644 --- a/src/Exceptionless.Core/Extensions/DataDictionaryExtensions.cs +++ b/src/Exceptionless.Core/Extensions/DataDictionaryExtensions.cs @@ -78,7 +78,7 @@ public static class DataDictionaryExtensions if (result is not null) return result; } - catch + catch (Exception ex) when (ex is JsonException or InvalidOperationException or FormatException) { // Ignored - fall through to next handler } @@ -94,7 +94,7 @@ public static class DataDictionaryExtensions if (result is not null) return result; } - catch + catch (Exception ex) when (ex is JsonException or InvalidOperationException or FormatException) { // Ignored - fall through to next handler } diff --git a/src/Exceptionless.Core/Extensions/JsonNodeExtensions.cs b/src/Exceptionless.Core/Extensions/JsonNodeExtensions.cs index 85b6f89e17..2e0afb66d1 100644 --- a/src/Exceptionless.Core/Extensions/JsonNodeExtensions.cs +++ b/src/Exceptionless.Core/Extensions/JsonNodeExtensions.cs @@ -1,7 +1,5 @@ -using System.Text.Encodings.Web; using System.Text.Json; using System.Text.Json.Nodes; -using System.Text.Unicode; namespace Exceptionless.Core.Extensions; @@ -11,28 +9,6 @@ namespace Exceptionless.Core.Extensions; /// public static class JsonNodeExtensions { - /// - /// XSS-safe encoder for JSON output formatting. - /// This encoder ensures proper XSS protection while allowing Unicode characters - /// for internationalization support. - /// - /// Security features: - /// - HTML-sensitive characters (<, >, &) are escaped for XSS protection - /// - Single quotes are escaped as \u0027 (per ECMAScript spec) - /// - Control characters are escaped for security - /// - private static readonly JavaScriptEncoder SafeJsonEncoder = JavaScriptEncoder.Create(new TextEncoderSettings(UnicodeRanges.All)); - - /// - /// JSON options with safe XSS encoding for tests. - /// Validates that dangerous characters (<, >, &, ') are properly escaped. - /// Production code should use from DI. - /// - internal static readonly JsonSerializerOptions SafeSerializerOptions = new() - { - Encoder = SafeJsonEncoder - }; - /// /// Checks if a JsonNode is null or empty (no values for objects/arrays). /// @@ -192,11 +168,9 @@ public static bool RenameOrRemoveIfNullOrEmpty(this JsonObject target, string cu /// public static void MoveOrRemoveIfNullOrEmpty(this JsonObject target, JsonObject source, params string[] names) { - foreach (string name in names) + foreach (string name in names.Where(source.ContainsKey)) { - if (!source.TryGetPropertyValue(name, out var value)) - continue; - + source.TryGetPropertyValue(name, out var value); bool isNullOrEmpty = value.IsNullOrEmpty(); source.Remove(name); @@ -222,7 +196,7 @@ public static bool RenameAll(this JsonObject target, string currentName, string obj.Rename(currentName, newName); } - return true; + return objectsWithProperty.Count > 0; } /// @@ -293,14 +267,6 @@ public static bool RenameAll(this JsonObject target, string currentName, string yield return desc; } - /// - /// Converts an object to a JsonNode using System.Text.Json serialization. - /// - public static JsonNode? ToJsonNode(T value, JsonSerializerOptions options) - { - return JsonSerializer.SerializeToNode(value, options); - } - /// /// Checks if a JsonNode has any values (for objects: has properties, for arrays: has items). /// @@ -331,14 +297,6 @@ public static bool HasValues(this JsonNode? node) return array.Deserialize>(options); } - /// - /// Creates a JsonValue from a primitive value. - /// - public static JsonValue? CreateValue(T value) - { - return JsonValue.Create(value); - } - /// /// Converts a JsonNode to a pretty-printed JSON string. /// Uses 2-space indentation. Normalizes dates to match existing data format (Z → +00:00). diff --git a/src/Exceptionless.Core/Extensions/ProjectExtensions.cs b/src/Exceptionless.Core/Extensions/ProjectExtensions.cs index 930fa466b4..d5880de903 100644 --- a/src/Exceptionless.Core/Extensions/ProjectExtensions.cs +++ b/src/Exceptionless.Core/Extensions/ProjectExtensions.cs @@ -1,4 +1,5 @@ using System.Text; +using System.Text.Json; using Exceptionless.Core.Models; using Exceptionless.DateTimeExtensions; using Foundatio.Serializer; @@ -58,9 +59,9 @@ public static string BuildFilter(this IList projects) { return project.Data.GetValue(Project.KnownDataKeys.SlackToken, serializer); } - catch (Exception) + catch (Exception ex) when (ex is JsonException or InvalidOperationException or FormatException) { - // Ignored + // Ignored — data may be stored in an incompatible format } return null; diff --git a/src/Exceptionless.Core/Jobs/EventPostsJob.cs b/src/Exceptionless.Core/Jobs/EventPostsJob.cs index 87efabe916..c7dcbb3806 100644 --- a/src/Exceptionless.Core/Jobs/EventPostsJob.cs +++ b/src/Exceptionless.Core/Jobs/EventPostsJob.cs @@ -302,7 +302,7 @@ private async Task RetryEventsAsync(List eventsToRetry, EventPo { try { - var stream = new MemoryStream(ev.GetBytes(_serializer)); + using var stream = new MemoryStream(ev.GetBytes(_serializer)); // Put this single event back into the queue so we can retry it separately. await _eventPostService.EnqueueAsync(new EventPost(false) diff --git a/src/Exceptionless.Core/Models/Event.cs b/src/Exceptionless.Core/Models/Event.cs index 8007f06aff..b91326439d 100644 --- a/src/Exceptionless.Core/Models/Event.cs +++ b/src/Exceptionless.Core/Models/Event.cs @@ -98,7 +98,7 @@ void IJsonOnDeserialized.OnDeserialized() return element.ValueKind switch { JsonValueKind.String => element.GetString(), - JsonValueKind.Number => element.TryGetInt64(out long l) ? l : element.GetDouble(), + JsonValueKind.Number => element.TryGetInt64(out long l) ? (object)l : element.GetDouble(), JsonValueKind.True => true, JsonValueKind.False => false, JsonValueKind.Null or JsonValueKind.Undefined => null, diff --git a/src/Exceptionless.Core/Plugins/EventUpgrader/Default/V2_EventUpgrade.cs b/src/Exceptionless.Core/Plugins/EventUpgrader/Default/V2_EventUpgrade.cs index fdf43c9f78..96f02f172c 100644 --- a/src/Exceptionless.Core/Plugins/EventUpgrader/Default/V2_EventUpgrade.cs +++ b/src/Exceptionless.Core/Plugins/EventUpgrader/Default/V2_EventUpgrade.cs @@ -150,11 +150,8 @@ private void RenameAndValidateExtraExceptionProperties(string? id, JsonObject er var extraProperties = JsonNode.Parse(json) as JsonObject; if (extraProperties is not null) { - foreach (var property in extraProperties.ToList()) + foreach (var property in extraProperties.ToList().Where(p => !p.Value.IsNullOrEmpty())) { - if (property.Value.IsNullOrEmpty()) - continue; - string dataKey = property.Key; if (extendedData[dataKey] is not null) dataKey = "_" + dataKey; @@ -165,7 +162,7 @@ private void RenameAndValidateExtraExceptionProperties(string? id, JsonObject er } } } - catch (Exception) { } + catch (JsonException) { } if (ext.IsNullOrEmpty()) return; diff --git a/src/Exceptionless.Core/Serialization/ElasticSystemTextJsonSerializer.cs b/src/Exceptionless.Core/Serialization/ElasticSystemTextJsonSerializer.cs index 52aa22cd9c..8a20598845 100644 --- a/src/Exceptionless.Core/Serialization/ElasticSystemTextJsonSerializer.cs +++ b/src/Exceptionless.Core/Serialization/ElasticSystemTextJsonSerializer.cs @@ -59,11 +59,19 @@ private static JsonSerializerOptions CreateOptions(JsonSerializerOptions? baseOp options.DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull; options.WriteIndented = writeIndented; - // Insert Elasticsearch converters at the beginning for priority + // Replace the default ObjectToInferredTypesConverter with one that returns Int64 + // for all integers, matching JSON.NET DataObjectConverter behavior. This ensures + // Event.Data values round-trip through Elasticsearch with consistent types. + var defaultConverter = options.Converters.FirstOrDefault(c => c is ObjectToInferredTypesConverter); + if (defaultConverter is not null) + options.Converters.Remove(defaultConverter); + options.Converters.Insert(0, new ObjectToInferredTypesConverter(preferInt64: true)); + + // Insert Elasticsearch converters for priority // Order matters: more specific converters should come first - options.Converters.Insert(0, new DynamicDictionaryConverter()); - options.Converters.Insert(1, new Iso8601DateTimeOffsetConverter()); - options.Converters.Insert(2, new Iso8601DateTimeConverter()); + options.Converters.Insert(1, new DynamicDictionaryConverter()); + options.Converters.Insert(2, new Iso8601DateTimeOffsetConverter()); + options.Converters.Insert(3, new Iso8601DateTimeConverter()); return options; } @@ -79,8 +87,11 @@ private JsonSerializerOptions GetOptions(SerializationFormatting formatting) => if (IsEmptyStream(stream)) return null; - var buffer = ReadStreamToSpan(stream); - return JsonSerializer.Deserialize(buffer, type, _optionsCompact.Value); + // Fast path: MemoryStream with accessible buffer avoids buffering + if (stream is MemoryStream ms && ms.TryGetBuffer(out var segment)) + return JsonSerializer.Deserialize(segment.AsSpan((int)ms.Position), type, _optionsCompact.Value); + + return JsonSerializer.Deserialize(stream, type, _optionsCompact.Value); } /// @@ -89,15 +100,22 @@ private JsonSerializerOptions GetOptions(SerializationFormatting formatting) => if (IsEmptyStream(stream)) return default; - var buffer = ReadStreamToSpan(stream); - return JsonSerializer.Deserialize(buffer, _optionsCompact.Value); + // Fast path: MemoryStream with accessible buffer avoids buffering + if (stream is MemoryStream ms && ms.TryGetBuffer(out var segment)) + return JsonSerializer.Deserialize(segment.AsSpan((int)ms.Position), _optionsCompact.Value); + + return JsonSerializer.Deserialize(stream, _optionsCompact.Value); } /// public void Serialize(T data, Stream stream, SerializationFormatting formatting = SerializationFormatting.None) { - using var writer = new Utf8JsonWriter(stream); var options = GetOptions(formatting); + using var writer = new Utf8JsonWriter(stream, new JsonWriterOptions + { + Indented = formatting == SerializationFormatting.Indented, + Encoder = options.Encoder + }); if (data is null) { @@ -153,31 +171,10 @@ public Task SerializeAsync( #endregion - #region Stream Helpers - private static bool IsEmptyStream(Stream? stream) { return stream is null || stream == Stream.Null || (stream.CanSeek && stream.Length == 0); } - - private static ReadOnlySpan ReadStreamToSpan(Stream stream) - { - // Fast path: if already a MemoryStream with accessible buffer, use it directly - if (stream is MemoryStream ms && ms.TryGetBuffer(out var segment)) - { - return segment.AsSpan(); - } - - // Slow path: copy to new buffer - using var buffer = stream.CanSeek - ? new MemoryStream((int)stream.Length) - : new MemoryStream(); - - stream.CopyTo(buffer); - return buffer.TryGetBuffer(out var seg) ? seg.AsSpan() : buffer.ToArray(); - } - - #endregion } #region Elasticsearch-Specific Converters diff --git a/src/Exceptionless.Core/Serialization/ObjectToInferredTypesConverter.cs b/src/Exceptionless.Core/Serialization/ObjectToInferredTypesConverter.cs index e824b58f1a..2313dc93d6 100644 --- a/src/Exceptionless.Core/Serialization/ObjectToInferredTypesConverter.cs +++ b/src/Exceptionless.Core/Serialization/ObjectToInferredTypesConverter.cs @@ -18,7 +18,7 @@ namespace Exceptionless.Core.Serialization; /// /// /// true/false → -/// Numbers → (if fits) or +/// Numbers → (if fits), , or ; with preferInt64, always for integers and for floats /// Strings with ISO 8601 date format → /// Other strings → /// null → null @@ -44,6 +44,25 @@ namespace Exceptionless.Core.Serialization; /// public sealed class ObjectToInferredTypesConverter : JsonConverter { + private readonly bool _preferInt64; + + /// + /// Initializes a new instance with default settings (integers that fit Int32 are returned as ). + /// + public ObjectToInferredTypesConverter() : this(preferInt64: false) { } + + /// + /// Initializes a new instance with configurable integer handling. + /// + /// + /// When true, all integers are returned as to match JSON.NET behavior. + /// Used by the Elasticsearch serializer to maintain compatibility with DataObjectConverter. + /// + public ObjectToInferredTypesConverter(bool preferInt64) + { + _preferInt64 = preferInt64; + } + /// public override object? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { @@ -103,7 +122,7 @@ public override void Write(Utf8JsonWriter writer, object? value, JsonSerializerO /// Serializing back would lose the decimal representation /// /// - private static object ReadNumber(ref Utf8JsonReader reader) + private object ReadNumber(ref Utf8JsonReader reader) { // Check the raw text to preserve decimal vs integer representation // This is critical for data integrity - 0.0 should stay as double, not become 0L @@ -111,31 +130,37 @@ private static object ReadNumber(ref Utf8JsonReader reader) ? reader.ValueSequence.ToArray() : reader.ValueSpan; - // If the raw text contains a decimal point, treat as floating-point - if (rawValue.Contains((byte)'.')) + // If the raw text contains a decimal point or exponent, treat as floating-point + if (rawValue.Contains((byte)'.') || rawValue.Contains((byte)'e') || rawValue.Contains((byte)'E')) { - // Try decimal for precise values (e.g., financial data) before double - if (reader.TryGetDecimal(out decimal d)) - return d; + if (_preferInt64) + return reader.GetDouble(); - // Fall back to double for floating-point - return reader.GetDouble(); + return reader.GetDecimal(); } // No decimal point - this is an integer - // Try int32 first for smaller values, then long for larger integers - if (reader.TryGetInt32(out int i)) - return i; + if (_preferInt64) + { + // Match JSON.NET DataObjectConverter behavior: always return Int64 + if (reader.TryGetInt64(out long l)) + return l; + } + else + { + // Default STJ behavior: return smallest fitting integer type + if (reader.TryGetInt32(out int i)) + return i; - if (reader.TryGetInt64(out long l)) - return l; + if (reader.TryGetInt64(out long l)) + return l; + } - // For very large integers, try decimal first to preserve precision - if (reader.TryGetDecimal(out decimal dec)) - return dec; + // For very large integers that don't fit in long, fall back to decimal/double + if (_preferInt64) + return reader.GetDouble(); - // Fall back to double only if decimal also fails - return reader.GetDouble(); + return reader.GetDecimal(); } /// @@ -160,7 +185,7 @@ private static object ReadNumber(ref Utf8JsonReader reader) /// Uses for property name matching, /// consistent with behavior. /// - private static Dictionary ReadObject(ref Utf8JsonReader reader, JsonSerializerOptions options) + private Dictionary ReadObject(ref Utf8JsonReader reader, JsonSerializerOptions options) { var dictionary = new Dictionary(StringComparer.OrdinalIgnoreCase); @@ -186,7 +211,7 @@ private static object ReadNumber(ref Utf8JsonReader reader) /// /// Recursively reads a JSON array into a of objects. /// - private static List ReadArray(ref Utf8JsonReader reader, JsonSerializerOptions options) + private List ReadArray(ref Utf8JsonReader reader, JsonSerializerOptions options) { var list = new List(); @@ -204,7 +229,7 @@ private static object ReadNumber(ref Utf8JsonReader reader) /// /// Reads a single JSON value of any type, dispatching to the appropriate reader method. /// - private static object? ReadValue(ref Utf8JsonReader reader, JsonSerializerOptions options) + private object? ReadValue(ref Utf8JsonReader reader, JsonSerializerOptions options) { return reader.TokenType switch { diff --git a/src/Exceptionless.Web/Controllers/EventController.cs b/src/Exceptionless.Web/Controllers/EventController.cs index e8735eefc4..a37be67be3 100644 --- a/src/Exceptionless.Web/Controllers/EventController.cs +++ b/src/Exceptionless.Web/Controllers/EventController.cs @@ -1125,7 +1125,7 @@ private async Task GetSubmitEventAsync(string? projectId = null, i charSet = contentTypeHeader.Charset.ToString(); } - var stream = new MemoryStream(ev.GetBytes(_serializer)); + using var stream = new MemoryStream(ev.GetBytes(_serializer)); await _eventPostService.EnqueueAsync(new EventPost(_appOptions.EnableArchive) { ApiVersion = apiVersion, diff --git a/tests/Exceptionless.Tests/IntegrationTestsBase.cs b/tests/Exceptionless.Tests/IntegrationTestsBase.cs index 46530e605f..22a812e0c9 100644 --- a/tests/Exceptionless.Tests/IntegrationTestsBase.cs +++ b/tests/Exceptionless.Tests/IntegrationTestsBase.cs @@ -235,6 +235,13 @@ protected async Task SendRequestAsync(Action(ensureSuccess); diff --git a/tests/Exceptionless.Tests/Plugins/EventParserTests.cs b/tests/Exceptionless.Tests/Plugins/EventParserTests.cs index 3852c88a2c..20e0a76851 100644 --- a/tests/Exceptionless.Tests/Plugins/EventParserTests.cs +++ b/tests/Exceptionless.Tests/Plugins/EventParserTests.cs @@ -54,14 +54,21 @@ public void VerifyEventParserSerialization(string eventsFilePath) var events = _parser.ParseEvents(json, 2, "exceptionless/2.0.0.0"); Assert.Single(events); + var ev = events.First(); - // Verify parsed event can round-trip through STJ serialization - string serialized = _serializer.SerializeToString(events.First()); + // Verify round-trip: parse → serialize → deserialize preserves all data. + // Must deserialize as PersistentEvent (same type the parser produces) so + // PersistentEvent-specific properties don't leak into Data via JsonExtensionData. + string serialized = _serializer.SerializeToString(ev); Assert.NotNull(serialized); - var roundTripped = _serializer.Deserialize(serialized); + var roundTripped = _serializer.Deserialize(serialized); Assert.NotNull(roundTripped); - Assert.Equal(events.First().Type, roundTripped.Type); - Assert.Equal(events.First().Message, roundTripped.Message); + Assert.Equal(ev.Type, roundTripped.Type); + Assert.Equal(ev.Message, roundTripped.Message); + Assert.Equal(ev.Source, roundTripped.Source); + Assert.Equal(ev.ReferenceId, roundTripped.ReferenceId); + Assert.Equal(ev.Tags?.Count ?? 0, roundTripped.Tags?.Count ?? 0); + Assert.Equal(ev.Data?.Count ?? 0, roundTripped.Data?.Count ?? 0); } [Theory] diff --git a/tests/Exceptionless.Tests/Serializer/Models/DataDictionaryTests.cs b/tests/Exceptionless.Tests/Serializer/Models/DataDictionaryTests.cs index 3a865e4e8f..24aadbf6aa 100644 --- a/tests/Exceptionless.Tests/Serializer/Models/DataDictionaryTests.cs +++ b/tests/Exceptionless.Tests/Serializer/Models/DataDictionaryTests.cs @@ -391,7 +391,7 @@ public void Deserialize_DataDictionaryWithMixedTypesAfterRoundTrip_PreservesAllT // Assert Assert.NotNull(deserialized); Assert.Equal("hello", deserialized["string_value"]); - Assert.Equal(42, deserialized["int_value"]); // JSON integers deserialize to int when they fit + Assert.Equal(42, deserialized["int_value"]); Assert.True(deserialized["bool_value"] as bool?); } @@ -584,7 +584,7 @@ public void Deserialize_MixedDataTypesAfterRoundTrip_PreservesAllTypes() Assert.Equal("user@test.com", userInfo.Identity); Assert.Equal("1.0.0", deserialized["@version"]); - Assert.Equal(42, deserialized["count"]); // JSON integers deserialize to int when they fit + Assert.Equal(42, deserialized["count"]); Assert.True(deserialized["enabled"] as bool?); } @@ -613,7 +613,7 @@ public void Deserialize_NestedDataDictionaryAfterRoundTrip_PreservesNestedData() Assert.Equal("user@test.com", result.Identity); Assert.NotNull(result.Data); Assert.Equal("custom_value", result.Data["custom_field"]); - Assert.Equal(100, result.Data["score"]); // JSON integers deserialize to int when they fit + Assert.Equal(100, result.Data["score"]); } [Fact] diff --git a/tests/Exceptionless.Tests/Serializer/SerializerTests.cs b/tests/Exceptionless.Tests/Serializer/SerializerTests.cs index e6dce389cf..17904fe30b 100644 --- a/tests/Exceptionless.Tests/Serializer/SerializerTests.cs +++ b/tests/Exceptionless.Tests/Serializer/SerializerTests.cs @@ -1,3 +1,5 @@ +using System.Text.Json; +using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Models.Data; using Exceptionless.Core.Services; @@ -26,30 +28,43 @@ public void CanDeserializeEventWithData() var ev = _serializer.Deserialize(json); // Assert - Assert.NotNull(ev?.Data); + Assert.NotNull(ev); + Assert.NotNull(ev.Data); Assert.Single(ev.Data); Assert.Equal("Hello", ev.Message); Assert.Equal("SomeVal", ev.Data["Blah"]); } [Fact] - public void CanRoundTripEventWithUnknownProperties() + public void CanDeserializeEventWithUnknownNamesAndProperties() { - // Arrange + // Arrange - unknown root properties go through [JsonExtensionData] → ConvertJsonElement. + // With STJ, unknown nested objects stay as JsonElement (GetValue handles typed access). + // Primitives are converted: strings, bools, numbers. Objects/arrays stay as JsonElement. /* language=json */ - const string json = """{"tags":["One","Two"],"reference_id":"12","message":"Hello","data":{"SomeString":"Hi","SomeBool":false,"SomeNum":1}}"""; + const string json = """{"tags":["One","Two"],"reference_id":"12","message":"Hello","SomeString":"Hi","SomeBool":false,"SomeNum":1,"UnknownProp":{"Blah":"SomeVal"},"UnknownSerializedProp":"{\"Blah\":\"SomeVal\"}"}"""; // Act var ev = _serializer.Deserialize(json); - string roundTrippedJson = _serializer.SerializeToString(ev); - var roundTripped = _serializer.Deserialize(roundTrippedJson); - // Assert - Assert.NotNull(ev?.Data); - Assert.Equal(3, ev.Data.Count); + // Assert — verify all properties captured correctly + Assert.NotNull(ev); + Assert.NotNull(ev.Data); + Assert.Equal(5, ev.Data.Count); + + // Primitive types are converted by ConvertJsonElement Assert.Equal("Hi", ev.Data["SomeString"]); Assert.Equal(false, ev.Data["SomeBool"]); - Assert.Equal(1, ev.Data["SomeNum"]); + Assert.Equal(1L, ev.Data["SomeNum"]); + + // Unknown nested objects stay as JsonElement for deferred typed access + Assert.IsType(ev.Data["UnknownProp"]); + var unknownProp = (JsonElement)ev.Data["UnknownProp"]!; + Assert.Equal("SomeVal", unknownProp.GetProperty("Blah").GetString()); + + // Serialized JSON strings stay as strings + Assert.IsType(ev.Data["UnknownSerializedProp"]); + Assert.Equal("Hello", ev.Message); Assert.NotNull(ev.Tags); Assert.Equal(2, ev.Tags.Count); @@ -58,6 +73,8 @@ public void CanRoundTripEventWithUnknownProperties() Assert.Equal("12", ev.ReferenceId); // Verify round-trip preserves data + string roundTrippedJson = _serializer.SerializeToString(ev); + var roundTripped = _serializer.Deserialize(roundTrippedJson); Assert.NotNull(roundTripped); Assert.Equal(ev.Message, roundTripped.Message); Assert.Equal(ev.ReferenceId, roundTripped.ReferenceId); @@ -69,14 +86,22 @@ public void CanRoundTripEventWithUnknownProperties() public void CanRoundTripEventWithKnownDataTypes() { // Arrange - Event with known data types (error, request info) + var originalError = new Error + { + Message = "Something went wrong", + Type = "System.Exception", + Data = new DataDictionary { { "SomeProp", "SomeVal" } } + }; + var originalRequest = new RequestInfo { HttpMethod = "GET", Path = "/api/test" }; + var ev = new Event { Message = "Test error", Type = Event.KnownTypes.Error, Data = new DataDictionary { - { Event.KnownDataKeys.Error, new Error { Message = "Something went wrong", Type = "System.Exception" } }, - { Event.KnownDataKeys.RequestInfo, new RequestInfo { HttpMethod = "GET", Path = "/api/test" } } + { Event.KnownDataKeys.Error, originalError }, + { Event.KnownDataKeys.RequestInfo, originalRequest } } }; @@ -92,6 +117,20 @@ public void CanRoundTripEventWithKnownDataTypes() Assert.Equal(2, roundTripped.Data.Count); Assert.True(roundTripped.Data.ContainsKey(Event.KnownDataKeys.Error)); Assert.True(roundTripped.Data.ContainsKey(Event.KnownDataKeys.RequestInfo)); + + // Verify error data round-tripped with values intact + var error = roundTripped.Data.GetValue(Event.KnownDataKeys.Error, _serializer); + Assert.NotNull(error); + Assert.Equal(originalError.Message, error.Message); + Assert.Equal(originalError.Type, error.Type); + Assert.NotNull(error.Data); + Assert.Equal("SomeVal", error.Data["SomeProp"]); + + // Verify request info round-tripped + var request = roundTripped.Data.GetValue(Event.KnownDataKeys.RequestInfo, _serializer); + Assert.NotNull(request); + Assert.Equal(originalRequest.HttpMethod, request.HttpMethod); + Assert.Equal(originalRequest.Path, request.Path); } [Fact] From 47f134e2770f9c929c907f39d9db36bfc704886d Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Sun, 22 Mar 2026 11:11:53 -0500 Subject: [PATCH 3/7] Migrate serializer, config, and bootstrapper to Elastic.Clients.Elasticsearch Replace IElasticsearchSerializer with Elastic.Transport.Serializer, update ExceptionlessElasticConfiguration to use ElasticsearchClient/ElasticsearchClientSettings/ StaticNodePool, pass ITextSerializer to Foundatio base, and register ElasticsearchClient explicitly in DI. --- src/Exceptionless.Core/Bootstrapper.cs | 3 +- .../Exceptionless.Core.csproj | 4 +- .../ExceptionlessElasticConfiguration.cs | 28 ++-- .../ElasticSystemTextJsonSerializer.cs | 128 ++++++------------ .../HealthChecks/ElasticsearchHealthCheck.cs | 13 +- .../Exceptionless.Tests.csproj | 3 - 6 files changed, 59 insertions(+), 120 deletions(-) diff --git a/src/Exceptionless.Core/Bootstrapper.cs b/src/Exceptionless.Core/Bootstrapper.cs index 229ffc835a..88f7f6a359 100644 --- a/src/Exceptionless.Core/Bootstrapper.cs +++ b/src/Exceptionless.Core/Bootstrapper.cs @@ -1,4 +1,5 @@ using System.Text.Json; +using Elastic.Clients.Elasticsearch; using Exceptionless.Core.Authentication; using Exceptionless.Core.Billing; using Exceptionless.Core.Configuration; @@ -70,7 +71,7 @@ public static void RegisterServices(IServiceCollection services, AppOptions appO })); services.AddSingleton(); - services.AddSingleton(s => s.GetRequiredService().Client); + services.AddSingleton(s => s.GetRequiredService().Client); services.AddSingleton(s => s.GetRequiredService()); services.AddStartupAction(); diff --git a/src/Exceptionless.Core/Exceptionless.Core.csproj b/src/Exceptionless.Core/Exceptionless.Core.csproj index 55f4524012..b4a526b7c0 100644 --- a/src/Exceptionless.Core/Exceptionless.Core.csproj +++ b/src/Exceptionless.Core/Exceptionless.Core.csproj @@ -23,8 +23,6 @@ - - @@ -34,7 +32,7 @@ - + diff --git a/src/Exceptionless.Core/Repositories/Configuration/ExceptionlessElasticConfiguration.cs b/src/Exceptionless.Core/Repositories/Configuration/ExceptionlessElasticConfiguration.cs index 5da06ca9ad..44952764dc 100644 --- a/src/Exceptionless.Core/Repositories/Configuration/ExceptionlessElasticConfiguration.cs +++ b/src/Exceptionless.Core/Repositories/Configuration/ExceptionlessElasticConfiguration.cs @@ -1,4 +1,5 @@ -using Elasticsearch.Net; +using Elastic.Clients.Elasticsearch; +using Elastic.Transport; using Exceptionless.Core.Configuration; using Exceptionless.Core.Extensions; using Exceptionless.Core.Repositories.Queries; @@ -12,8 +13,8 @@ using Foundatio.Repositories.Elasticsearch.Configuration; using Foundatio.Repositories.Elasticsearch.Queries.Builders; using Foundatio.Resilience; +using Foundatio.Serializer; using Microsoft.Extensions.Logging; -using Nest; using System.Text.Json; namespace Exceptionless.Core.Repositories.Configuration; @@ -30,10 +31,11 @@ public ExceptionlessElasticConfiguration( ICacheClient cacheClient, IMessageBus messageBus, IServiceProvider serviceProvider, + ITextSerializer serializer, TimeProvider timeProvider, IResiliencePolicyProvider resiliencePolicyProvider, ILoggerFactory loggerFactory - ) : base(workItemQueue, cacheClient, messageBus, timeProvider, resiliencePolicyProvider, loggerFactory) + ) : base(workItemQueue, cacheClient, messageBus, serializer, timeProvider, resiliencePolicyProvider, loggerFactory) { _appOptions = appOptions; _jsonSerializerOptions = jsonSerializerOptions; @@ -75,36 +77,36 @@ public override void ConfigureGlobalQueryBuilders(ElasticQueryBuilder builder) public UserIndex Users { get; } public WebHookIndex WebHooks { get; } - protected override IElasticClient CreateElasticClient() + protected override ElasticsearchClient CreateElasticClient() { var connectionPool = CreateConnectionPool(); var serializer = new ElasticSystemTextJsonSerializer(_jsonSerializerOptions); - var settings = new ConnectionSettings(connectionPool, (_, _) => serializer); + var settings = new ElasticsearchClientSettings(connectionPool, sourceSerializer: (_, _) => serializer); ConfigureSettings(settings); foreach (var index in Indexes) index.ConfigureSettings(settings); if (!String.IsNullOrEmpty(_appOptions.ElasticsearchOptions.UserName) && !String.IsNullOrEmpty(_appOptions.ElasticsearchOptions.Password)) - settings.BasicAuthentication(_appOptions.ElasticsearchOptions.UserName, _appOptions.ElasticsearchOptions.Password); + settings.Authentication(new BasicAuthentication(_appOptions.ElasticsearchOptions.UserName, _appOptions.ElasticsearchOptions.Password)); - var client = new ElasticClient(settings); + var client = new ElasticsearchClient(settings); return client; } - protected override IConnectionPool CreateConnectionPool() + protected override NodePool CreateConnectionPool() { - var serverUris = Options?.ServerUrl.Split(',').Select(url => new Uri(url)); - return new StaticConnectionPool(serverUris); + var serverUris = Options.ServerUrl?.Split(',').Select(url => new Uri(url)) + ?? throw new InvalidOperationException("ElasticsearchOptions.ServerUrl is not configured."); + return new StaticNodePool(serverUris); } - protected override void ConfigureSettings(ConnectionSettings settings) + protected override void ConfigureSettings(ElasticsearchClientSettings settings) { if (_appOptions.AppMode == AppMode.Development) settings.EnableDebugMode(); - settings.ServerCertificateValidationCallback(CertificateValidations.AllowAll); - settings.EnableApiVersioningHeader(); + settings.ServerCertificateValidationCallback((_, _, _, _) => true); settings.DisableDirectStreaming(); settings.EnableTcpKeepAlive(TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(2)); settings.DefaultFieldNameInferrer(p => p.ToLowerUnderscoredWords()); diff --git a/src/Exceptionless.Core/Serialization/ElasticSystemTextJsonSerializer.cs b/src/Exceptionless.Core/Serialization/ElasticSystemTextJsonSerializer.cs index 8a20598845..8dfb318f7e 100644 --- a/src/Exceptionless.Core/Serialization/ElasticSystemTextJsonSerializer.cs +++ b/src/Exceptionless.Core/Serialization/ElasticSystemTextJsonSerializer.cs @@ -1,26 +1,19 @@ using System.Globalization; using System.Text.Json; using System.Text.Json.Serialization; -using Elasticsearch.Net; +using Elastic.Transport; namespace Exceptionless.Core.Serialization; /// -/// System.Text.Json serializer for Elasticsearch NEST client. +/// System.Text.Json serializer for the Elastic.Clients.Elasticsearch 8.x client. /// -/// This serializer implements to enable the NEST 7.x -/// client to use System.Text.Json instead of Newtonsoft.Json for document serialization. +/// This serializer extends to use System.Text.Json for document +/// serialization instead of the built-in serializer. /// /// Why custom converters are needed: /// /// -/// DynamicDictionary -/// -/// Elasticsearch returns dynamic responses as which STJ -/// doesn't know how to serialize/deserialize. This converter handles the round-trip. -/// -/// -/// /// DateTime/DateTimeOffset /// /// Elasticsearch expects ISO 8601 dates. STJ's default output uses "Z" suffix for UTC @@ -31,7 +24,7 @@ namespace Exceptionless.Core.Serialization; /// /// Thread Safety: This class is thread-safe. Options are lazily initialized once. /// -public sealed class ElasticSystemTextJsonSerializer : IElasticsearchSerializer +public sealed class ElasticSystemTextJsonSerializer : Serializer { private readonly Lazy _optionsIndented; private readonly Lazy _optionsCompact; @@ -69,9 +62,8 @@ private static JsonSerializerOptions CreateOptions(JsonSerializerOptions? baseOp // Insert Elasticsearch converters for priority // Order matters: more specific converters should come first - options.Converters.Insert(1, new DynamicDictionaryConverter()); - options.Converters.Insert(2, new Iso8601DateTimeOffsetConverter()); - options.Converters.Insert(3, new Iso8601DateTimeConverter()); + options.Converters.Insert(1, new Iso8601DateTimeOffsetConverter()); + options.Converters.Insert(2, new Iso8601DateTimeConverter()); return options; } @@ -82,7 +74,7 @@ private JsonSerializerOptions GetOptions(SerializationFormatting formatting) => #region Synchronous API /// - public object? Deserialize(Type type, Stream stream) + public override object? Deserialize(Type type, Stream stream) { if (IsEmptyStream(stream)) return null; @@ -95,20 +87,20 @@ private JsonSerializerOptions GetOptions(SerializationFormatting formatting) => } /// - public T? Deserialize(Stream stream) + public override T Deserialize(Stream stream) { if (IsEmptyStream(stream)) - return default; + return default!; // Fast path: MemoryStream with accessible buffer avoids buffering if (stream is MemoryStream ms && ms.TryGetBuffer(out var segment)) - return JsonSerializer.Deserialize(segment.AsSpan((int)ms.Position), _optionsCompact.Value); + return JsonSerializer.Deserialize(segment.AsSpan((int)ms.Position), _optionsCompact.Value)!; - return JsonSerializer.Deserialize(stream, _optionsCompact.Value); + return JsonSerializer.Deserialize(stream, _optionsCompact.Value)!; } /// - public void Serialize(T data, Stream stream, SerializationFormatting formatting = SerializationFormatting.None) + public override void Serialize(T data, Stream stream, SerializationFormatting formatting = SerializationFormatting.None) { var options = GetOptions(formatting); using var writer = new Utf8JsonWriter(stream, new JsonWriterOptions @@ -133,7 +125,7 @@ public void Serialize(T data, Stream stream, SerializationFormatting formatti #region Asynchronous API /// - public async Task DeserializeAsync(Type type, Stream stream, CancellationToken cancellationToken = default) + public override async ValueTask DeserializeAsync(Type type, Stream stream, CancellationToken cancellationToken = default) { if (IsEmptyStream(stream)) return null; @@ -143,17 +135,30 @@ public void Serialize(T data, Stream stream, SerializationFormatting formatti } /// - public async Task DeserializeAsync(Stream stream, CancellationToken cancellationToken = default) + public override async ValueTask DeserializeAsync(Stream stream, CancellationToken cancellationToken = default) { if (IsEmptyStream(stream)) - return default; + return default!; - return await JsonSerializer.DeserializeAsync(stream, _optionsCompact.Value, cancellationToken) + var result = await JsonSerializer.DeserializeAsync(stream, _optionsCompact.Value, cancellationToken) .ConfigureAwait(false); + return result!; } /// - public Task SerializeAsync( + public override void Serialize(object? data, Type type, Stream stream, SerializationFormatting formatting = SerializationFormatting.None, CancellationToken cancellationToken = default) + { + var options = GetOptions(formatting); + using var writer = new Utf8JsonWriter(stream, new JsonWriterOptions + { + Indented = formatting == SerializationFormatting.Indented, + Encoder = options.Encoder + }); + JsonSerializer.Serialize(writer, data, type, options); + } + + /// + public override Task SerializeAsync( T data, Stream stream, SerializationFormatting formatting = SerializationFormatting.None, @@ -169,6 +174,13 @@ public Task SerializeAsync( return JsonSerializer.SerializeAsync(stream, data, data.GetType(), options, cancellationToken); } + /// + public override Task SerializeAsync(object? data, Type type, Stream stream, SerializationFormatting formatting = SerializationFormatting.None, CancellationToken cancellationToken = default) + { + var options = GetOptions(formatting); + return JsonSerializer.SerializeAsync(stream, data, type, options, cancellationToken); + } + #endregion private static bool IsEmptyStream(Stream? stream) @@ -179,70 +191,6 @@ private static bool IsEmptyStream(Stream? stream) #region Elasticsearch-Specific Converters -/// -/// Converts to/from JSON. -/// -/// Why this converter exists: -/// Elasticsearch.Net uses for dynamic responses (e.g., script fields, -/// aggregation buckets). STJ has no built-in support for this type, so we must provide custom -/// serialization logic. -/// -/// Serialization: Writes as a JSON object with key-value pairs. -/// Deserialization: Reads JSON objects/arrays into DynamicDictionary. -/// -internal sealed class DynamicDictionaryConverter : JsonConverter -{ - public override DynamicDictionary? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) - { - return reader.TokenType switch - { - JsonTokenType.StartArray => ReadFromArray(ref reader, options), - JsonTokenType.StartObject => ReadFromObject(ref reader, options), - JsonTokenType.Null => null, - _ => throw new JsonException($"Unexpected token type {reader.TokenType} when deserializing DynamicDictionary") - }; - } - - private static DynamicDictionary ReadFromArray(ref Utf8JsonReader reader, JsonSerializerOptions options) - { - var array = JsonSerializer.Deserialize(ref reader, options); - var dict = new Dictionary(StringComparer.OrdinalIgnoreCase); - - if (array is not null) - { - for (var i = 0; i < array.Length; i++) - { - dict[i.ToString(CultureInfo.InvariantCulture)] = new DynamicValue(array[i]); - } - } - - return DynamicDictionary.Create(dict); - } - - private static DynamicDictionary ReadFromObject(ref Utf8JsonReader reader, JsonSerializerOptions options) - { - var dict = JsonSerializer.Deserialize>(ref reader, options); - return dict is not null ? DynamicDictionary.Create(dict!) : new DynamicDictionary(); - } - - public override void Write(Utf8JsonWriter writer, DynamicDictionary dictionary, JsonSerializerOptions options) - { - writer.WriteStartObject(); - - foreach (var (key, dynamicValue) in dictionary.GetKeyValues()) - { - // Skip null values (consistent with DefaultIgnoreCondition.WhenWritingNull) - if (dynamicValue?.Value is null) - continue; - - writer.WritePropertyName(key); - JsonSerializer.Serialize(writer, dynamicValue.Value, options); - } - - writer.WriteEndObject(); - } -} - /// /// Converts to/from ISO 8601 format for Elasticsearch. /// diff --git a/src/Exceptionless.Insulation/HealthChecks/ElasticsearchHealthCheck.cs b/src/Exceptionless.Insulation/HealthChecks/ElasticsearchHealthCheck.cs index 4d191de743..8f8c402375 100644 --- a/src/Exceptionless.Insulation/HealthChecks/ElasticsearchHealthCheck.cs +++ b/src/Exceptionless.Insulation/HealthChecks/ElasticsearchHealthCheck.cs @@ -1,9 +1,8 @@ using System.Diagnostics; -using Elasticsearch.Net; +using Elastic.Clients.Elasticsearch; using Exceptionless.Core.Repositories.Configuration; using Microsoft.Extensions.Diagnostics.HealthChecks; using Microsoft.Extensions.Logging; -using Nest; namespace Exceptionless.Insulation.HealthChecks; @@ -24,14 +23,8 @@ public async Task CheckHealthAsync(HealthCheckContext context try { - var pingResult = await _config.Client.LowLevel.PingAsync(ctx: cancellationToken, requestParameters: new PingRequestParameters - { - RequestConfiguration = new RequestConfiguration - { - RequestTimeout = TimeSpan.FromSeconds(60) // 60 seconds is default for NEST - } - }); - bool isSuccess = pingResult.ApiCall.HttpStatusCode == 200; + var pingResult = await _config.Client.PingAsync(cancellationToken); + bool isSuccess = pingResult.IsValidResponse; return isSuccess ? HealthCheckResult.Healthy() : new HealthCheckResult(context.Registration.FailureStatus); } diff --git a/tests/Exceptionless.Tests/Exceptionless.Tests.csproj b/tests/Exceptionless.Tests/Exceptionless.Tests.csproj index 1917a08ec3..67934d2855 100644 --- a/tests/Exceptionless.Tests/Exceptionless.Tests.csproj +++ b/tests/Exceptionless.Tests/Exceptionless.Tests.csproj @@ -8,9 +8,6 @@ - - - From 9f7914b961f553991b80ba384c25836003db47bb Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Sun, 22 Mar 2026 11:12:07 -0500 Subject: [PATCH 4/7] Migrate index configurations to new Elasticsearch 8 fluent API Update all index files to use CreateIndexRequestDescriptor, void returns for ConfigureIndex/ConfigureIndexMapping, expression-based property mappings, DynamicMapping enum, and renamed analysis methods. --- .../Configuration/Indexes/EventIndex.cs | 334 +++++++++--------- .../Indexes/OrganizationIndex.cs | 77 ++-- .../Configuration/Indexes/ProjectIndex.cs | 58 +-- .../Configuration/Indexes/StackIndex.cs | 85 ++--- .../Configuration/Indexes/TokenIndex.cs | 43 +-- .../Configuration/Indexes/UserIndex.cs | 46 +-- .../Configuration/Indexes/WebHookIndex.cs | 27 +- 7 files changed, 345 insertions(+), 325 deletions(-) diff --git a/src/Exceptionless.Core/Repositories/Configuration/Indexes/EventIndex.cs b/src/Exceptionless.Core/Repositories/Configuration/Indexes/EventIndex.cs index 725bd7672b..ebc0076695 100644 --- a/src/Exceptionless.Core/Repositories/Configuration/Indexes/EventIndex.cs +++ b/src/Exceptionless.Core/Repositories/Configuration/Indexes/EventIndex.cs @@ -1,3 +1,7 @@ +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.Analysis; +using Elastic.Clients.Elasticsearch.IndexManagement; +using Elastic.Clients.Elasticsearch.Mapping; using Exceptionless.Core.Configuration; using Exceptionless.Core.Models; using Exceptionless.Core.Models.Data; @@ -10,7 +14,6 @@ using Foundatio.Repositories.Elasticsearch.Queries.Builders; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; -using Nest; namespace Exceptionless.Core.Repositories.Configuration; @@ -42,88 +45,87 @@ protected override void ConfigureQueryBuilder(ElasticQueryBuilder builder) builder.RegisterBefore(new EventStackFilterQueryBuilder(stacksRepository, cacheClient, _configuration.LoggerFactory)); } - public override TypeMappingDescriptor ConfigureIndexMapping(TypeMappingDescriptor map) + public override void ConfigureIndexMapping(TypeMappingDescriptor map) { - var mapping = map - .Dynamic(false) + map + .Dynamic(DynamicMapping.False) .DynamicTemplates(dt => dt - .DynamicTemplate("idx_bool", t => t.Match("*-b").Mapping(m => m.Boolean(s => s))) - .DynamicTemplate("idx_date", t => t.Match("*-d").Mapping(m => m.Date(s => s))) - .DynamicTemplate("idx_number", t => t.Match("*-n").Mapping(m => m.Number(s => s.Type(NumberType.Double)))) - .DynamicTemplate("idx_reference", t => t.Match("*-r").Mapping(m => m.Keyword(s => s.IgnoreAbove(256)))) - .DynamicTemplate("idx_string", t => t.Match("*-s").Mapping(m => m.Keyword(s => s.IgnoreAbove(1024))))) + .Add("idx_bool", t => t.Match("*-b").Mapping(m => m.Boolean(s => { }))) + .Add("idx_date", t => t.Match("*-d").Mapping(m => m.Date(s => { }))) + .Add("idx_number", t => t.Match("*-n").Mapping(m => m.DoubleNumber(s => { }))) + .Add("idx_reference", t => t.Match("*-r").Mapping(m => m.Keyword(s => s.IgnoreAbove(256)))) + .Add("idx_string", t => t.Match("*-s").Mapping(m => m.Keyword(s => s.IgnoreAbove(1024))))) .Properties(p => p .SetupDefaults() - .Keyword(f => f.Name(e => e.Id)) - .Keyword(f => f.Name(e => e.OrganizationId)) - .FieldAlias(a => a.Name(Alias.OrganizationId).Path(f => f.OrganizationId)) - .Keyword(f => f.Name(e => e.ProjectId)) - .FieldAlias(a => a.Name(Alias.ProjectId).Path(f => f.ProjectId)) - .Keyword(f => f.Name(e => e.StackId)) - .FieldAlias(a => a.Name(Alias.StackId).Path(f => f.StackId)) - .Keyword(f => f.Name(e => e.ReferenceId)) - .FieldAlias(a => a.Name(Alias.ReferenceId).Path(f => f.ReferenceId)) - .Text(f => f.Name(e => e.Type).Analyzer(LOWER_KEYWORD_ANALYZER).AddKeywordField()) - .Text(f => f.Name(e => e.Source).Analyzer(STANDARDPLUS_ANALYZER).SearchAnalyzer(WHITESPACE_LOWERCASE_ANALYZER).AddKeywordField()) - .Date(f => f.Name(e => e.Date)) - .Text(f => f.Name(e => e.Message)) - .Text(f => f.Name(e => e.Tags).Analyzer(LOWER_KEYWORD_ANALYZER).AddKeywordField()) - .FieldAlias(a => a.Name(Alias.Tags).Path(f => f.Tags)) - .GeoPoint(f => f.Name(e => e.Geo)) - .Scalar(f => f.Value) - .Scalar(f => f.Count) - .Boolean(f => f.Name(e => e.IsFirstOccurrence)) - .FieldAlias(a => a.Name(Alias.IsFirstOccurrence).Path(f => f.IsFirstOccurrence)) - .Object(f => f.Name(e => e.Idx).Dynamic()) - .Object(f => f.Name(e => e.Data).Properties(p2 => p2 - .AddVersionMapping() - .AddLevelMapping() - .AddSubmissionMethodMapping() - .AddSubmissionClientMapping() - .AddLocationMapping() - .AddRequestInfoMapping() - .AddErrorMapping() - .AddSimpleErrorMapping() - .AddEnvironmentInfoMapping() - .AddUserDescriptionMapping() - .AddUserInfoMapping())) + .Keyword(e => e.Id) + .Keyword(e => e.OrganizationId) + .FieldAlias(Alias.OrganizationId, a => a.Path(f => f.OrganizationId)) + .Keyword(e => e.ProjectId) + .FieldAlias(Alias.ProjectId, a => a.Path(f => f.ProjectId)) + .Keyword(e => e.StackId) + .FieldAlias(Alias.StackId, a => a.Path(f => f.StackId)) + .Keyword(e => e.ReferenceId) + .FieldAlias(Alias.ReferenceId, a => a.Path(f => f.ReferenceId)) + .Text(e => e.Type, t => t.Analyzer(LOWER_KEYWORD_ANALYZER).AddKeywordField()) + .Text(e => e.Source, t => t.Analyzer(STANDARDPLUS_ANALYZER).SearchAnalyzer(WHITESPACE_LOWERCASE_ANALYZER).AddKeywordField()) + .Date(e => e.Date) + .Text(e => e.Message) + .Text(e => e.Tags, t => t.Analyzer(LOWER_KEYWORD_ANALYZER).AddKeywordField()) + .FieldAlias(Alias.Tags, a => a.Path(f => f.Tags)) + .GeoPoint(e => e.Geo) + .DoubleNumber(e => e.Value) + .IntegerNumber(e => e.Count) + .Boolean(e => e.IsFirstOccurrence) + .FieldAlias(Alias.IsFirstOccurrence, a => a.Path(f => f.IsFirstOccurrence)) + .Object(e => e.Idx, o => o.Dynamic(DynamicMapping.True)) + .Object(e => e.Data, o => o.Properties(p2 => p2 + .AddVersionMapping() + .AddLevelMapping() + .AddSubmissionMethodMapping() + .AddSubmissionClientMapping() + .AddLocationMapping() + .AddRequestInfoMapping() + .AddErrorMapping() + .AddSimpleErrorMapping() + .AddEnvironmentInfoMapping() + .AddUserDescriptionMapping() + .AddUserInfoMapping())) .AddCopyToMappings() .AddDataDictionaryAliases() ); - if (Options is not null && Options.EnableMapperSizePlugin) - return mapping.SizeField(s => s.Enabled()); - - return mapping; + // SizeField is not available in the v8 Elastic client + // if (Options is not null && Options.EnableMapperSizePlugin) + // map.SizeField(s => s.Enabled(true)); } - public override CreateIndexDescriptor ConfigureIndex(CreateIndexDescriptor idx) + public override void ConfigureIndex(CreateIndexRequestDescriptor idx) { - return base.ConfigureIndex(idx.Settings(s => s - .Analysis(BuildAnalysis) + base.ConfigureIndex(idx); + idx.Settings(s => s + .Analysis(a => BuildAnalysis(a)) .NumberOfShards(_configuration.Options.NumberOfShards) .NumberOfReplicas(_configuration.Options.NumberOfReplicas) - .Setting("index.mapping.total_fields.limit", _configuration.Options.FieldsLimit) - .Setting("index.mapping.ignore_malformed", true) - .Priority(1))); + .AddOtherSetting("index.mapping.total_fields.limit", _configuration.Options.FieldsLimit.ToString()) + .AddOtherSetting("index.mapping.ignore_malformed", "true") + .Priority(1)); } public override async Task ConfigureAsync() { const string pipeline = "events-pipeline"; - var response = await Configuration.Client.Ingest.PutPipelineAsync(pipeline, d => d.Processors(p => p - .Script(s => new ScriptProcessor - { - Source = FLATTEN_ERRORS_SCRIPT.Replace("\r", String.Empty).Replace("\n", String.Empty).Replace(" ", " ") - }))); + var response = await Configuration.Client.Ingest.PutPipelineAsync(pipeline, d => d + .Processors(p => p.Script(s => s + .Source(FLATTEN_ERRORS_SCRIPT.Replace("\r", String.Empty).Replace("\n", String.Empty).Replace(" ", " "))))); var logger = Configuration.LoggerFactory.CreateLogger(); logger.LogRequest(response); - if (!response.IsValid) + if (!response.IsValidResponse) { - logger.LogError(response.OriginalException, "Error creating the pipeline {Pipeline}: {Message}", pipeline, response.GetErrorMessage()); - throw new ApplicationException($"Error creating the pipeline {pipeline}: {response.GetErrorMessage()}", response.OriginalException); + string errorMessage = response.DebugInformation; + logger.LogError(response.ApiCallDetails.OriginalException, "Error creating the pipeline {Pipeline}: {Message}", pipeline, errorMessage); + throw new ApplicationException($"Error creating the pipeline {pipeline}: {errorMessage}", response.ApiCallDetails.OriginalException); } await base.ConfigureAsync(); @@ -162,40 +164,40 @@ protected override void ConfigureQueryParser(ElasticQueryParserConfiguration con public ElasticsearchOptions Options => _configuration.Options; - private AnalysisDescriptor BuildAnalysis(AnalysisDescriptor ad) + private void BuildAnalysis(IndexSettingsAnalysisDescriptor ad) { - return ad.Analyzers(a => a + ad.Analyzers(a => a .Pattern(COMMA_WHITESPACE_ANALYZER, p => p.Pattern(@"[,\s]+")) - .Custom(EMAIL_ANALYZER, c => c.Filters(EMAIL_TOKEN_FILTER, "lowercase", TLD_STOPWORDS_TOKEN_FILTER, EDGE_NGRAM_TOKEN_FILTER, "unique").Tokenizer("keyword")) - .Custom(VERSION_INDEX_ANALYZER, c => c.Filters(VERSION_PAD1_TOKEN_FILTER, VERSION_PAD2_TOKEN_FILTER, VERSION_PAD3_TOKEN_FILTER, VERSION_PAD4_TOKEN_FILTER, VERSION_TOKEN_FILTER, "lowercase", "unique").Tokenizer("whitespace")) - .Custom(VERSION_SEARCH_ANALYZER, c => c.Filters(VERSION_PAD1_TOKEN_FILTER, VERSION_PAD2_TOKEN_FILTER, VERSION_PAD3_TOKEN_FILTER, VERSION_PAD4_TOKEN_FILTER, "lowercase").Tokenizer("whitespace")) - .Custom(WHITESPACE_LOWERCASE_ANALYZER, c => c.Filters("lowercase").Tokenizer(COMMA_WHITESPACE_TOKENIZER)) - .Custom(TYPENAME_ANALYZER, c => c.Filters(TYPENAME_TOKEN_FILTER, "lowercase", "unique").Tokenizer(TYPENAME_HIERARCHY_TOKENIZER)) - .Custom(STANDARDPLUS_ANALYZER, c => c.Filters(STANDARDPLUS_TOKEN_FILTER, "lowercase", "unique").Tokenizer(COMMA_WHITESPACE_TOKENIZER)) - .Custom(LOWER_KEYWORD_ANALYZER, c => c.Filters("lowercase").Tokenizer("keyword")) - .Custom(HOST_ANALYZER, c => c.Filters("lowercase").Tokenizer(HOST_TOKENIZER)) - .Custom(URL_PATH_ANALYZER, c => c.Filters("lowercase").Tokenizer(URL_PATH_TOKENIZER))) + .Custom(EMAIL_ANALYZER, c => c.Filter(EMAIL_TOKEN_FILTER, "lowercase", TLD_STOPWORDS_TOKEN_FILTER, EDGE_NGRAM_TOKEN_FILTER, "unique").Tokenizer("keyword")) + .Custom(VERSION_INDEX_ANALYZER, c => c.Filter(VERSION_PAD1_TOKEN_FILTER, VERSION_PAD2_TOKEN_FILTER, VERSION_PAD3_TOKEN_FILTER, VERSION_PAD4_TOKEN_FILTER, VERSION_TOKEN_FILTER, "lowercase", "unique").Tokenizer("whitespace")) + .Custom(VERSION_SEARCH_ANALYZER, c => c.Filter(VERSION_PAD1_TOKEN_FILTER, VERSION_PAD2_TOKEN_FILTER, VERSION_PAD3_TOKEN_FILTER, VERSION_PAD4_TOKEN_FILTER, "lowercase").Tokenizer("whitespace")) + .Custom(WHITESPACE_LOWERCASE_ANALYZER, c => c.Filter("lowercase").Tokenizer(COMMA_WHITESPACE_TOKENIZER)) + .Custom(TYPENAME_ANALYZER, c => c.Filter(TYPENAME_TOKEN_FILTER, "lowercase", "unique").Tokenizer(TYPENAME_HIERARCHY_TOKENIZER)) + .Custom(STANDARDPLUS_ANALYZER, c => c.Filter(STANDARDPLUS_TOKEN_FILTER, "lowercase", "unique").Tokenizer(COMMA_WHITESPACE_TOKENIZER)) + .Custom(LOWER_KEYWORD_ANALYZER, c => c.Filter("lowercase").Tokenizer("keyword")) + .Custom(HOST_ANALYZER, c => c.Filter("lowercase").Tokenizer(HOST_TOKENIZER)) + .Custom(URL_PATH_ANALYZER, c => c.Filter("lowercase").Tokenizer(URL_PATH_TOKENIZER))) .TokenFilters(f => f - .EdgeNGram(EDGE_NGRAM_TOKEN_FILTER, p => p.MaxGram(50).MinGram(2).Side(EdgeNGramSide.Front)) - .PatternCapture(EMAIL_TOKEN_FILTER, p => p.PreserveOriginal().Patterns("(\\w+)", "(\\p{L}+)", "(\\d+)", "@(.+)", "@(.+)\\.", "(.+)@")) - .PatternCapture(STANDARDPLUS_TOKEN_FILTER, p => p.PreserveOriginal().Patterns( + .EdgeNGram(EDGE_NGRAM_TOKEN_FILTER, p => p.MaxGram(50).MinGram(2).Side(Elastic.Clients.Elasticsearch.Analysis.EdgeNGramSide.Front)) + .PatternCapture(EMAIL_TOKEN_FILTER, p => p.PreserveOriginal(true).Patterns("(\\w+)", "(\\p{L}+)", "(\\d+)", "@(.+)", "@(.+)\\.", "(.+)@")) + .PatternCapture(STANDARDPLUS_TOKEN_FILTER, p => p.PreserveOriginal(true).Patterns( @"([^\.\(\)\[\]\/\\\{\}\?=&;:\<\>]+)", @"([^\.\(\)\[\]\/\\\{\}\?=&;:\<\>]+[\.\/\\][^\.\(\)\[\]\/\\\{\}\?=&;:\<\>]+)", @"([^\.\(\)\[\]\/\\\{\}\?=&;:\<\>]+[\.\/\\][^\.\(\)\[\]\/\\\{\}\?=&;:\<\>]+[\.\/\\][^\.\(\)\[\]\/\\\{\}\?=&;:\<\>]+)" )) - .PatternCapture(TYPENAME_TOKEN_FILTER, p => p.PreserveOriginal().Patterns(@" ^ (\w+)", @"\.(\w+)", @"([^\(\)]+)")) + .PatternCapture(TYPENAME_TOKEN_FILTER, p => p.PreserveOriginal(true).Patterns(@" ^ (\w+)", @"\.(\w+)", @"([^\(\)]+)")) .PatternCapture(VERSION_TOKEN_FILTER, p => p.Patterns(@"^(\d+)\.", @"^(\d+\.\d+)", @"^(\d+\.\d+\.\d+)")) .PatternReplace(VERSION_PAD1_TOKEN_FILTER, p => p.Pattern(@"(\.|^)(\d{1})(?=\.|-|$)").Replacement("$10000$2")) .PatternReplace(VERSION_PAD2_TOKEN_FILTER, p => p.Pattern(@"(\.|^)(\d{2})(?=\.|-|$)").Replacement("$1000$2")) .PatternReplace(VERSION_PAD3_TOKEN_FILTER, p => p.Pattern(@"(\.|^)(\d{3})(?=\.|-|$)").Replacement("$100$2")) .PatternReplace(VERSION_PAD4_TOKEN_FILTER, p => p.Pattern(@"(\.|^)(\d{4})(?=\.|-|$)").Replacement("$10$2")) - .Stop(TLD_STOPWORDS_TOKEN_FILTER, p => p.StopWords("com", "net", "org", "info", "me", "edu", "mil", "gov", "biz", "co", "io", "dev")) - .WordDelimiter(ALL_WORDS_DELIMITER_TOKEN_FILTER, p => p.CatenateNumbers().PreserveOriginal().CatenateAll().CatenateWords())) + .Stop(TLD_STOPWORDS_TOKEN_FILTER, p => p.Stopwords(new string[] { "com", "net", "org", "info", "me", "edu", "mil", "gov", "biz", "co", "io", "dev" })) + .WordDelimiter(ALL_WORDS_DELIMITER_TOKEN_FILTER, p => p.CatenateNumbers(true).PreserveOriginal(true).CatenateAll(true).CatenateWords(true))) .Tokenizers(t => t - .CharGroup(COMMA_WHITESPACE_TOKENIZER, p => p.TokenizeOnCharacters(",", "whitespace")) - .CharGroup(URL_PATH_TOKENIZER, p => p.TokenizeOnCharacters("/", "-", ".")) - .CharGroup(HOST_TOKENIZER, p => p.TokenizeOnCharacters(".")) - .PathHierarchy(TYPENAME_HIERARCHY_TOKENIZER, p => p.Delimiter('.'))); + .CharGroup(COMMA_WHITESPACE_TOKENIZER, p => p.TokenizeOnChars(",", "whitespace")) + .CharGroup(URL_PATH_TOKENIZER, p => p.TokenizeOnChars("/", "-", ".")) + .CharGroup(HOST_TOKENIZER, p => p.TokenizeOnChars(".")) + .PathHierarchy(TYPENAME_HIERARCHY_TOKENIZER, p => p.Delimiter("."))); } private const string ALL_WORDS_DELIMITER_TOKEN_FILTER = "all_word_delimiter"; @@ -321,128 +323,128 @@ internal static class EventIndexExtensions public static PropertiesDescriptor AddCopyToMappings(this PropertiesDescriptor descriptor) { return descriptor - .Text(f => f.Name(EventIndex.Alias.IpAddress).Analyzer(EventIndex.COMMA_WHITESPACE_ANALYZER)) - .Text(f => f.Name(EventIndex.Alias.OperatingSystem).Analyzer(EventIndex.WHITESPACE_LOWERCASE_ANALYZER).AddKeywordField()) - .Object(f => f.Name(EventIndex.Alias.Error).Properties(p1 => p1 - .Keyword(f3 => f3.Name("code").IgnoreAbove(1024)) - .Text(f3 => f3.Name("message").AddKeywordField()) - .Text(f3 => f3.Name("type").Analyzer(EventIndex.TYPENAME_ANALYZER).SearchAnalyzer(EventIndex.WHITESPACE_LOWERCASE_ANALYZER).AddKeywordField()) - .Text(f6 => f6.Name("targettype").Analyzer(EventIndex.TYPENAME_ANALYZER).SearchAnalyzer(EventIndex.WHITESPACE_LOWERCASE_ANALYZER).AddKeywordField()) - .Text(f6 => f6.Name("targetmethod").Analyzer(EventIndex.TYPENAME_ANALYZER).SearchAnalyzer(EventIndex.WHITESPACE_LOWERCASE_ANALYZER).AddKeywordField()))); + .Text(EventIndex.Alias.IpAddress, t => t.Analyzer(EventIndex.COMMA_WHITESPACE_ANALYZER)) + .Text(EventIndex.Alias.OperatingSystem, t => t.Analyzer(EventIndex.WHITESPACE_LOWERCASE_ANALYZER).AddKeywordField()) + .Object(EventIndex.Alias.Error, o => o.Properties(p1 => p1 + .Keyword("code", k => k.IgnoreAbove(1024)) + .Text("message", t => t.AddKeywordField()) + .Text("type", t => t.Analyzer(EventIndex.TYPENAME_ANALYZER).SearchAnalyzer(EventIndex.WHITESPACE_LOWERCASE_ANALYZER).AddKeywordField()) + .Text("targettype", t => t.Analyzer(EventIndex.TYPENAME_ANALYZER).SearchAnalyzer(EventIndex.WHITESPACE_LOWERCASE_ANALYZER).AddKeywordField()) + .Text("targetmethod", t => t.Analyzer(EventIndex.TYPENAME_ANALYZER).SearchAnalyzer(EventIndex.WHITESPACE_LOWERCASE_ANALYZER).AddKeywordField()))); } public static PropertiesDescriptor AddDataDictionaryAliases(this PropertiesDescriptor descriptor) { return descriptor - .FieldAlias(a => a.Name(EventIndex.Alias.Version).Path(f => (string)f.Data![Event.KnownDataKeys.Version]!)) - .FieldAlias(a => a.Name(EventIndex.Alias.Level).Path(f => (string)f.Data![Event.KnownDataKeys.Level]!)) - .FieldAlias(a => a.Name(EventIndex.Alias.SubmissionMethod).Path(f => (string)f.Data![Event.KnownDataKeys.SubmissionMethod]!)) - .FieldAlias(a => a.Name(EventIndex.Alias.ClientUserAgent).Path(f => (((SubmissionClient)f.Data![Event.KnownDataKeys.SubmissionClient]!)!).UserAgent)) - .FieldAlias(a => a.Name(EventIndex.Alias.ClientVersion).Path(f => (((SubmissionClient)f.Data![Event.KnownDataKeys.SubmissionClient]!)!).Version)) - .FieldAlias(a => a.Name(EventIndex.Alias.LocationCountry).Path(f => (((Location)f.Data![Event.KnownDataKeys.Location]!)!).Country)) - .FieldAlias(a => a.Name(EventIndex.Alias.LocationLevel1).Path(f => (((Location)f.Data![Event.KnownDataKeys.Location]!)!).Level1)) - .FieldAlias(a => a.Name(EventIndex.Alias.LocationLevel2).Path(f => (((Location)f.Data![Event.KnownDataKeys.Location]!)!).Level2)) - .FieldAlias(a => a.Name(EventIndex.Alias.LocationLocality).Path(f => (((Location)f.Data![Event.KnownDataKeys.Location]!)!).Locality)) - .FieldAlias(a => a.Name(EventIndex.Alias.Browser).Path(f => (((RequestInfo)f.Data![Event.KnownDataKeys.RequestInfo]!)!).Data![RequestInfo.KnownDataKeys.Browser])) - .FieldAlias(a => a.Name(EventIndex.Alias.Device).Path(f => (((RequestInfo)f.Data![Event.KnownDataKeys.RequestInfo]!)!).Data![RequestInfo.KnownDataKeys.Device])) - .FieldAlias(a => a.Name(EventIndex.Alias.RequestIsBot).Path(f => (((RequestInfo)f.Data![Event.KnownDataKeys.RequestInfo]!)!).Data![RequestInfo.KnownDataKeys.IsBot])) - .FieldAlias(a => a.Name(EventIndex.Alias.RequestPath).Path(f => (((RequestInfo)f.Data![Event.KnownDataKeys.RequestInfo]!)!).Path)) - .FieldAlias(a => a.Name(EventIndex.Alias.RequestUserAgent).Path(f => (((RequestInfo)f.Data![Event.KnownDataKeys.RequestInfo]!)!).UserAgent)) - .FieldAlias(a => a.Name(EventIndex.Alias.CommandLine).Path(f => (((EnvironmentInfo)f.Data![Event.KnownDataKeys.EnvironmentInfo]!)!).CommandLine)) - .FieldAlias(a => a.Name(EventIndex.Alias.MachineArchitecture).Path(f => (((EnvironmentInfo)f.Data![Event.KnownDataKeys.EnvironmentInfo]!)!).Architecture)) - .FieldAlias(a => a.Name(EventIndex.Alias.MachineName).Path(f => (((EnvironmentInfo)f.Data![Event.KnownDataKeys.EnvironmentInfo]!)!).MachineName)); + .FieldAlias(EventIndex.Alias.Version, a => a.Path($"data.{Event.KnownDataKeys.Version}")) + .FieldAlias(EventIndex.Alias.Level, a => a.Path($"data.{Event.KnownDataKeys.Level}")) + .FieldAlias(EventIndex.Alias.SubmissionMethod, a => a.Path($"data.{Event.KnownDataKeys.SubmissionMethod}")) + .FieldAlias(EventIndex.Alias.ClientUserAgent, a => a.Path($"data.{Event.KnownDataKeys.SubmissionClient}.user_agent")) + .FieldAlias(EventIndex.Alias.ClientVersion, a => a.Path($"data.{Event.KnownDataKeys.SubmissionClient}.version")) + .FieldAlias(EventIndex.Alias.LocationCountry, a => a.Path($"data.{Event.KnownDataKeys.Location}.country")) + .FieldAlias(EventIndex.Alias.LocationLevel1, a => a.Path($"data.{Event.KnownDataKeys.Location}.level1")) + .FieldAlias(EventIndex.Alias.LocationLevel2, a => a.Path($"data.{Event.KnownDataKeys.Location}.level2")) + .FieldAlias(EventIndex.Alias.LocationLocality, a => a.Path($"data.{Event.KnownDataKeys.Location}.locality")) + .FieldAlias(EventIndex.Alias.Browser, a => a.Path($"data.{Event.KnownDataKeys.RequestInfo}.data.{RequestInfo.KnownDataKeys.Browser}")) + .FieldAlias(EventIndex.Alias.Device, a => a.Path($"data.{Event.KnownDataKeys.RequestInfo}.data.{RequestInfo.KnownDataKeys.Device}")) + .FieldAlias(EventIndex.Alias.RequestIsBot, a => a.Path($"data.{Event.KnownDataKeys.RequestInfo}.data.{RequestInfo.KnownDataKeys.IsBot}")) + .FieldAlias(EventIndex.Alias.RequestPath, a => a.Path($"data.{Event.KnownDataKeys.RequestInfo}.path")) + .FieldAlias(EventIndex.Alias.RequestUserAgent, a => a.Path($"data.{Event.KnownDataKeys.RequestInfo}.user_agent")) + .FieldAlias(EventIndex.Alias.CommandLine, a => a.Path($"data.{Event.KnownDataKeys.EnvironmentInfo}.command_line")) + .FieldAlias(EventIndex.Alias.MachineArchitecture, a => a.Path($"data.{Event.KnownDataKeys.EnvironmentInfo}.architecture")) + .FieldAlias(EventIndex.Alias.MachineName, a => a.Path($"data.{Event.KnownDataKeys.EnvironmentInfo}.machine_name")); } - public static PropertiesDescriptor AddVersionMapping(this PropertiesDescriptor descriptor) + public static PropertiesDescriptor AddVersionMapping(this PropertiesDescriptor descriptor) where T : class { - return descriptor.Text(f2 => f2.Name(Event.KnownDataKeys.Version).Analyzer(EventIndex.VERSION_INDEX_ANALYZER).SearchAnalyzer(EventIndex.VERSION_SEARCH_ANALYZER).AddKeywordField()); + return descriptor.Text(Event.KnownDataKeys.Version, t => t.Analyzer(EventIndex.VERSION_INDEX_ANALYZER).SearchAnalyzer(EventIndex.VERSION_SEARCH_ANALYZER).AddKeywordField()); } - public static PropertiesDescriptor AddLevelMapping(this PropertiesDescriptor descriptor) + public static PropertiesDescriptor AddLevelMapping(this PropertiesDescriptor descriptor) where T : class { - return descriptor.Text(f2 => f2.Name(Event.KnownDataKeys.Level).Analyzer(EventIndex.LOWER_KEYWORD_ANALYZER).AddKeywordField()); + return descriptor.Text(Event.KnownDataKeys.Level, t => t.Analyzer(EventIndex.LOWER_KEYWORD_ANALYZER).AddKeywordField()); } - public static PropertiesDescriptor AddSubmissionMethodMapping(this PropertiesDescriptor descriptor) + public static PropertiesDescriptor AddSubmissionMethodMapping(this PropertiesDescriptor descriptor) where T : class { - return descriptor.Keyword(f2 => f2.Name(Event.KnownDataKeys.SubmissionMethod).IgnoreAbove(1024)); + return descriptor.Keyword(Event.KnownDataKeys.SubmissionMethod, k => k.IgnoreAbove(1024)); } - public static PropertiesDescriptor AddSubmissionClientMapping(this PropertiesDescriptor descriptor) + public static PropertiesDescriptor AddSubmissionClientMapping(this PropertiesDescriptor descriptor) where T : class { - return descriptor.Object(f2 => f2.Name(Event.KnownDataKeys.SubmissionClient).Properties(p3 => p3 - .Text(f3 => f3.Name(r => r.IpAddress).Analyzer(EventIndex.COMMA_WHITESPACE_ANALYZER).CopyTo(fd => fd.Field(EventIndex.Alias.IpAddress))) - .Text(f3 => f3.Name(r => r.UserAgent).Analyzer(EventIndex.LOWER_KEYWORD_ANALYZER).AddKeywordField()) - .Keyword(f3 => f3.Name(r => r.Version).IgnoreAbove(1024)))); + return descriptor.Object(Event.KnownDataKeys.SubmissionClient, o => o.Properties(p3 => p3 + .Text("ip_address", t => t.Analyzer(EventIndex.COMMA_WHITESPACE_ANALYZER).CopyTo(EventIndex.Alias.IpAddress)) + .Text("user_agent", t => t.Analyzer(EventIndex.LOWER_KEYWORD_ANALYZER).AddKeywordField()) + .Keyword("version", k => k.IgnoreAbove(1024)))); } - public static PropertiesDescriptor AddLocationMapping(this PropertiesDescriptor descriptor) + public static PropertiesDescriptor AddLocationMapping(this PropertiesDescriptor descriptor) where T : class { - return descriptor.Object(f2 => f2.Name(Event.KnownDataKeys.Location).Properties(p3 => p3 - .Text(f3 => f3.Name(r => r.Country).Analyzer(EventIndex.LOWER_KEYWORD_ANALYZER).AddKeywordField()) - .Keyword(f3 => f3.Name(r => r.Level1).IgnoreAbove(1024)) - .Keyword(f3 => f3.Name(r => r.Level2).IgnoreAbove(1024)) - .Keyword(f3 => f3.Name(r => r.Locality).IgnoreAbove(1024)))); + return descriptor.Object(Event.KnownDataKeys.Location, o => o.Properties(p3 => p3 + .Text("country", t => t.Analyzer(EventIndex.LOWER_KEYWORD_ANALYZER).AddKeywordField()) + .Keyword("level1", k => k.IgnoreAbove(1024)) + .Keyword("level2", k => k.IgnoreAbove(1024)) + .Keyword("locality", k => k.IgnoreAbove(1024)))); } - public static PropertiesDescriptor AddRequestInfoMapping(this PropertiesDescriptor descriptor) + public static PropertiesDescriptor AddRequestInfoMapping(this PropertiesDescriptor descriptor) where T : class { - return descriptor.Object(f2 => f2.Name(Event.KnownDataKeys.RequestInfo).Properties(p3 => p3 - .Text(f3 => f3.Name(r => r.ClientIpAddress).Analyzer(EventIndex.COMMA_WHITESPACE_ANALYZER).CopyTo(fd => fd.Field(EventIndex.Alias.IpAddress))) - .Text(f3 => f3.Name(r => r.UserAgent).AddKeywordField()) - .Text(f3 => f3.Name(r => r.Path).Analyzer(EventIndex.URL_PATH_ANALYZER).AddKeywordField()) - .Text(f3 => f3.Name(r => r.Host).Analyzer(EventIndex.HOST_ANALYZER).AddKeywordField()) - .Scalar(r => r.Port) - .Keyword(f3 => f3.Name(r => r.HttpMethod)) - .Object(f3 => f3.Name(e => e.Data).Properties(p4 => p4 - .Text(f4 => f4.Name(RequestInfo.KnownDataKeys.Browser).Analyzer(EventIndex.WHITESPACE_LOWERCASE_ANALYZER).AddKeywordField()) - .Keyword(f4 => f4.Name(RequestInfo.KnownDataKeys.BrowserVersion).IgnoreAbove(1024)) - .Keyword(f4 => f4.Name(RequestInfo.KnownDataKeys.BrowserMajorVersion).IgnoreAbove(1024)) - .Text(f4 => f4.Name(RequestInfo.KnownDataKeys.Device).Analyzer(EventIndex.WHITESPACE_LOWERCASE_ANALYZER).AddKeywordField()) - .Text(f4 => f4.Name(RequestInfo.KnownDataKeys.OS).Analyzer(EventIndex.WHITESPACE_LOWERCASE_ANALYZER).AddKeywordField().CopyTo(fd => fd.Field(EventIndex.Alias.OperatingSystem)).Index(false)) - .Keyword(f4 => f4.Name(RequestInfo.KnownDataKeys.OSVersion).IgnoreAbove(1024)) - .Keyword(f4 => f4.Name(RequestInfo.KnownDataKeys.OSMajorVersion).IgnoreAbove(1024)) - .Boolean(f4 => f4.Name(RequestInfo.KnownDataKeys.IsBot)))))); + return descriptor.Object(Event.KnownDataKeys.RequestInfo, o => o.Properties(p3 => p3 + .Text("client_ip_address", t => t.Analyzer(EventIndex.COMMA_WHITESPACE_ANALYZER).CopyTo(EventIndex.Alias.IpAddress)) + .Text("user_agent", t => t.AddKeywordField()) + .Text("path", t => t.Analyzer(EventIndex.URL_PATH_ANALYZER).AddKeywordField()) + .Text("host", t => t.Analyzer(EventIndex.HOST_ANALYZER).AddKeywordField()) + .IntegerNumber("port") + .Keyword("http_method") + .Object("data", oi => oi.Properties(p4 => p4 + .Text(RequestInfo.KnownDataKeys.Browser, t => t.Analyzer(EventIndex.WHITESPACE_LOWERCASE_ANALYZER).AddKeywordField()) + .Keyword(RequestInfo.KnownDataKeys.BrowserVersion, k => k.IgnoreAbove(1024)) + .Keyword(RequestInfo.KnownDataKeys.BrowserMajorVersion, k => k.IgnoreAbove(1024)) + .Text(RequestInfo.KnownDataKeys.Device, t => t.Analyzer(EventIndex.WHITESPACE_LOWERCASE_ANALYZER).AddKeywordField()) + .Text(RequestInfo.KnownDataKeys.OS, t => t.Analyzer(EventIndex.WHITESPACE_LOWERCASE_ANALYZER).AddKeywordField().CopyTo(EventIndex.Alias.OperatingSystem)) + .Keyword(RequestInfo.KnownDataKeys.OSVersion, k => k.IgnoreAbove(1024)) + .Keyword(RequestInfo.KnownDataKeys.OSMajorVersion, k => k.IgnoreAbove(1024)) + .Boolean(RequestInfo.KnownDataKeys.IsBot))))); } - public static PropertiesDescriptor AddErrorMapping(this PropertiesDescriptor descriptor) + public static PropertiesDescriptor AddErrorMapping(this PropertiesDescriptor descriptor) where T : class { - return descriptor.Object(f2 => f2.Name(Event.KnownDataKeys.Error).Properties(p3 => p3 - .Object(f4 => f4.Name(e => e.Data).Properties(p4 => p4 - .Object(f5 => f5.Name(Error.KnownDataKeys.TargetInfo).Properties(p5 => p5 - .Keyword(f6 => f6.Name("ExceptionType").IgnoreAbove(1024).CopyTo(fd => fd.Field(EventIndex.Alias.ErrorTargetType))) - .Keyword(f6 => f6.Name("Method").IgnoreAbove(1024).CopyTo(fd => fd.Field(EventIndex.Alias.ErrorTargetMethod))))))))); + return descriptor.Object(Event.KnownDataKeys.Error, o => o.Properties(p3 => p3 + .Object("data", oi => oi.Properties(p4 => p4 + .Object(Error.KnownDataKeys.TargetInfo, oi2 => oi2.Properties(p5 => p5 + .Keyword("ExceptionType", k => k.IgnoreAbove(1024).CopyTo(EventIndex.Alias.ErrorTargetType)) + .Keyword("Method", k => k.IgnoreAbove(1024).CopyTo(EventIndex.Alias.ErrorTargetMethod)))))))); } - public static PropertiesDescriptor AddSimpleErrorMapping(this PropertiesDescriptor descriptor) + public static PropertiesDescriptor AddSimpleErrorMapping(this PropertiesDescriptor descriptor) where T : class { - return descriptor.Object(f2 => f2.Name(Event.KnownDataKeys.SimpleError).Properties(p3 => p3 - .Object(f4 => f4.Name(e => e.Data).Properties(p4 => p4 - .Object(f5 => f5.Name(Error.KnownDataKeys.TargetInfo).Properties(p5 => p5 - .Keyword(f6 => f6.Name("ExceptionType").IgnoreAbove(1024).CopyTo(fd => fd.Field(EventIndex.Alias.ErrorTargetType))))))))); + return descriptor.Object(Event.KnownDataKeys.SimpleError, o => o.Properties(p3 => p3 + .Object("data", oi => oi.Properties(p4 => p4 + .Object(Error.KnownDataKeys.TargetInfo, oi2 => oi2.Properties(p5 => p5 + .Keyword("ExceptionType", k => k.IgnoreAbove(1024).CopyTo(EventIndex.Alias.ErrorTargetType)))))))); } - public static PropertiesDescriptor AddEnvironmentInfoMapping(this PropertiesDescriptor descriptor) + public static PropertiesDescriptor AddEnvironmentInfoMapping(this PropertiesDescriptor descriptor) where T : class { - return descriptor.Object(f2 => f2.Name(Event.KnownDataKeys.EnvironmentInfo).Properties(p3 => p3 - .Text(f3 => f3.Name(r => r.IpAddress).Analyzer(EventIndex.COMMA_WHITESPACE_ANALYZER).CopyTo(fd => fd.Field(EventIndex.Alias.IpAddress))) - .Text(f3 => f3.Name(r => r.MachineName).Analyzer(EventIndex.LOWER_KEYWORD_ANALYZER).AddKeywordField()) - .Text(f3 => f3.Name(r => r.OSName).Analyzer(EventIndex.WHITESPACE_LOWERCASE_ANALYZER).AddKeywordField().CopyTo(fd => fd.Field(EventIndex.Alias.OperatingSystem))) - .Keyword(f3 => f3.Name(r => r.CommandLine).IgnoreAbove(1024)) - .Keyword(f3 => f3.Name(r => r.Architecture).IgnoreAbove(1024)))); + return descriptor.Object(Event.KnownDataKeys.EnvironmentInfo, o => o.Properties(p3 => p3 + .Text("ip_address", t => t.Analyzer(EventIndex.COMMA_WHITESPACE_ANALYZER).CopyTo(EventIndex.Alias.IpAddress)) + .Text("machine_name", t => t.Analyzer(EventIndex.LOWER_KEYWORD_ANALYZER).AddKeywordField()) + .Text("o_s_name", t => t.Analyzer(EventIndex.WHITESPACE_LOWERCASE_ANALYZER).AddKeywordField().CopyTo(EventIndex.Alias.OperatingSystem)) + .Keyword("command_line", k => k.IgnoreAbove(1024)) + .Keyword("architecture", k => k.IgnoreAbove(1024)))); } - public static PropertiesDescriptor AddUserDescriptionMapping(this PropertiesDescriptor descriptor) + public static PropertiesDescriptor AddUserDescriptionMapping(this PropertiesDescriptor descriptor) where T : class { - return descriptor.Object(f2 => f2.Name(Event.KnownDataKeys.UserDescription).Properties(p3 => p3 - .Text(f3 => f3.Name(r => r.Description)) - .Text(f3 => f3.Name(r => r.EmailAddress).Analyzer(EventIndex.EMAIL_ANALYZER).SearchAnalyzer("simple").AddKeywordField().CopyTo(f4 => f4.Field($"data.{Event.KnownDataKeys.UserInfo}.identity"))))); + return descriptor.Object(Event.KnownDataKeys.UserDescription, o => o.Properties(p3 => p3 + .Text("description") + .Text("email_address", t => t.Analyzer(EventIndex.EMAIL_ANALYZER).SearchAnalyzer("simple").AddKeywordField().CopyTo($"data.{Event.KnownDataKeys.UserInfo}.identity")))); } - public static PropertiesDescriptor AddUserInfoMapping(this PropertiesDescriptor descriptor) + public static PropertiesDescriptor AddUserInfoMapping(this PropertiesDescriptor descriptor) where T : class { - return descriptor.Object(f2 => f2.Name(Event.KnownDataKeys.UserInfo).Properties(p3 => p3 - .Text(f3 => f3.Name(r => r.Identity).Analyzer(EventIndex.EMAIL_ANALYZER).SearchAnalyzer(EventIndex.WHITESPACE_LOWERCASE_ANALYZER).AddKeywordField()) - .Text(f3 => f3.Name(r => r.Name).Analyzer(EventIndex.LOWER_KEYWORD_ANALYZER).AddKeywordField()))); + return descriptor.Object(Event.KnownDataKeys.UserInfo, o => o.Properties(p3 => p3 + .Text("identity", t => t.Analyzer(EventIndex.EMAIL_ANALYZER).SearchAnalyzer(EventIndex.WHITESPACE_LOWERCASE_ANALYZER).AddKeywordField()) + .Text("name", t => t.Analyzer(EventIndex.LOWER_KEYWORD_ANALYZER).AddKeywordField()))); } } diff --git a/src/Exceptionless.Core/Repositories/Configuration/Indexes/OrganizationIndex.cs b/src/Exceptionless.Core/Repositories/Configuration/Indexes/OrganizationIndex.cs index 079a916f2e..acd30922b6 100644 --- a/src/Exceptionless.Core/Repositories/Configuration/Indexes/OrganizationIndex.cs +++ b/src/Exceptionless.Core/Repositories/Configuration/Indexes/OrganizationIndex.cs @@ -1,7 +1,10 @@ -using Exceptionless.Core.Models; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.IndexManagement; +using Elastic.Clients.Elasticsearch.Mapping; +using Exceptionless.Core.Models; +using Foundatio.Parsers.ElasticQueries.Extensions; using Foundatio.Repositories.Elasticsearch.Configuration; using Foundatio.Repositories.Elasticsearch.Extensions; -using Nest; namespace Exceptionless.Core.Repositories.Configuration; @@ -15,36 +18,37 @@ public OrganizationIndex(ExceptionlessElasticConfiguration configuration) : base _configuration = configuration; } - public override TypeMappingDescriptor ConfigureIndexMapping(TypeMappingDescriptor map) + public override void ConfigureIndexMapping(TypeMappingDescriptor map) { - return map - .Dynamic(false) + map + .Dynamic(DynamicMapping.False) .Properties(p => p .SetupDefaults() - .Text(f => f.Name(e => e.Name).AddKeywordField()) - .Keyword(f => f.Name(u => u.StripeCustomerId)) - .Boolean(f => f.Name(u => u.HasPremiumFeatures)) - .Keyword(f => f.Name(u => u.PlanId)) - .Keyword(f => f.Name(u => u.PlanName).IgnoreAbove(256)) - .Date(f => f.Name(u => u.SubscribeDate)) - .Number(f => f.Name(u => u.BillingStatus)) - .Scalar(f => f.BillingPrice, f => f) - .Boolean(f => f.Name(u => u.IsSuspended)) - .Scalar(f => f.RetentionDays, f => f) - .Object(f => f.Name(o => o.Invites.First()).Properties(ip => ip - .Keyword(fu => fu.Name(i => i.Token)) - .Text(fu => fu.Name(i => i.EmailAddress).Analyzer(KEYWORD_LOWERCASE_ANALYZER)))) - .Date(f => f.Name(s => s.LastEventDateUtc)) + .Text(e => e.Name, t => t.AddKeywordField()) + .Keyword(e => e.StripeCustomerId) + .Boolean(e => e.HasPremiumFeatures) + .Keyword(e => e.PlanId) + .Keyword(e => e.PlanName, k => k.IgnoreAbove(256)) + .Date(e => e.SubscribeDate) + .IntegerNumber(e => e.BillingStatus) + .DoubleNumber(e => e.BillingPrice) + .Boolean(e => e.IsSuspended) + .IntegerNumber(e => e.RetentionDays) + .Object(e => e.Invites, o => o.Properties(ip => ip + .Keyword("token") + .Text("email_address", t => t.Analyzer(KEYWORD_LOWERCASE_ANALYZER)))) + .Date(e => e.LastEventDateUtc) .AddUsageMappings()); } - public override CreateIndexDescriptor ConfigureIndex(CreateIndexDescriptor idx) + public override void ConfigureIndex(CreateIndexRequestDescriptor idx) { - return base.ConfigureIndex(idx.Settings(s => s - .Analysis(d => d.Analyzers(b => b.Custom(KEYWORD_LOWERCASE_ANALYZER, c => c.Filters("lowercase").Tokenizer("keyword")))) + base.ConfigureIndex(idx); + idx.Settings(s => s + .Analysis(d => d.Analyzers(b => b.Custom(KEYWORD_LOWERCASE_ANALYZER, c => c.Filter("lowercase").Tokenizer("keyword")))) .NumberOfShards(_configuration.Options.NumberOfShards) .NumberOfReplicas(_configuration.Options.NumberOfReplicas) - .Priority(10))); + .Priority(10)); } } @@ -53,19 +57,18 @@ internal static class OrganizationIndexExtensions public static PropertiesDescriptor AddUsageMappings(this PropertiesDescriptor descriptor) { return descriptor - .Object(ui => ui.Name(o => o.Usage.First()).Properties(p => p - .Date(fu => fu.Name(i => i.Date)) - .Number(fu => fu.Name(i => i.Total)) - .Number(fu => fu.Name(i => i.Blocked)) - .Number(fu => fu.Name(i => i.Discarded)) - .Number(fu => fu.Name(i => i.Limit)) - .Number(fu => fu.Name(i => i.TooBig)))) - .Object(ui => ui.Name(o => o.UsageHours.First()).Properties(p => p - .Date(fu => fu.Name(i => i.Date)) - .Number(fu => fu.Name(i => i.Total)) - .Number(fu => fu.Name(i => i.Blocked)) - .Number(fu => fu.Name(i => i.Discarded)) - .Number(fu => fu.Name(i => i.Limit)) - .Number(fu => fu.Name(i => i.TooBig)))); + .Object(o => o.Usage, ui => ui.Properties(p => p + .Date("date") + .IntegerNumber("total") + .IntegerNumber("blocked") + .IntegerNumber("discarded") + .IntegerNumber("limit") + .IntegerNumber("too_big"))) + .Object(o => o.UsageHours, ui => ui.Properties(p => p + .Date("date") + .IntegerNumber("total") + .IntegerNumber("blocked") + .IntegerNumber("discarded") + .IntegerNumber("too_big"))); } } diff --git a/src/Exceptionless.Core/Repositories/Configuration/Indexes/ProjectIndex.cs b/src/Exceptionless.Core/Repositories/Configuration/Indexes/ProjectIndex.cs index 38585e9061..4e44c9b3f4 100644 --- a/src/Exceptionless.Core/Repositories/Configuration/Indexes/ProjectIndex.cs +++ b/src/Exceptionless.Core/Repositories/Configuration/Indexes/ProjectIndex.cs @@ -1,7 +1,10 @@ -using Exceptionless.Core.Models; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.IndexManagement; +using Elastic.Clients.Elasticsearch.Mapping; +using Exceptionless.Core.Models; +using Foundatio.Parsers.ElasticQueries.Extensions; using Foundatio.Repositories.Elasticsearch.Configuration; using Foundatio.Repositories.Elasticsearch.Extensions; -using Nest; namespace Exceptionless.Core.Repositories.Configuration; @@ -15,28 +18,28 @@ public ProjectIndex(ExceptionlessElasticConfiguration configuration) : base(conf _configuration = configuration; } - - public override TypeMappingDescriptor ConfigureIndexMapping(TypeMappingDescriptor map) + public override void ConfigureIndexMapping(TypeMappingDescriptor map) { - return map - .Dynamic(false) + map + .Dynamic(DynamicMapping.False) .Properties(p => p .SetupDefaults() - .Keyword(f => f.Name(e => e.OrganizationId)) - .Text(f => f.Name(e => e.Name).AddKeywordField()) - .Scalar(f => f.NextSummaryEndOfDayTicks, f => f) - .Date(f => f.Name(s => s.LastEventDateUtc)) + .Keyword(e => e.OrganizationId) + .Text(e => e.Name, t => t.AddKeywordField()) + .LongNumber(e => e.NextSummaryEndOfDayTicks) + .Date(e => e.LastEventDateUtc) .AddUsageMappings() ); } - public override CreateIndexDescriptor ConfigureIndex(CreateIndexDescriptor idx) + public override void ConfigureIndex(CreateIndexRequestDescriptor idx) { - return base.ConfigureIndex(idx.Settings(s => s - .Analysis(d => d.Analyzers(b => b.Custom(KEYWORD_LOWERCASE_ANALYZER, c => c.Filters("lowercase").Tokenizer("keyword")))) + base.ConfigureIndex(idx); + idx.Settings(s => s + .Analysis(d => d.Analyzers(b => b.Custom(KEYWORD_LOWERCASE_ANALYZER, c => c.Filter("lowercase").Tokenizer("keyword")))) .NumberOfShards(_configuration.Options.NumberOfShards) .NumberOfReplicas(_configuration.Options.NumberOfReplicas) - .Priority(10))); + .Priority(10)); } } @@ -45,19 +48,18 @@ internal static class ProjectIndexExtensions public static PropertiesDescriptor AddUsageMappings(this PropertiesDescriptor descriptor) { return descriptor - .Object(ui => ui.Name(o => o.Usage.First()).Properties(p => p - .Date(fu => fu.Name(i => i.Date)) - .Number(fu => fu.Name(i => i.Total)) - .Number(fu => fu.Name(i => i.Blocked)) - .Number(fu => fu.Name(i => i.Discarded)) - .Number(fu => fu.Name(i => i.Limit)) - .Number(fu => fu.Name(i => i.TooBig)))) - .Object(ui => ui.Name(o => o.UsageHours.First()).Properties(p => p - .Date(fu => fu.Name(i => i.Date)) - .Number(fu => fu.Name(i => i.Total)) - .Number(fu => fu.Name(i => i.Blocked)) - .Number(fu => fu.Name(i => i.Discarded)) - .Number(fu => fu.Name(i => i.Limit)) - .Number(fu => fu.Name(i => i.TooBig)))); + .Object(o => o.Usage, ui => ui.Properties(p => p + .Date("date") + .IntegerNumber("total") + .IntegerNumber("blocked") + .IntegerNumber("discarded") + .IntegerNumber("limit") + .IntegerNumber("too_big"))) + .Object(o => o.UsageHours, ui => ui.Properties(p => p + .Date("date") + .IntegerNumber("total") + .IntegerNumber("blocked") + .IntegerNumber("discarded") + .IntegerNumber("too_big"))); } } diff --git a/src/Exceptionless.Core/Repositories/Configuration/Indexes/StackIndex.cs b/src/Exceptionless.Core/Repositories/Configuration/Indexes/StackIndex.cs index aa9ad57b92..4e2dc3de99 100644 --- a/src/Exceptionless.Core/Repositories/Configuration/Indexes/StackIndex.cs +++ b/src/Exceptionless.Core/Repositories/Configuration/Indexes/StackIndex.cs @@ -1,9 +1,11 @@ +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.IndexManagement; +using Elastic.Clients.Elasticsearch.Mapping; using Exceptionless.Core.Models; using Exceptionless.Core.Repositories.Queries; using Foundatio.Parsers.ElasticQueries; using Foundatio.Repositories.Elasticsearch.Configuration; using Foundatio.Repositories.Elasticsearch.Extensions; -using Nest; namespace Exceptionless.Core.Repositories.Configuration; @@ -21,50 +23,51 @@ public StackIndex(ExceptionlessElasticConfiguration configuration) : base(config _configuration = configuration; } - public override CreateIndexDescriptor ConfigureIndex(CreateIndexDescriptor idx) + public override void ConfigureIndex(CreateIndexRequestDescriptor idx) { - return base.ConfigureIndex(idx.Settings(s => s - .Analysis(BuildAnalysis) + base.ConfigureIndex(idx); + idx.Settings(s => s + .Analysis(a => BuildAnalysis(a)) .NumberOfShards(_configuration.Options.NumberOfShards) .NumberOfReplicas(_configuration.Options.NumberOfReplicas) - .Priority(5))); + .Priority(5)); } - public override TypeMappingDescriptor ConfigureIndexMapping(TypeMappingDescriptor map) + public override void ConfigureIndexMapping(TypeMappingDescriptor map) { - return map - .Dynamic(false) + map + .Dynamic(DynamicMapping.False) .Properties(p => p .SetupDefaults() - .Keyword(f => f.Name(s => s.OrganizationId).IgnoreAbove(1024)) - .FieldAlias(a => a.Name(Alias.OrganizationId).Path(f => f.OrganizationId)) - .Keyword(f => f.Name(s => s.ProjectId).IgnoreAbove(1024)) - .FieldAlias(a => a.Name(Alias.ProjectId).Path(f => f.ProjectId)) - .Keyword(f => f.Name(s => s.Status)) - .Date(f => f.Name(s => s.SnoozeUntilUtc)) - .Keyword(f => f.Name(s => s.SignatureHash).IgnoreAbove(1024)) - .FieldAlias(a => a.Name(Alias.SignatureHash).Path(f => f.SignatureHash)) - .Keyword(f => f.Name(s => s.DuplicateSignature)) - .Keyword(f => f.Name(e => e.Type).IgnoreAbove(1024)) - .Date(f => f.Name(s => s.FirstOccurrence)) - .FieldAlias(a => a.Name(Alias.FirstOccurrence).Path(f => f.FirstOccurrence)) - .Date(f => f.Name(s => s.LastOccurrence)) - .FieldAlias(a => a.Name(Alias.LastOccurrence).Path(f => f.LastOccurrence)) - .Text(f => f.Name(s => s.Title)) - .Text(f => f.Name(s => s.Description)) - .Keyword(f => f.Name(s => s.Tags).IgnoreAbove(1024)) - .FieldAlias(a => a.Name(Alias.Tags).Path(f => f.Tags)) - .Keyword(f => f.Name(s => s.References).IgnoreAbove(1024)) - .FieldAlias(a => a.Name(Alias.References).Path(f => f.References)) - .Date(f => f.Name(s => s.DateFixed)) - .FieldAlias(a => a.Name(Alias.DateFixed).Path(f => f.DateFixed)) - .Boolean(f => f.Name(Alias.IsFixed)) - .Keyword(f => f.Name(s => s.FixedInVersion).IgnoreAbove(1024)) - .FieldAlias(a => a.Name(Alias.FixedInVersion).Path(f => f.FixedInVersion)) - .Boolean(f => f.Name(s => s.OccurrencesAreCritical)) - .FieldAlias(a => a.Name(Alias.OccurrencesAreCritical).Path(f => f.OccurrencesAreCritical)) - .Scalar(f => f.TotalOccurrences) - .FieldAlias(a => a.Name(Alias.TotalOccurrences).Path(f => f.TotalOccurrences)) + .Keyword(e => e.OrganizationId, k => k.IgnoreAbove(1024)) + .FieldAlias(Alias.OrganizationId, a => a.Path(f => f.OrganizationId)) + .Keyword(e => e.ProjectId, k => k.IgnoreAbove(1024)) + .FieldAlias(Alias.ProjectId, a => a.Path(f => f.ProjectId)) + .Keyword(e => e.Status) + .Date(e => e.SnoozeUntilUtc) + .Keyword(e => e.SignatureHash, k => k.IgnoreAbove(1024)) + .FieldAlias(Alias.SignatureHash, a => a.Path(f => f.SignatureHash)) + .Keyword(e => e.DuplicateSignature) + .Keyword(e => e.Type, k => k.IgnoreAbove(1024)) + .Date(e => e.FirstOccurrence) + .FieldAlias(Alias.FirstOccurrence, a => a.Path(f => f.FirstOccurrence)) + .Date(e => e.LastOccurrence) + .FieldAlias(Alias.LastOccurrence, a => a.Path(f => f.LastOccurrence)) + .Text(e => e.Title) + .Text(e => e.Description) + .Keyword(e => e.Tags, k => k.IgnoreAbove(1024)) + .FieldAlias(Alias.Tags, a => a.Path(f => f.Tags)) + .Keyword(e => e.References, k => k.IgnoreAbove(1024)) + .FieldAlias(Alias.References, a => a.Path(f => f.References)) + .Date(e => e.DateFixed) + .FieldAlias(Alias.DateFixed, a => a.Path(f => f.DateFixed)) + .Boolean(Alias.IsFixed) + .Keyword(e => e.FixedInVersion, k => k.IgnoreAbove(1024)) + .FieldAlias(Alias.FixedInVersion, a => a.Path(f => f.FixedInVersion)) + .Boolean(e => e.OccurrencesAreCritical) + .FieldAlias(Alias.OccurrencesAreCritical, a => a.Path(f => f.OccurrencesAreCritical)) + .IntegerNumber(e => e.TotalOccurrences) + .FieldAlias(Alias.TotalOccurrences, a => a.Path(f => f.TotalOccurrences)) ); } @@ -79,12 +82,12 @@ protected override void ConfigureQueryParser(ElasticQueryParserConfiguration con }); } - private AnalysisDescriptor BuildAnalysis(AnalysisDescriptor ad) + private void BuildAnalysis(IndexSettingsAnalysisDescriptor ad) { - return ad.Analyzers(a => a + ad.Analyzers(a => a .Pattern(COMMA_WHITESPACE_ANALYZER, p => p.Pattern(@"[,\s]+")) - .Custom(STANDARDPLUS_ANALYZER, c => c.Filters("lowercase", "stop", "unique").Tokenizer(COMMA_WHITESPACE_TOKENIZER)) - .Custom(WHITESPACE_LOWERCASE_ANALYZER, c => c.Filters("lowercase").Tokenizer("whitespace"))) + .Custom(STANDARDPLUS_ANALYZER, c => c.Filter("lowercase", "stop", "unique").Tokenizer(COMMA_WHITESPACE_TOKENIZER)) + .Custom(WHITESPACE_LOWERCASE_ANALYZER, c => c.Filter("lowercase").Tokenizer("whitespace"))) .Tokenizers(t => t .Pattern(COMMA_WHITESPACE_TOKENIZER, p => p.Pattern(@"[,\s]+"))); } diff --git a/src/Exceptionless.Core/Repositories/Configuration/Indexes/TokenIndex.cs b/src/Exceptionless.Core/Repositories/Configuration/Indexes/TokenIndex.cs index c5e86ef488..55ea32030a 100644 --- a/src/Exceptionless.Core/Repositories/Configuration/Indexes/TokenIndex.cs +++ b/src/Exceptionless.Core/Repositories/Configuration/Indexes/TokenIndex.cs @@ -1,6 +1,8 @@ -using Foundatio.Repositories.Elasticsearch.Configuration; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.IndexManagement; +using Elastic.Clients.Elasticsearch.Mapping; +using Foundatio.Repositories.Elasticsearch.Configuration; using Foundatio.Repositories.Elasticsearch.Extensions; -using Nest; namespace Exceptionless.Core.Repositories.Configuration; @@ -14,31 +16,32 @@ public TokenIndex(ExceptionlessElasticConfiguration configuration) : base(config _configuration = configuration; } - public override TypeMappingDescriptor ConfigureIndexMapping(TypeMappingDescriptor map) + public override void ConfigureIndexMapping(TypeMappingDescriptor map) { - return map - .Dynamic(false) + map + .Dynamic(DynamicMapping.False) .Properties(p => p .SetupDefaults() - .Date(f => f.Name(e => e.ExpiresUtc)) - .Keyword(f => f.Name(e => e.OrganizationId)) - .Keyword(f => f.Name(e => e.ProjectId)) - .Keyword(f => f.Name(e => e.DefaultProjectId)) - .Keyword(f => f.Name(e => e.UserId)) - .Keyword(f => f.Name(u => u.CreatedBy)) - .Keyword(f => f.Name(e => e.Refresh)) - .Keyword(f => f.Name(e => e.Scopes)) - .Boolean(f => f.Name(e => e.IsDisabled)) - .Boolean(f => f.Name(e => e.IsSuspended)) - .Number(f => f.Name(e => e.Type).Type(NumberType.Byte))); + .Date(e => e.ExpiresUtc) + .Keyword(e => e.OrganizationId) + .Keyword(e => e.ProjectId) + .Keyword(e => e.DefaultProjectId) + .Keyword(e => e.UserId) + .Keyword(e => e.CreatedBy) + .Keyword(e => e.Refresh) + .Keyword(e => e.Scopes) + .Boolean(e => e.IsDisabled) + .Boolean(e => e.IsSuspended) + .IntegerNumber(e => e.Type)); } - public override CreateIndexDescriptor ConfigureIndex(CreateIndexDescriptor idx) + public override void ConfigureIndex(CreateIndexRequestDescriptor idx) { - return base.ConfigureIndex(idx.Settings(s => s - .Analysis(d => d.Analyzers(b => b.Custom(KEYWORD_LOWERCASE_ANALYZER, c => c.Filters("lowercase").Tokenizer("keyword")))) + base.ConfigureIndex(idx); + idx.Settings(s => s + .Analysis(d => d.Analyzers(b => b.Custom(KEYWORD_LOWERCASE_ANALYZER, c => c.Filter("lowercase").Tokenizer("keyword")))) .NumberOfShards(_configuration.Options.NumberOfShards) .NumberOfReplicas(_configuration.Options.NumberOfReplicas) - .Priority(10))); + .Priority(10)); } } diff --git a/src/Exceptionless.Core/Repositories/Configuration/Indexes/UserIndex.cs b/src/Exceptionless.Core/Repositories/Configuration/Indexes/UserIndex.cs index 0bfb36a32c..be39dfaeda 100644 --- a/src/Exceptionless.Core/Repositories/Configuration/Indexes/UserIndex.cs +++ b/src/Exceptionless.Core/Repositories/Configuration/Indexes/UserIndex.cs @@ -1,7 +1,10 @@ +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.IndexManagement; +using Elastic.Clients.Elasticsearch.Mapping; using Exceptionless.Core.Models; +using Foundatio.Parsers.ElasticQueries.Extensions; using Foundatio.Repositories.Elasticsearch.Configuration; using Foundatio.Repositories.Elasticsearch.Extensions; -using Nest; namespace Exceptionless.Core.Repositories.Configuration; @@ -15,34 +18,35 @@ public UserIndex(ExceptionlessElasticConfiguration configuration) : base(configu _configuration = configuration; } - public override TypeMappingDescriptor ConfigureIndexMapping(TypeMappingDescriptor map) + public override void ConfigureIndexMapping(TypeMappingDescriptor map) { - return map - .Dynamic(false) + map + .Dynamic(DynamicMapping.False) .Properties(p => p .SetupDefaults() - .Keyword(f => f.Name(e => e.OrganizationIds)) - .Text(f => f.Name(u => u.FullName).AddKeywordField()) - .Text(f => f.Name(u => u.EmailAddress).Analyzer(KEYWORD_LOWERCASE_ANALYZER).AddKeywordField()) - .Boolean(f => f.Name(u => u.IsEmailAddressVerified)) - .Keyword(f => f.Name(u => u.VerifyEmailAddressToken)) - .Date(f => f.Name(u => u.VerifyEmailAddressTokenExpiration)) - .Keyword(f => f.Name(u => u.PasswordResetToken)) - .Date(f => f.Name(u => u.PasswordResetTokenExpiration)) - .Keyword(f => f.Name(u => u.Roles)) - .Object(f => f.Name(o => o.OAuthAccounts.First()).Properties(mp => mp - .Keyword(fu => fu.Name(m => m.Provider)) - .Keyword(fu => fu.Name(m => m.ProviderUserId)) - .Keyword(fu => fu.Name(m => m.Username)))) + .Keyword(e => e.OrganizationIds) + .Text(e => e.FullName, t => t.AddKeywordField()) + .Text(e => e.EmailAddress, t => t.Analyzer(KEYWORD_LOWERCASE_ANALYZER).AddKeywordField()) + .Boolean(e => e.IsEmailAddressVerified) + .Keyword(e => e.VerifyEmailAddressToken) + .Date(e => e.VerifyEmailAddressTokenExpiration) + .Keyword(e => e.PasswordResetToken) + .Date(e => e.PasswordResetTokenExpiration) + .Keyword(e => e.Roles) + .Object(e => e.OAuthAccounts, o => o.Properties(mp => mp + .Keyword("provider") + .Keyword("provider_user_id") + .Keyword("username"))) ); } - public override CreateIndexDescriptor ConfigureIndex(CreateIndexDescriptor idx) + public override void ConfigureIndex(CreateIndexRequestDescriptor idx) { - return base.ConfigureIndex(idx.Settings(s => s - .Analysis(d => d.Analyzers(b => b.Custom(KEYWORD_LOWERCASE_ANALYZER, c => c.Filters("lowercase").Tokenizer("keyword")))) + base.ConfigureIndex(idx); + idx.Settings(s => s + .Analysis(d => d.Analyzers(b => b.Custom(KEYWORD_LOWERCASE_ANALYZER, c => c.Filter("lowercase").Tokenizer("keyword")))) .NumberOfShards(_configuration.Options.NumberOfShards) .NumberOfReplicas(_configuration.Options.NumberOfReplicas) - .Priority(5))); + .Priority(5)); } } diff --git a/src/Exceptionless.Core/Repositories/Configuration/Indexes/WebHookIndex.cs b/src/Exceptionless.Core/Repositories/Configuration/Indexes/WebHookIndex.cs index bd6f85da2e..29cf0d4d08 100644 --- a/src/Exceptionless.Core/Repositories/Configuration/Indexes/WebHookIndex.cs +++ b/src/Exceptionless.Core/Repositories/Configuration/Indexes/WebHookIndex.cs @@ -1,7 +1,9 @@ +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.IndexManagement; +using Elastic.Clients.Elasticsearch.Mapping; using Exceptionless.Core.Models; using Foundatio.Repositories.Elasticsearch.Configuration; using Foundatio.Repositories.Elasticsearch.Extensions; -using Nest; namespace Exceptionless.Core.Repositories.Configuration; @@ -14,25 +16,26 @@ public WebHookIndex(ExceptionlessElasticConfiguration configuration) : base(conf _configuration = configuration; } - public override TypeMappingDescriptor ConfigureIndexMapping(TypeMappingDescriptor map) + public override void ConfigureIndexMapping(TypeMappingDescriptor map) { - return map - .Dynamic(false) + map + .Dynamic(DynamicMapping.False) .Properties(p => p .SetupDefaults() - .Keyword(f => f.Name(e => e.OrganizationId)) - .Keyword(f => f.Name(e => e.ProjectId)) - .Keyword(f => f.Name(e => e.Url)) - .Keyword(f => f.Name(e => e.EventTypes)) - .Boolean(f => f.Name(e => e.IsEnabled)) + .Keyword(e => e.OrganizationId) + .Keyword(e => e.ProjectId) + .Keyword(e => e.Url) + .Keyword(e => e.EventTypes) + .Boolean(e => e.IsEnabled) ); } - public override CreateIndexDescriptor ConfigureIndex(CreateIndexDescriptor idx) + public override void ConfigureIndex(CreateIndexRequestDescriptor idx) { - return base.ConfigureIndex(idx.Settings(s => s + base.ConfigureIndex(idx); + idx.Settings(s => s .NumberOfShards(_configuration.Options.NumberOfShards) .NumberOfReplicas(_configuration.Options.NumberOfReplicas) - .Priority(5))); + .Priority(5)); } } From 4559885318b81311e96ff079a839b24d4e09ab83 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Sun, 22 Mar 2026 11:12:20 -0500 Subject: [PATCH 5/7] Migrate queries and repositories to Elastic.Clients.Elasticsearch Replace QueryContainer with Query, use TermQuery/TermsQuery/BoolQuery/DateRangeQuery object initializers with Infer.Field expressions. --- .../Repositories/EventRepository.cs | 21 +++++------ .../Repositories/OrganizationRepository.cs | 35 ++++++++---------- .../Repositories/ProjectRepository.cs | 12 ++++--- .../Repositories/Queries/AppFilterQuery.cs | 36 ++++++++++--------- .../Queries/EventStackFilterQuery.cs | 2 +- .../Repositories/Queries/OrganizationQuery.cs | 7 ++-- .../Repositories/Queries/ProjectQuery.cs | 7 ++-- .../Repositories/Queries/StackQuery.cs | 11 +++--- .../Visitors/StackDateFixedQueryVisitor.cs | 18 ++++++++-- .../Repositories/StackRepository.cs | 8 ++--- .../Repositories/TokenRepository.cs | 17 +++------ .../Repositories/UserRepository.cs | 15 ++++---- .../Repositories/WebHookRepository.cs | 4 +-- 13 files changed, 98 insertions(+), 95 deletions(-) diff --git a/src/Exceptionless.Core/Repositories/EventRepository.cs b/src/Exceptionless.Core/Repositories/EventRepository.cs index c28518a735..a36311126d 100644 --- a/src/Exceptionless.Core/Repositories/EventRepository.cs +++ b/src/Exceptionless.Core/Repositories/EventRepository.cs @@ -1,11 +1,11 @@ using Exceptionless.Core.Models; using Exceptionless.Core.Repositories.Configuration; using Exceptionless.Core.Repositories.Queries; +using Elastic.Clients.Elasticsearch; using Exceptionless.DateTimeExtensions; using FluentValidation; using Foundatio.Repositories; using Foundatio.Repositories.Models; -using Nest; namespace Exceptionless.Core.Repositories; @@ -33,11 +33,13 @@ public EventRepository(ExceptionlessElasticConfiguration configuration, AppOptio public Task> GetOpenSessionsAsync(DateTime createdBeforeUtc, CommandOptionsDescriptor? options = null) { - var filter = Query.Term(e => e.Type, Event.KnownTypes.Session) && !Query.Exists(f => f.Field(e => e.Idx![Event.KnownDataKeys.SessionEnd + "-d"])); + var query = new RepositoryQuery() + .FilterExpression($"type:{Event.KnownTypes.Session} AND -_exists_:idx.{Event.KnownDataKeys.SessionEnd}-d"); + if (createdBeforeUtc.Ticks > 0) - filter &= Query.DateRange(r => r.Field(e => e.Date).LessThanOrEquals(createdBeforeUtc)); + query = query.DateRange(null, createdBeforeUtc, (PersistentEvent e) => e.Date); - return FindAsync(q => q.ElasticFilter(filter).SortDescending(e => e.Date), options); + return FindAsync(q => query.SortDescending(e => e.Date), options); } /// @@ -64,9 +66,9 @@ public Task RemoveAllAsync(string organizationId, string? clientIpAddress, if (utcStart.HasValue && utcEnd.HasValue) query = query.DateRange(utcStart, utcEnd, InferField(e => e.Date)).Index(utcStart, utcEnd); else if (utcEnd.HasValue) - query = query.ElasticFilter(Query.DateRange(r => r.Field(e => e.Date).LessThan(utcEnd))); + query = query.FilterExpression($"date:<{utcEnd.Value:O}"); else if (utcStart.HasValue) - query = query.ElasticFilter(Query.DateRange(r => r.Field(e => e.Date).GreaterThan(utcStart))); + query = query.FilterExpression($"date:>{utcStart.Value:O}"); if (!String.IsNullOrEmpty(clientIpAddress)) query = query.FieldEquals(EventIndex.Alias.IpAddress, clientIpAddress); @@ -76,8 +78,7 @@ public Task RemoveAllAsync(string organizationId, string? clientIpAddress, public Task> GetByReferenceIdAsync(string projectId, string referenceId) { - var filter = Query.Term(e => e.ReferenceId, referenceId); - return FindAsync(q => q.Project(projectId).ElasticFilter(filter).SortDescending(e => e.Date), o => o.PageLimit(10)); + return FindAsync(q => q.Project(projectId).FilterExpression($"reference_id:{referenceId}").SortDescending(e => e.Date), o => o.PageLimit(10)); } public async Task GetPreviousAndNextEventIdsAsync(PersistentEvent ev, AppFilter? systemFilter = null, DateTime? utcStart = null, DateTime? utcEnd = null) @@ -113,8 +114,8 @@ public async Task GetPreviousAndNextEventIdsAsync( .SortDescending(e => e.Date) .Include(e => e.Id, e => e.Date) .AppFilter(systemFilter) - .ElasticFilter(!Query.Ids(ids => ids.Values(ev.Id))) .FilterExpression(String.Concat(EventIndex.Alias.StackId, ":", ev.StackId)) + .FilterExpression($"-_id:{ev.Id}") .EnforceEventStackFilter(false), o => o.PageLimit(10)); if (results.Total == 0) @@ -153,8 +154,8 @@ public async Task GetPreviousAndNextEventIdsAsync( .SortAscending(e => e.Date) .Include(e => e.Id, e => e.Date) .AppFilter(systemFilter) - .ElasticFilter(!Query.Ids(ids => ids.Values(ev.Id))) .FilterExpression(String.Concat(EventIndex.Alias.StackId, ":", ev.StackId)) + .FilterExpression($"-_id:{ev.Id}") .EnforceEventStackFilter(false), o => o.PageLimit(10)); if (results.Total == 0) diff --git a/src/Exceptionless.Core/Repositories/OrganizationRepository.cs b/src/Exceptionless.Core/Repositories/OrganizationRepository.cs index a1e9cb4e3a..96d9c5f564 100644 --- a/src/Exceptionless.Core/Repositories/OrganizationRepository.cs +++ b/src/Exceptionless.Core/Repositories/OrganizationRepository.cs @@ -3,10 +3,10 @@ using Exceptionless.Core.Models; using Exceptionless.Core.Models.Billing; using Exceptionless.Core.Repositories.Configuration; +using Elastic.Clients.Elasticsearch; using FluentValidation; using Foundatio.Repositories; using Foundatio.Repositories.Models; -using Nest; namespace Exceptionless.Core.Repositories; @@ -33,8 +33,7 @@ private void OnDocumentsChanging(object sender, DocumentsChangeEventArgs.Term(f => f.Field(o => o.Invites.First().Token).Value(token)); - var hit = await FindOneAsync(q => q.ElasticFilter(filter)); + var hit = await FindOneAsync(q => q.FilterExpression($"invites.token:{token}")); return hit?.Document; } @@ -42,41 +41,37 @@ private void OnDocumentsChanging(object sender, DocumentsChangeEventArgs.Term(f => f.Field(o => o.StripeCustomerId).Value(customerId)); - var hit = await FindOneAsync(q => q.ElasticFilter(filter)); + var hit = await FindOneAsync(q => q.FilterExpression($"stripe_customer_id:{customerId}")); return hit?.Document; } public Task> GetByCriteriaAsync(string? criteria, CommandOptionsDescriptor options, OrganizationSortBy sortBy, bool? paid = null, bool? suspended = null) { - var filter = Query.MatchAll(); + var filterParts = new List(); + if (!String.IsNullOrWhiteSpace(criteria)) - filter &= (Query.Term(o => o.Id, criteria) || Query.Term(o => o.Name, criteria)); + filterParts.Add($"(id:{criteria} OR name:{criteria})"); if (paid.HasValue) { if (paid.Value) - filter &= !Query.Term(o => o.PlanId, _plans.FreePlan.Id); + filterParts.Add($"-plan_id:{_plans.FreePlan.Id}"); else - filter &= Query.Term(o => o.PlanId, _plans.FreePlan.Id); + filterParts.Add($"plan_id:{_plans.FreePlan.Id}"); } if (suspended.HasValue) { if (suspended.Value) - filter &= (!Query.Term(o => o.BillingStatus, BillingStatus.Active) && - !Query.Term(o => o.BillingStatus, BillingStatus.Trialing) && - !Query.Term(o => o.BillingStatus, BillingStatus.Canceled) - ) || Query.Term(o => o.IsSuspended, true); + filterParts.Add($"((-billing_status:{(int)BillingStatus.Active} AND -billing_status:{(int)BillingStatus.Trialing} AND -billing_status:{(int)BillingStatus.Canceled}) OR is_suspended:true)"); else - filter &= ( - Query.Term(o => o.BillingStatus, BillingStatus.Active) && - Query.Term(o => o.BillingStatus, BillingStatus.Trialing) && - Query.Term(o => o.BillingStatus, BillingStatus.Canceled) - ) || Query.Term(o => o.IsSuspended, false); + filterParts.Add($"((billing_status:{(int)BillingStatus.Active} AND billing_status:{(int)BillingStatus.Trialing} AND billing_status:{(int)BillingStatus.Canceled}) OR is_suspended:false)"); } - var query = new RepositoryQuery().ElasticFilter(filter); + var query = new RepositoryQuery(); + if (filterParts.Count > 0) + query = query.FilterExpression(String.Join(" AND ", filterParts)); + switch (sortBy) { case OrganizationSortBy.Newest: @@ -89,7 +84,7 @@ public Task> GetByCriteriaAsync(string? criteria, Comm // query.WithSortDescending((Organization o) => o.TotalEventCount); // break; default: - query.SortAscending(o => o.Name.Suffix("keyword")); + query.SortAscending((Field)"name.keyword"); break; } diff --git a/src/Exceptionless.Core/Repositories/ProjectRepository.cs b/src/Exceptionless.Core/Repositories/ProjectRepository.cs index 5d7ca4ea5e..4fe5030bcc 100644 --- a/src/Exceptionless.Core/Repositories/ProjectRepository.cs +++ b/src/Exceptionless.Core/Repositories/ProjectRepository.cs @@ -2,11 +2,11 @@ using Exceptionless.Core.Models; using Exceptionless.Core.Repositories.Configuration; using Exceptionless.Core.Repositories.Queries; +using Elastic.Clients.Elasticsearch; using FluentValidation; using Foundatio.Repositories; using Foundatio.Repositories.Models; using Foundatio.Repositories.Options; -using Nest; namespace Exceptionless.Core.Repositories; @@ -59,7 +59,7 @@ public Task> GetByOrganizationIdsAsync(ICollection if (organizationIds.Count == 0) return Task.FromResult(new FindResults()); - return FindAsync(q => q.Organization(organizationIds).SortAscending(p => p.Name.Suffix("keyword")), options); + return FindAsync(q => q.Organization(organizationIds).SortAscending((Field)"name.keyword"), options); } public Task> GetByFilterAsync(AppFilter systemFilter, string? userFilter, string? sort, CommandOptionsDescriptor? options = null) @@ -68,14 +68,16 @@ public Task> GetByFilterAsync(AppFilter systemFilter, strin .AppFilter(systemFilter) .FilterExpression(userFilter); - query = !String.IsNullOrEmpty(sort) ? query.SortExpression(sort) : query.SortAscending(p => p.Name.Suffix("keyword")); + query = !String.IsNullOrEmpty(sort) ? query.SortExpression(sort) : query.SortAscending((Field)"name.keyword"); return FindAsync(q => query, options); } public Task> GetByNextSummaryNotificationOffsetAsync(byte hourToSendNotificationsAfterUtcMidnight, int limit = 50) { - var filter = Query.Range(r => r.Field(o => o.NextSummaryEndOfDayTicks).LessThan(_timeProvider.GetUtcNow().UtcDateTime.Ticks - (TimeSpan.TicksPerHour * hourToSendNotificationsAfterUtcMidnight))); - return FindAsync(q => q.ElasticFilter(filter).SortAscending(p => p.OrganizationId), o => o.SearchAfterPaging().PageLimit(limit)); + long threshold = _timeProvider.GetUtcNow().UtcDateTime.Ticks - (TimeSpan.TicksPerHour * hourToSendNotificationsAfterUtcMidnight); + return FindAsync(q => q + .FilterExpression($"next_summary_end_of_day_ticks:<{threshold}") + .SortAscending(p => p.OrganizationId), o => o.SearchAfterPaging().PageLimit(limit)); } public async Task IncrementNextSummaryEndOfDayTicksAsync(IReadOnlyCollection projects) diff --git a/src/Exceptionless.Core/Repositories/Queries/AppFilterQuery.cs b/src/Exceptionless.Core/Repositories/Queries/AppFilterQuery.cs index c357601b9b..5cf6fa9222 100644 --- a/src/Exceptionless.Core/Repositories/Queries/AppFilterQuery.cs +++ b/src/Exceptionless.Core/Repositories/Queries/AppFilterQuery.cs @@ -8,7 +8,7 @@ using Foundatio.Repositories.Elasticsearch.Configuration; using Foundatio.Repositories.Elasticsearch.Queries.Builders; using Foundatio.Repositories.Options; -using Nest; +using Elastic.Clients.Elasticsearch.QueryDsl; namespace Exceptionless.Core.Repositories { @@ -109,7 +109,7 @@ public AppFilterQueryBuilder(AppOptions options, TimeProvider timeProvider) var allowedOrganizations = sfq.Organizations.Where(o => o.HasPremiumFeatures || (!o.HasPremiumFeatures && !sfq.UsesPremiumFeatures)).ToList(); if (allowedOrganizations.Count == 0) { - ctx.Filter &= Query.Term(_organizationIdFieldName, "none"); + ctx.Filter &= new TermQuery { Field = _organizationIdFieldName, Value = "none" }; return Task.CompletedTask; } @@ -124,21 +124,21 @@ public AppFilterQueryBuilder(AppOptions options, TimeProvider timeProvider) if (organization is not null) { if (shouldApplyRetentionFilter) - ctx.Filter &= (Query.Term(stackIdFieldName, sfq.Stack.Id) && GetRetentionFilter(field, organization, _options.MaximumRetentionDays, sfq.Stack.FirstOccurrence)); + ctx.Filter &= new TermQuery { Field = stackIdFieldName, Value = sfq.Stack.Id } & GetRetentionFilter(field, organization, _options.MaximumRetentionDays, sfq.Stack.FirstOccurrence); else { - ctx.Filter &= Query.Term(stackIdFieldName, sfq.Stack.Id); + ctx.Filter &= new TermQuery { Field = stackIdFieldName, Value = sfq.Stack.Id }; } } else { - ctx.Filter &= Query.Term(stackIdFieldName, "none"); + ctx.Filter &= new TermQuery { Field = stackIdFieldName, Value = "none" }; } return Task.CompletedTask; } - QueryContainer? container = null; + Query? container = null; if (sfq.Projects?.Count > 0) { var allowedProjects = sfq.Projects.ToDictionary(p => p, p => allowedOrganizations.SingleOrDefault(o => o.Id == p.OrganizationId)).Where(kvp => kvp.Value is not null).ToList(); @@ -146,40 +146,42 @@ public AppFilterQueryBuilder(AppOptions options, TimeProvider timeProvider) { foreach (var project in allowedProjects) { + Query termQuery = new TermQuery { Field = _projectIdFieldName, Value = project.Key.Id }; if (shouldApplyRetentionFilter) - container |= (Query.Term(_projectIdFieldName, project.Key.Id) && GetRetentionFilter(field, project.Value!, _options.MaximumRetentionDays, project.Key.CreatedUtc.SafeSubtract(TimeSpan.FromDays(3)))); - else - container |= Query.Term(_projectIdFieldName, project.Key.Id); + termQuery &= GetRetentionFilter(field, project.Value!, _options.MaximumRetentionDays, project.Key.CreatedUtc.SafeSubtract(TimeSpan.FromDays(3))); + container = container is not null ? container | termQuery : termQuery; } - ctx.Filter &= container; + if (container is not null) + ctx.Filter &= container; return Task.CompletedTask; } - ctx.Filter &= (Query.Term(_projectIdFieldName, "none")); + ctx.Filter &= new TermQuery { Field = _projectIdFieldName, Value = "none" }; return Task.CompletedTask; } foreach (var organization in allowedOrganizations) { + Query termQuery = new TermQuery { Field = _organizationIdFieldName, Value = organization.Id }; if (shouldApplyRetentionFilter) - container |= (Query.Term(_organizationIdFieldName, organization.Id) && GetRetentionFilter(field, organization, _options.MaximumRetentionDays)); - else - container |= Query.Term(_organizationIdFieldName, organization.Id); + termQuery &= GetRetentionFilter(field, organization, _options.MaximumRetentionDays); + container = container is not null ? container | termQuery : termQuery; } - ctx.Filter &= container; + if (container is not null) + ctx.Filter &= container; return Task.CompletedTask; } - private QueryContainer GetRetentionFilter(string? field, Organization organization, int maximumRetentionDays, DateTime? oldestPossibleEventAge = null) where T : class, new() + private Query GetRetentionFilter(string? field, Organization organization, int maximumRetentionDays, DateTime? oldestPossibleEventAge = null) { if (field is null) throw new ArgumentNullException(nameof(field), "Retention field not specified for this index"); var retentionDate = organization.GetRetentionUtcCutoff(maximumRetentionDays, oldestPossibleEventAge, _timeProvider); double retentionDays = Math.Max(Math.Round(Math.Abs(_timeProvider.GetUtcNow().UtcDateTime.Subtract(retentionDate).TotalDays), MidpointRounding.AwayFromZero), 1); - return Query.DateRange(r => r.Field(field).GreaterThanOrEquals($"now/d-{(int)retentionDays}d").LessThanOrEquals("now/d+1d")); + return new DateRangeQuery { Field = field, Gte = $"now/d-{(int)retentionDays}d", Lte = "now/d+1d" }; } private static bool ShouldApplyRetentionFilter(IIndex index, QueryBuilderContext ctx) where T : class, new() diff --git a/src/Exceptionless.Core/Repositories/Queries/EventStackFilterQuery.cs b/src/Exceptionless.Core/Repositories/Queries/EventStackFilterQuery.cs index 72637e5370..d3fcf596d1 100644 --- a/src/Exceptionless.Core/Repositories/Queries/EventStackFilterQuery.cs +++ b/src/Exceptionless.Core/Repositories/Queries/EventStackFilterQuery.cs @@ -11,7 +11,7 @@ using Foundatio.Repositories.Models; using Foundatio.Repositories.Options; using Microsoft.Extensions.Logging; -using Nest; +using Elastic.Clients.Elasticsearch; using DateRange = Foundatio.Repositories.DateRange; namespace Exceptionless.Core.Repositories diff --git a/src/Exceptionless.Core/Repositories/Queries/OrganizationQuery.cs b/src/Exceptionless.Core/Repositories/Queries/OrganizationQuery.cs index bda527bb58..93220edbc3 100644 --- a/src/Exceptionless.Core/Repositories/Queries/OrganizationQuery.cs +++ b/src/Exceptionless.Core/Repositories/Queries/OrganizationQuery.cs @@ -4,7 +4,8 @@ using Foundatio.Repositories; using Foundatio.Repositories.Elasticsearch.Queries.Builders; using Foundatio.Repositories.Options; -using Nest; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.QueryDsl; namespace Exceptionless.Core.Repositories { @@ -54,9 +55,9 @@ public class OrganizationQueryBuilder : IElasticQueryBuilder return Task.CompletedTask; if (organizationIds.Count == 1) - ctx.Filter &= Query.Term(_organizationIdFieldName, organizationIds.Single()); + ctx.Filter &= new TermQuery { Field = _organizationIdFieldName, Value = organizationIds.Single() }; else - ctx.Filter &= Query.Terms(d => d.Field(_organizationIdFieldName).Terms(organizationIds)); + ctx.Filter &= new TermsQuery { Field = _organizationIdFieldName, Terms = new TermsQueryField(organizationIds.Select(id => (FieldValue)id).ToList()) }; return Task.CompletedTask; } diff --git a/src/Exceptionless.Core/Repositories/Queries/ProjectQuery.cs b/src/Exceptionless.Core/Repositories/Queries/ProjectQuery.cs index 9be2eb330d..301a5776e7 100644 --- a/src/Exceptionless.Core/Repositories/Queries/ProjectQuery.cs +++ b/src/Exceptionless.Core/Repositories/Queries/ProjectQuery.cs @@ -4,7 +4,8 @@ using Foundatio.Repositories; using Foundatio.Repositories.Elasticsearch.Queries.Builders; using Foundatio.Repositories.Options; -using Nest; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.QueryDsl; namespace Exceptionless.Core.Repositories { @@ -48,9 +49,9 @@ public class ProjectQueryBuilder : IElasticQueryBuilder return Task.CompletedTask; if (projectIds.Count == 1) - ctx.Filter &= Query.Term(_projectIdFieldName, projectIds.Single()); + ctx.Filter &= new TermQuery { Field = _projectIdFieldName, Value = projectIds.Single() }; else - ctx.Filter &= Query.Terms(d => d.Field(_projectIdFieldName).Terms(projectIds)); + ctx.Filter &= new TermsQuery { Field = _projectIdFieldName, Terms = new TermsQueryField(projectIds.Select(id => (FieldValue)id).ToList()) }; return Task.CompletedTask; } diff --git a/src/Exceptionless.Core/Repositories/Queries/StackQuery.cs b/src/Exceptionless.Core/Repositories/Queries/StackQuery.cs index db49aeb7cf..b13ede427e 100644 --- a/src/Exceptionless.Core/Repositories/Queries/StackQuery.cs +++ b/src/Exceptionless.Core/Repositories/Queries/StackQuery.cs @@ -4,7 +4,8 @@ using Foundatio.Repositories; using Foundatio.Repositories.Elasticsearch.Queries.Builders; using Foundatio.Repositories.Options; -using Nest; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.QueryDsl; namespace Exceptionless.Core.Repositories { @@ -63,14 +64,14 @@ public class StackQueryBuilder : IElasticQueryBuilder var excludedStackIds = ctx.Source.GetExcludedStacks(); if (stackIds.Count == 1) - ctx.Filter &= Query.Term(_stackIdFieldName, stackIds.Single()); + ctx.Filter &= new TermQuery { Field = _stackIdFieldName, Value = stackIds.Single() }; else if (stackIds.Count > 1) - ctx.Filter &= Query.Terms(d => d.Field(_stackIdFieldName).Terms(stackIds)); + ctx.Filter &= new TermsQuery { Field = _stackIdFieldName, Terms = new TermsQueryField(stackIds.Select(id => (FieldValue)id).ToList()) }; if (excludedStackIds.Count == 1) - ctx.Filter &= Query.Bool(b => b.MustNot(Query.Term(_stackIdFieldName, excludedStackIds.Single()))); + ctx.Filter &= new BoolQuery { MustNot = new Query[] { new TermQuery { Field = _stackIdFieldName, Value = excludedStackIds.Single() } } }; else if (excludedStackIds.Count > 1) - ctx.Filter &= Query.Bool(b => b.MustNot(Query.Terms(d => d.Field(_stackIdFieldName).Terms(excludedStackIds)))); + ctx.Filter &= new BoolQuery { MustNot = new Query[] { new TermsQuery { Field = _stackIdFieldName, Terms = new TermsQueryField(excludedStackIds.Select(id => (FieldValue)id).ToList()) } } }; return Task.CompletedTask; } diff --git a/src/Exceptionless.Core/Repositories/Queries/Visitors/StackDateFixedQueryVisitor.cs b/src/Exceptionless.Core/Repositories/Queries/Visitors/StackDateFixedQueryVisitor.cs index 95909ba731..46274cab4d 100644 --- a/src/Exceptionless.Core/Repositories/Queries/Visitors/StackDateFixedQueryVisitor.cs +++ b/src/Exceptionless.Core/Repositories/Queries/Visitors/StackDateFixedQueryVisitor.cs @@ -1,7 +1,7 @@ using Foundatio.Parsers.ElasticQueries.Extensions; using Foundatio.Parsers.LuceneQueries.Nodes; using Foundatio.Parsers.LuceneQueries.Visitors; -using Nest; +using Elastic.Clients.Elasticsearch.QueryDsl; namespace Exceptionless.Core.Repositories.Queries; @@ -23,8 +23,20 @@ public override Task VisitAsync(TermNode node, IQueryVisitorContext if (!Boolean.TryParse(node.Term, out bool isFixed)) return Task.FromResult(node); - var query = new ExistsQuery { Field = _dateFixedFieldName }; - node.SetQuery(isFixed ? query : !query); + Query query; + if (isFixed) + { + query = new ExistsQuery { Field = _dateFixedFieldName }; + } + else + { + query = new BoolQuery + { + MustNot = new Query[] { new ExistsQuery { Field = _dateFixedFieldName } } + }; + } + + node.SetQuery(query); return Task.FromResult(node); } diff --git a/src/Exceptionless.Core/Repositories/StackRepository.cs b/src/Exceptionless.Core/Repositories/StackRepository.cs index f85dc8c9ba..dd0e208993 100644 --- a/src/Exceptionless.Core/Repositories/StackRepository.cs +++ b/src/Exceptionless.Core/Repositories/StackRepository.cs @@ -1,12 +1,12 @@ using Exceptionless.Core.Extensions; using Exceptionless.Core.Models; using Exceptionless.Core.Repositories.Configuration; +using Elastic.Clients.Elasticsearch; using FluentValidation; using Foundatio.Repositories; using Foundatio.Repositories.Exceptions; using Foundatio.Repositories.Models; using Foundatio.Repositories.Options; -using Nest; namespace Exceptionless.Core.Repositories; @@ -24,14 +24,14 @@ public StackRepository(ExceptionlessElasticConfiguration configuration, IValidat public Task> GetExpiredSnoozedStatuses(DateTime utcNow, CommandOptionsDescriptor? options = null) { - return FindAsync(q => q.ElasticFilter(Query.DateRange(d => d.Field(f => f.SnoozeUntilUtc).LessThanOrEquals(utcNow))), options); + return FindAsync(q => q.DateRange(null, utcNow, (Stack s) => s.SnoozeUntilUtc), options); } public Task> GetStacksForCleanupAsync(string organizationId, DateTime cutoff) { return FindAsync(q => q .Organization(organizationId) - .ElasticFilter(Query.DateRange(d => d.Field(f => f.LastOccurrence).LessThanOrEquals(cutoff))) + .DateRange(null, cutoff, (Stack s) => s.LastOccurrence) .FieldEquals(f => f.Status, StackStatus.Open) .FieldEmpty(f => f.References) .Include(f => f.Id, f => f.OrganizationId, f => f.ProjectId, f => f.SignatureHash) @@ -154,7 +154,7 @@ Instant parseDate(def dt) { public async Task GetStackBySignatureHashAsync(string projectId, string signatureHash) { string key = GetStackSignatureCacheKey(projectId, signatureHash); - var hit = await FindOneAsync(q => q.Project(projectId).ElasticFilter(Query.Term(s => s.SignatureHash, signatureHash)), o => o.Cache(key)); + var hit = await FindOneAsync(q => q.Project(projectId).FilterExpression($"signature_hash:{signatureHash}"), o => o.Cache(key)); return hit?.Document; } diff --git a/src/Exceptionless.Core/Repositories/TokenRepository.cs b/src/Exceptionless.Core/Repositories/TokenRepository.cs index e76ea8bf22..17f498de4d 100644 --- a/src/Exceptionless.Core/Repositories/TokenRepository.cs +++ b/src/Exceptionless.Core/Repositories/TokenRepository.cs @@ -4,7 +4,6 @@ using FluentValidation; using Foundatio.Repositories; using Foundatio.Repositories.Models; -using Nest; using Token = Exceptionless.Core.Models.Token; namespace Exceptionless.Core.Repositories; @@ -19,36 +18,30 @@ public TokenRepository(ExceptionlessElasticConfiguration configuration, IValidat public Task> GetByTypeAndUserIdAsync(TokenType type, string userId, CommandOptionsDescriptor? options = null) { - var filter = Query.Term(e => e.UserId, userId) && Query.Term(t => t.Type, type); - return FindAsync(q => q.ElasticFilter(filter).Sort(f => f.CreatedUtc), options); + return FindAsync(q => q.FilterExpression($"user_id:{userId} AND type:{(int)type}").Sort(f => f.CreatedUtc), options); } public Task> GetByTypeAndOrganizationIdAsync(TokenType type, string organizationId, CommandOptionsDescriptor? options = null) { return FindAsync(q => q .Organization(organizationId) - .ElasticFilter(Query.Term(t => t.Type, type)) + .FilterExpression($"type:{(int)type}") .Sort(f => f.CreatedUtc), options); } public Task> GetByTypeAndProjectIdAsync(TokenType type, string projectId, CommandOptionsDescriptor? options = null) { - var filter = ( - Query.Term(t => t.ProjectId, projectId) || Query.Term(t => t.DefaultProjectId, projectId) - ) && Query.Term(t => t.Type, type); - - return FindAsync(q => q.ElasticFilter(filter).Sort(f => f.CreatedUtc), options); + return FindAsync(q => q.FilterExpression($"(project_id:{projectId} OR default_project_id:{projectId}) AND type:{(int)type}").Sort(f => f.CreatedUtc), options); } public override Task> GetByProjectIdAsync(string projectId, CommandOptionsDescriptor? options = null) { - var filter = (Query.Term(t => t.ProjectId, projectId) || Query.Term(t => t.DefaultProjectId, projectId)); - return FindAsync(q => q.ElasticFilter(filter).Sort(f => f.CreatedUtc), options); + return FindAsync(q => q.FilterExpression($"project_id:{projectId} OR default_project_id:{projectId}").Sort(f => f.CreatedUtc), options); } public Task RemoveAllByUserIdAsync(string userId, CommandOptionsDescriptor? options = null) { - return RemoveAllAsync(q => q.ElasticFilter(Query.Term(t => t.UserId, userId)), options); + return RemoveAllAsync(q => q.FilterExpression($"user_id:{userId}"), options); } protected override Task PublishChangeTypeMessageAsync(ChangeType changeType, Token? document, IDictionary? data = null, TimeSpan? delay = null) diff --git a/src/Exceptionless.Core/Repositories/UserRepository.cs b/src/Exceptionless.Core/Repositories/UserRepository.cs index 7c9fbd122e..37409b507e 100644 --- a/src/Exceptionless.Core/Repositories/UserRepository.cs +++ b/src/Exceptionless.Core/Repositories/UserRepository.cs @@ -1,10 +1,10 @@ using Exceptionless.Core.Extensions; using Exceptionless.Core.Repositories.Configuration; using Exceptionless.Core.Validation; +using Elastic.Clients.Elasticsearch; using Foundatio.Repositories; using Foundatio.Repositories.Models; using Foundatio.Repositories.Options; -using Nest; using User = Exceptionless.Core.Models.User; namespace Exceptionless.Core.Repositories; @@ -33,7 +33,7 @@ protected override Task ValidateAndThrowAsync(User document) return null; emailAddress = emailAddress.Trim().ToLowerInvariant(); - var hit = await FindOneAsync(q => q.ElasticFilter(Query.Term(u => u.EmailAddress.Suffix("keyword"), emailAddress)), o => o.Cache(EmailCacheKey(emailAddress))); + var hit = await FindOneAsync(q => q.FilterExpression($"email_address.keyword:\"{emailAddress}\""), o => o.Cache(EmailCacheKey(emailAddress))); return hit?.Document; } @@ -42,7 +42,7 @@ protected override Task ValidateAndThrowAsync(User document) if (String.IsNullOrEmpty(token)) return null; - var hit = await FindOneAsync(q => q.ElasticFilter(Query.Term(u => u.PasswordResetToken, token))); + var hit = await FindOneAsync(q => q.FilterExpression($"password_reset_token:{token}")); return hit?.Document; } @@ -52,8 +52,7 @@ protected override Task ValidateAndThrowAsync(User document) return null; provider = provider.ToLowerInvariant(); - var filter = Query.Term(u => u.OAuthAccounts.First().ProviderUserId, providerUserId); - var results = (await FindAsync(q => q.ElasticFilter(filter))).Documents; + var results = (await FindAsync(q => q.FilterExpression($"oauth_accounts.provider_user_id:{providerUserId}"))).Documents; return results.FirstOrDefault(u => u.OAuthAccounts.Any(o => o.Provider == provider)); } @@ -62,8 +61,7 @@ protected override Task ValidateAndThrowAsync(User document) if (String.IsNullOrEmpty(token)) return null; - var filter = Query.Term(u => u.VerifyEmailAddressToken, token); - var hit = await FindOneAsync(q => q.ElasticFilter(filter)); + var hit = await FindOneAsync(q => q.FilterExpression($"verify_email_address_token:{token}")); return hit?.Document; } @@ -76,8 +74,7 @@ public Task> GetByOrganizationIdAsync(string organizationId, C if (commandOptions.ShouldUseCache()) throw new Exception("Caching of paged queries is not allowed"); - var filter = Query.Term(u => u.OrganizationIds, organizationId); - return FindAsync(q => q.ElasticFilter(filter).SortAscending(u => u.EmailAddress.Suffix("keyword")), o => commandOptions); + return FindAsync(q => q.FilterExpression($"organization_ids:{organizationId}").SortAscending((Field)"email_address.keyword"), o => commandOptions); } protected override async Task AddDocumentsToCacheAsync(ICollection> findHits, ICommandOptions options, bool isDirtyRead) diff --git a/src/Exceptionless.Core/Repositories/WebHookRepository.cs b/src/Exceptionless.Core/Repositories/WebHookRepository.cs index 5d6cb74400..bc62b28ec0 100644 --- a/src/Exceptionless.Core/Repositories/WebHookRepository.cs +++ b/src/Exceptionless.Core/Repositories/WebHookRepository.cs @@ -4,7 +4,6 @@ using FluentValidation; using Foundatio.Repositories; using Foundatio.Repositories.Models; -using Nest; namespace Exceptionless.Core.Repositories; @@ -26,8 +25,7 @@ public Task> GetByOrganizationIdOrProjectIdAsync(string org ArgumentException.ThrowIfNullOrEmpty(organizationId); ArgumentException.ThrowIfNullOrEmpty(projectId); - var filter = (Query.Term(e => e.OrganizationId, organizationId) && !Query.Exists(e => e.Field(f => f.ProjectId))) || Query.Term(e => e.ProjectId, projectId); - return FindAsync(q => q.ElasticFilter(filter).Sort(f => f.CreatedUtc), o => o.Cache(PagedCacheKey(organizationId, projectId))); + return FindAsync(q => q.FilterExpression($"(organization_id:{organizationId} AND -_exists_:project_id) OR project_id:{projectId}").Sort(f => f.CreatedUtc), o => o.Cache(PagedCacheKey(organizationId, projectId))); } public override Task> GetByProjectIdAsync(string projectId, CommandOptionsDescriptor? options = null) From 6be39329a64ed2184d4633eb2a317899b2094823 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Sun, 22 Mar 2026 11:12:33 -0500 Subject: [PATCH 6/7] Migrate jobs, migrations, and controllers to Elastic.Clients.Elasticsearch Update CleanupOrphanedDataJob, DataMigrationJob, all migration files, AdminController, EventController, ProjectController, and test files to use new ES 8 client APIs. --- .../Jobs/CleanupOrphanedDataJob.cs | 96 ++++++++++--------- .../Jobs/Elastic/DataMigrationJob.cs | 71 ++++++++------ .../Migrations/001_UpdateIndexMappings.cs | 46 +++++---- .../Migrations/002_SetStackStatus.cs | 23 ++--- .../Migrations/FixDuplicateStacks.cs | 37 +++---- .../Migrations/SetStackDuplicateSignature.cs | 31 +++--- .../Migrations/UpdateEventUsage.cs | 2 +- .../Controllers/AdminController.cs | 82 ++++++++-------- .../Controllers/EventController.cs | 8 +- .../Controllers/ProjectController.cs | 1 + .../IntegrationTestsBase.cs | 3 +- .../FixDuplicateStacksMigrationTests.cs | 15 +-- ...etStackDuplicateSignatureMigrationTests.cs | 5 +- 13 files changed, 219 insertions(+), 201 deletions(-) diff --git a/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs b/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs index 5e09702161..bed64ad82e 100644 --- a/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs +++ b/src/Exceptionless.Core/Jobs/CleanupOrphanedDataJob.cs @@ -11,7 +11,10 @@ using Foundatio.Resilience; using Microsoft.Extensions.Diagnostics.HealthChecks; using Microsoft.Extensions.Logging; -using Nest; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.Aggregations; +using Elastic.Clients.Elasticsearch.Core.ReindexRethrottle; +using Elastic.Clients.Elasticsearch.QueryDsl; using LogLevel = Microsoft.Extensions.Logging.LogLevel; namespace Exceptionless.Core.Jobs; @@ -20,7 +23,7 @@ namespace Exceptionless.Core.Jobs; public class CleanupOrphanedDataJob : JobWithLockBase, IHealthCheck { private readonly ExceptionlessElasticConfiguration _config; - private readonly IElasticClient _elasticClient; + private readonly ElasticsearchClient _elasticClient; private readonly IStackRepository _stackRepository; private readonly IEventRepository _eventRepository; private readonly ICacheClient _cacheClient; @@ -61,10 +64,10 @@ protected override async Task RunInternalAsync(JobContext context) public async Task DeleteOrphanedEventsByStackAsync(JobContext context) { // get approximate number of unique stack ids - var stackCardinality = await _elasticClient.SearchAsync(s => s.Aggregations(a => a - .Cardinality("cardinality_stack_id", c => c.Field(f => f.StackId).PrecisionThreshold(40000)))); + var stackCardinality = await _elasticClient.SearchAsync(s => s + .AddAggregation("cardinality_stack_id", a => a.Cardinality(c => c.Field(f => f.StackId).PrecisionThreshold(40000)))); - double? uniqueStackIdCount = stackCardinality.Aggregations.Cardinality("cardinality_stack_id")?.Value; + double? uniqueStackIdCount = stackCardinality.Aggregations?.GetCardinality("cardinality_stack_id")?.Value; if (!uniqueStackIdCount.HasValue || uniqueStackIdCount.Value <= 0) return; @@ -79,17 +82,18 @@ public async Task DeleteOrphanedEventsByStackAsync(JobContext context) { await RenewLockAsync(context); - var stackIdTerms = await _elasticClient.SearchAsync(s => s.Aggregations(a => a - .Terms("terms_stack_id", c => c.Field(f => f.StackId).Include(batchNumber, buckets).Size(batchSize * 2)))); + var stackIdTerms = await _elasticClient.SearchAsync(s => s + .AddAggregation("terms_stack_id", a => a.Terms(c => c.Field(f => f.StackId).Include(new TermsInclude(batchNumber, buckets)).Size(batchSize * 2)))); - string[] stackIds = stackIdTerms.Aggregations.Terms("terms_stack_id").Buckets.Select(b => b.Key).ToArray(); + string[] stackIds = stackIdTerms.Aggregations?.GetStringTerms("terms_stack_id")?.Buckets.Select(b => b.Key.ToString()!).ToArray() ?? []; if (stackIds.Length == 0) continue; totalStackIds += stackIds.Length; - var stacks = await _elasticClient.MultiGetAsync(r => r.SourceEnabled(false).GetMany(stackIds)); - string[] missingStackIds = stacks.Hits.Where(h => !h.Found).Select(h => h.Id).ToArray(); + var stacks = await _elasticClient.MultiGetAsync(r => r.Source(false).Ids(stackIds)); + string[] missingStackIds = stacks.Docs + .Select(d => d.Value1).Where(r => r is not null && !r.Found).Select(r => r!.Id).ToArray(); if (missingStackIds.Length == 0) @@ -100,7 +104,7 @@ public async Task DeleteOrphanedEventsByStackAsync(JobContext context) totalOrphanedEventCount += missingStackIds.Length; _logger.LogInformation("{BatchNumber}/{BatchCount}: Found {OrphanedEventCount} orphaned events from missing stacks {MissingStackIds} out of {StackIdCount}", batchNumber, buckets, missingStackIds.Length, missingStackIds, stackIds.Length); - await _elasticClient.DeleteByQueryAsync(r => r.Query(q => q.Terms(t => t.Field(f => f.StackId).Terms(missingStackIds)))); + await _elasticClient.DeleteByQueryAsync(r => r.Query(q => q.Terms(t => t.Field(f => f.StackId).Terms(new TermsQueryField(missingStackIds.Select(id => (FieldValue)id).ToList()))))); } _logger.LogInformation("Found {OrphanedEventCount} orphaned events from missing stacks out of {StackIdCount}", totalOrphanedEventCount, totalStackIds); @@ -109,10 +113,10 @@ public async Task DeleteOrphanedEventsByStackAsync(JobContext context) public async Task DeleteOrphanedEventsByProjectAsync(JobContext context) { // get approximate number of unique project ids - var projectCardinality = await _elasticClient.SearchAsync(s => s.Aggregations(a => a - .Cardinality("cardinality_project_id", c => c.Field(f => f.ProjectId).PrecisionThreshold(40000)))); + var projectCardinality = await _elasticClient.SearchAsync(s => s + .AddAggregation("cardinality_project_id", a => a.Cardinality(c => c.Field(f => f.ProjectId).PrecisionThreshold(40000)))); - double? uniqueProjectIdCount = projectCardinality.Aggregations.Cardinality("cardinality_project_id")?.Value; + double? uniqueProjectIdCount = projectCardinality.Aggregations?.GetCardinality("cardinality_project_id")?.Value; if (!uniqueProjectIdCount.HasValue || uniqueProjectIdCount.Value <= 0) return; @@ -127,17 +131,18 @@ public async Task DeleteOrphanedEventsByProjectAsync(JobContext context) { await RenewLockAsync(context); - var projectIdTerms = await _elasticClient.SearchAsync(s => s.Aggregations(a => a - .Terms("terms_project_id", c => c.Field(f => f.ProjectId).Include(batchNumber, buckets).Size(batchSize * 2)))); + var projectIdTerms = await _elasticClient.SearchAsync(s => s + .AddAggregation("terms_project_id", a => a.Terms(c => c.Field(f => f.ProjectId).Include(new TermsInclude(batchNumber, buckets)).Size(batchSize * 2)))); - string[] projectIds = projectIdTerms.Aggregations.Terms("terms_project_id").Buckets.Select(b => b.Key).ToArray(); + string[] projectIds = projectIdTerms.Aggregations?.GetStringTerms("terms_project_id")?.Buckets.Select(b => b.Key.ToString()!).ToArray() ?? []; if (projectIds.Length == 0) continue; totalProjectIds += projectIds.Length; - var projects = await _elasticClient.MultiGetAsync(r => r.SourceEnabled(false).GetMany(projectIds)); - string[] missingProjectIds = projects.Hits.Where(h => !h.Found).Select(h => h.Id).ToArray(); + var projects = await _elasticClient.MultiGetAsync(r => r.Source(false).Ids(projectIds)); + string[] missingProjectIds = projects.Docs + .Select(d => d.Value1).Where(r => r is not null && !r.Found).Select(r => r!.Id).ToArray(); if (missingProjectIds.Length == 0) { @@ -146,7 +151,7 @@ public async Task DeleteOrphanedEventsByProjectAsync(JobContext context) } _logger.LogInformation("{BatchNumber}/{BatchCount}: Found {OrphanedEventCount} orphaned events from missing projects {MissingProjectIds} out of {ProjectIdCount}", batchNumber, buckets, missingProjectIds.Length, missingProjectIds, projectIds.Length); - await _elasticClient.DeleteByQueryAsync(r => r.Query(q => q.Terms(t => t.Field(f => f.ProjectId).Terms(missingProjectIds)))); + await _elasticClient.DeleteByQueryAsync(r => r.Query(q => q.Terms(t => t.Field(f => f.ProjectId).Terms(new TermsQueryField(missingProjectIds.Select(id => (FieldValue)id).ToList()))))); } _logger.LogInformation("Found {OrphanedEventCount} orphaned events from missing projects out of {ProjectIdCount}", totalOrphanedEventCount, totalProjectIds); @@ -155,10 +160,10 @@ public async Task DeleteOrphanedEventsByProjectAsync(JobContext context) public async Task DeleteOrphanedEventsByOrganizationAsync(JobContext context) { // get approximate number of unique organization ids - var organizationCardinality = await _elasticClient.SearchAsync(s => s.Aggregations(a => a - .Cardinality("cardinality_organization_id", c => c.Field(f => f.OrganizationId).PrecisionThreshold(40000)))); + var organizationCardinality = await _elasticClient.SearchAsync(s => s + .AddAggregation("cardinality_organization_id", a => a.Cardinality(c => c.Field(f => f.OrganizationId).PrecisionThreshold(40000)))); - double? uniqueOrganizationIdCount = organizationCardinality.Aggregations.Cardinality("cardinality_organization_id")?.Value; + double? uniqueOrganizationIdCount = organizationCardinality.Aggregations?.GetCardinality("cardinality_organization_id")?.Value; if (!uniqueOrganizationIdCount.HasValue || uniqueOrganizationIdCount.Value <= 0) return; @@ -173,17 +178,18 @@ public async Task DeleteOrphanedEventsByOrganizationAsync(JobContext context) { await RenewLockAsync(context); - var organizationIdTerms = await _elasticClient.SearchAsync(s => s.Aggregations(a => a - .Terms("terms_organization_id", c => c.Field(f => f.OrganizationId).Include(batchNumber, buckets).Size(batchSize * 2)))); + var organizationIdTerms = await _elasticClient.SearchAsync(s => s + .AddAggregation("terms_organization_id", a => a.Terms(c => c.Field(f => f.OrganizationId).Include(new TermsInclude(batchNumber, buckets)).Size(batchSize * 2)))); - string[] organizationIds = organizationIdTerms.Aggregations.Terms("terms_organization_id").Buckets.Select(b => b.Key).ToArray(); + string[] organizationIds = organizationIdTerms.Aggregations?.GetStringTerms("terms_organization_id")?.Buckets.Select(b => b.Key.ToString()!).ToArray() ?? []; if (organizationIds.Length == 0) continue; totalOrganizationIds += organizationIds.Length; - var organizations = await _elasticClient.MultiGetAsync(r => r.SourceEnabled(false).GetMany(organizationIds)); - string[] missingOrganizationIds = organizations.Hits.Where(h => !h.Found).Select(h => h.Id).ToArray(); + var organizations = await _elasticClient.MultiGetAsync(r => r.Source(false).Ids(organizationIds)); + string[] missingOrganizationIds = organizations.Docs + .Select(d => d.Value1).Where(r => r is not null && !r.Found).Select(r => r!.Id).ToArray(); if (missingOrganizationIds.Length == 0) { @@ -192,7 +198,7 @@ public async Task DeleteOrphanedEventsByOrganizationAsync(JobContext context) } _logger.LogInformation("{BatchNumber}/{BatchCount}: Found {OrphanedEventCount} orphaned events from missing organizations {MissingOrganizationIds} out of {OrganizationIdCount}", batchNumber, buckets, missingOrganizationIds.Length, missingOrganizationIds, organizationIds.Length); - await _elasticClient.DeleteByQueryAsync(r => r.Query(q => q.Terms(t => t.Field(f => f.OrganizationId).Terms(missingOrganizationIds)))); + await _elasticClient.DeleteByQueryAsync(r => r.Query(q => q.Terms(t => t.Field(f => f.OrganizationId).Terms(new TermsQueryField(missingOrganizationIds.Select(id => (FieldValue)id).ToList()))))); } _logger.LogInformation("Found {OrphanedEventCount} orphaned events from missing organizations out of {OrganizationIdCount}", totalOrphanedEventCount, totalOrganizationIds); @@ -203,12 +209,12 @@ public async Task FixDuplicateStacks(JobContext context) _logger.LogInformation("Getting duplicate stacks"); var duplicateStackAgg = await _elasticClient.SearchAsync(q => q - .QueryOnQueryString("is_deleted:false") + .Query(q => q.QueryString(qs => qs.Query("is_deleted:false"))) .Size(0) - .Aggregations(a => a.Terms("stacks", t => t.Field(f => f.DuplicateSignature).MinimumDocumentCount(2).Size(10000)))); + .AddAggregation("stacks", a => a.Terms(t => t.Field(f => f.DuplicateSignature).MinDocCount(2).Size(10000)))); _logger.LogRequest(duplicateStackAgg, LogLevel.Trace); - var buckets = duplicateStackAgg.Aggregations.Terms("stacks")?.Buckets ?? new List>(); + var buckets = duplicateStackAgg.Aggregations?.GetStringTerms("stacks")?.Buckets.ToList() ?? []; int total = buckets.Count; int processed = 0; int error = 0; @@ -227,10 +233,10 @@ public async Task FixDuplicateStacks(JobContext context) string? signature = null; try { - string[] parts = duplicateSignature.Key.Split(':'); + string[] parts = duplicateSignature.Key.ToString().Split(':'); if (parts.Length != 2) { - _logger.LogError("Error parsing duplicate signature {DuplicateSignature}", duplicateSignature.Key); + _logger.LogError("Error parsing duplicate signature {DuplicateSignature}", duplicateSignature.Key.ToString()); continue; } projectId = parts[0]; @@ -283,10 +289,10 @@ public async Task FixDuplicateStacks(JobContext context) { var response = await _elasticClient.UpdateByQueryAsync(u => u .Query(q => q.Bool(b => b.Must(m => m - .Terms(t => t.Field(f => f.StackId).Terms(duplicateStacks.Select(s => s.Id))) + .Terms(t => t.Field(f => f.StackId).Terms(new TermsQueryField(duplicateStacks.Select(s => (FieldValue)s.Id).ToList()))) ))) - .Script(s => s.Source($"ctx._source.stack_id = '{targetStack.Id}'").Lang(ScriptLang.Painless)) - .Conflicts(Elasticsearch.Net.Conflicts.Proceed) + .Script(s => s.Source($"ctx._source.stack_id = '{targetStack.Id}'").Lang(ScriptLanguage.Painless)) + .Conflicts(Conflicts.Proceed) .WaitForCompletion(false)); _logger.LogRequest(response, LogLevel.Trace); @@ -297,22 +303,22 @@ public async Task FixDuplicateStacks(JobContext context) do { attempts++; - var taskStatus = await _elasticClient.Tasks.GetTaskAsync(taskId); - var status = taskStatus.Task.Status; + var taskStatus = await _elasticClient.Tasks.GetAsync(taskId!.FullyQualifiedId); + var status = taskStatus.Task.Status as ReindexStatus; if (taskStatus.Completed) { // TODO: need to check to see if the task failed or completed successfully. Throw if it failed. if (_timeProvider.GetUtcNow().UtcDateTime.Subtract(taskStartedTime) > TimeSpan.FromSeconds(30)) - _logger.LogInformation("Script operation task ({TaskId}) completed: Created: {Created} Updated: {Updated} Deleted: {Deleted} Conflicts: {Conflicts} Total: {Total}", taskId, status.Created, status.Updated, status.Deleted, status.VersionConflicts, status.Total); + _logger.LogInformation("Script operation task ({TaskId}) completed: Created: {Created} Updated: {Updated} Deleted: {Deleted} Conflicts: {Conflicts} Total: {Total}", taskId, status?.Created, status?.Updated, status?.Deleted, status?.VersionConflicts, status?.Total); - affectedRecords += status.Created + status.Updated + status.Deleted; + affectedRecords += (status?.Created ?? 0) + (status?.Updated ?? 0) + (status?.Deleted ?? 0); break; } if (_timeProvider.GetUtcNow().UtcDateTime.Subtract(taskStartedTime) > TimeSpan.FromSeconds(30)) { await RenewLockAsync(context); - _logger.LogInformation("Checking script operation task ({TaskId}) status: Created: {Created} Updated: {Updated} Deleted: {Deleted} Conflicts: {Conflicts} Total: {Total}", taskId, status.Created, status.Updated, status.Deleted, status.VersionConflicts, status.Total); + _logger.LogInformation("Checking script operation task ({TaskId}) status: Created: {Created} Updated: {Updated} Deleted: {Deleted} Conflicts: {Conflicts} Total: {Total}", taskId, status?.Created, status?.Updated, status?.Deleted, status?.VersionConflicts, status?.Total); } var delay = TimeSpan.FromMilliseconds(50); @@ -347,12 +353,12 @@ public async Task FixDuplicateStacks(JobContext context) await _elasticClient.Indices.RefreshAsync(_config.Stacks.VersionedName); duplicateStackAgg = await _elasticClient.SearchAsync(q => q - .QueryOnQueryString("is_deleted:false") + .Query(q => q.QueryString(qs => qs.Query("is_deleted:false"))) .Size(0) - .Aggregations(a => a.Terms("stacks", t => t.Field(f => f.DuplicateSignature).MinimumDocumentCount(2).Size(10000)))); + .AddAggregation("stacks", a => a.Terms(t => t.Field(f => f.DuplicateSignature).MinDocCount(2).Size(10000)))); _logger.LogRequest(duplicateStackAgg, LogLevel.Trace); - buckets = duplicateStackAgg.Aggregations.Terms("stacks").Buckets; + buckets = duplicateStackAgg.Aggregations?.GetStringTerms("stacks")?.Buckets.ToList() ?? []; total += buckets.Count; batch++; diff --git a/src/Exceptionless.Core/Jobs/Elastic/DataMigrationJob.cs b/src/Exceptionless.Core/Jobs/Elastic/DataMigrationJob.cs index e3938e2785..64700a8f4a 100644 --- a/src/Exceptionless.Core/Jobs/Elastic/DataMigrationJob.cs +++ b/src/Exceptionless.Core/Jobs/Elastic/DataMigrationJob.cs @@ -1,4 +1,3 @@ -using Elasticsearch.Net; using Exceptionless.Core.Repositories.Configuration; using Exceptionless.DateTimeExtensions; using Foundatio.Caching; @@ -8,7 +7,11 @@ using Foundatio.Repositories.Elasticsearch.Extensions; using Foundatio.Resilience; using Microsoft.Extensions.Logging; -using Nest; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.Core.Reindex; +using Elastic.Clients.Elasticsearch.Core.ReindexRethrottle; +using Elastic.Clients.Elasticsearch.QueryDsl; +using Elastic.Clients.Elasticsearch.Tasks; namespace Exceptionless.Core.Jobs.Elastic; @@ -98,29 +101,33 @@ protected override async Task RunInternalAsync(JobContext context) else if (dequeuedWorkItem.Attempts >= 2) batchSize = 250; - var response = await client.ReindexOnServerAsync(r => r + var response = await client.ReindexAsync(r => r .Source(s => s .Remote(ConfigureRemoteElasticSource) - .Index(dequeuedWorkItem.SourceIndex) + .Indices(dequeuedWorkItem.SourceIndex) .Size(batchSize) - .Query(q => + .Query(q => { - var container = q.Term("_type", dequeuedWorkItem.SourceIndexType); if (!String.IsNullOrEmpty(dequeuedWorkItem.DateField)) - container &= q.DateRange(d => d.Field(dequeuedWorkItem.DateField).GreaterThanOrEquals(cutOffDate)); - - return container; + { + q.Bool(b => b.Must( + m => m.Term(t => t.Field("_type").Value(dequeuedWorkItem.SourceIndexType)), + m => m.Range(r => r.Date(d => d.Field(dequeuedWorkItem.DateField!).Gte(cutOffDate))) + )); + } + else + { + q.Term(t => t.Field("_type").Value(dequeuedWorkItem.SourceIndexType)); + } })) - .Destination(d => d + .Dest(d => d .Index(dequeuedWorkItem.TargetIndex)) .Conflicts(Conflicts.Proceed) .WaitForCompletion(false) .Script(s => { if (!String.IsNullOrEmpty(dequeuedWorkItem.Script)) - return s.Source(dequeuedWorkItem.Script); - - return null; + s.Source(dequeuedWorkItem.Script); })); dequeuedWorkItem.Attempts += 1; @@ -135,26 +142,26 @@ protected override async Task RunInternalAsync(JobContext context) double highestProgress = 0; foreach (var workItem in workingTasks.ToArray()) { - var taskStatus = await client.Tasks.GetTaskAsync(workItem.TaskId, t => t.WaitForCompletion(false)); + var taskStatus = await client.Tasks.GetAsync(workItem.TaskId!.FullyQualifiedId, t => t.WaitForCompletion(false)); _logger.LogRequest(taskStatus); - var status = taskStatus?.Task?.Status; + var status = taskStatus.Task?.Status as ReindexStatus; if (taskStatus?.Task is null || status is null) { - _logger.LogWarning(taskStatus?.OriginalException, "Error getting task status for {TargetIndex} {TaskId}: {Message}", workItem.TargetIndex, workItem.TaskId, taskStatus.GetErrorMessage()); - if (taskStatus?.ServerError?.Status == 429) + _logger.LogWarning(taskStatus?.ApiCallDetails?.OriginalException, "Error getting task status for {TargetIndex} {TaskId}: {Message}", workItem.TargetIndex, workItem.TaskId, taskStatus.GetErrorMessage()); + if (taskStatus?.ElasticsearchServerError?.Status == 429) await Task.Delay(TimeSpan.FromSeconds(1), _timeProvider); continue; } - var duration = TimeSpan.FromMilliseconds(taskStatus.Task.RunningTimeInNanoseconds * 0.000001); + var duration = taskStatus.Task.RunningTimeInNanos; double progress = status.Total > 0 ? (status.Created + status.Updated + status.Deleted + status.VersionConflicts * 1.0) / status.Total : 0; highestProgress = Math.Max(highestProgress, progress); - if (!taskStatus.IsValid) + if (!taskStatus.IsValidResponse) { - _logger.LogWarning(taskStatus.OriginalException, "Error getting task status for {TargetIndex} ({TaskId}): {Message}", workItem.TargetIndex, workItem.TaskId, taskStatus.GetErrorMessage()); + _logger.LogWarning(taskStatus.ApiCallDetails?.OriginalException, "Error getting task status for {TargetIndex} ({TaskId}): {Message}", workItem.TargetIndex, workItem.TaskId, taskStatus.GetErrorMessage()); workItem.ConsecutiveStatusErrors++; if (taskStatus.Completed || workItem.ConsecutiveStatusErrors > 5) { @@ -186,7 +193,7 @@ protected override async Task RunInternalAsync(JobContext context) workingTasks.Remove(workItem); workItem.LastTaskInfo = taskStatus.Task; completedTasks.Add(workItem); - var targetCount = await client.CountAsync(d => d.Index(workItem.TargetIndex)); + var targetCount = await client.CountAsync(d => d.Indices(workItem.TargetIndex)); _logger.LogInformation("COMPLETED - {TargetIndex} ({TargetCount}) in {Duration:hh\\:mm} C:{Created} U:{Updated} D:{Deleted} X:{Conflicts} T:{Total} A:{Attempts} ID:{TaskId}", workItem.TargetIndex, targetCount.Count, duration, status.Created, status.Updated, status.Deleted, status.VersionConflicts, status.Total, workItem.Attempts, workItem.TaskId); } @@ -201,21 +208,27 @@ protected override async Task RunInternalAsync(JobContext context) _logger.LogInformation("----- REINDEX COMPLETE - I:{Completed}/{Total} T:{Duration:d\\.hh\\:mm} F:{Failed} R:{Retries}", completedTasks.Count, totalTasks, _timeProvider.GetUtcNow().UtcDateTime.Subtract(started), failedTasks.Count, retriesCount); foreach (var task in completedTasks) { - var status = task.LastTaskInfo.Status; - var duration = TimeSpan.FromMilliseconds(task.LastTaskInfo.RunningTimeInNanoseconds * 0.000001); + var status = task.LastTaskInfo.Status as ReindexStatus; + if (status is null) + continue; + + var duration = task.LastTaskInfo.RunningTimeInNanos; double progress = status.Total > 0 ? (status.Created + status.Updated + status.Deleted + status.VersionConflicts * 1.0) / status.Total : 0; - var targetCount = await client.CountAsync(d => d.Index(task.TargetIndex)); + var targetCount = await client.CountAsync(d => d.Indices(task.TargetIndex)); _logger.LogInformation("SUCCESS - {TargetIndex} ({TargetCount}) in {Duration:hh\\:mm} P:{Progress:F0}% C:{Created} U:{Updated} D:{Deleted} X:{Conflicts} T:{Total} A:{Attempts} ID:{TaskId}", task.TargetIndex, targetCount.Count, duration, progress, status.Created, status.Updated, status.Deleted, status.VersionConflicts, status.Total, task.Attempts, task.TaskId); } foreach (var task in failedTasks) { - var status = task.LastTaskInfo.Status; - var duration = TimeSpan.FromMilliseconds(task.LastTaskInfo.RunningTimeInNanoseconds * 0.000001); + var status = task.LastTaskInfo.Status as ReindexStatus; + if (status is null) + continue; + + var duration = task.LastTaskInfo.RunningTimeInNanos; double progress = status.Total > 0 ? (status.Created + status.Updated + status.Deleted + status.VersionConflicts * 1.0) / status.Total : 0; - var targetCount = await client.CountAsync(d => d.Index(task.TargetIndex)); + var targetCount = await client.CountAsync(d => d.Indices(task.TargetIndex)); _logger.LogCritical("FAILED - {TargetIndex} ({TargetCount}) in {Duration:hh\\:mm} P:{Progress:F0}% C:{Created} U:{Updated} D:{Deleted} X:{Conflicts} T:{Total} A:{Attempts} ID:{TaskId}", task.TargetIndex, targetCount.Count, duration, progress, status.Created, status.Updated, status.Deleted, status.VersionConflicts, status.Total, task.Attempts, task.TaskId); } @@ -227,7 +240,7 @@ protected override async Task RunInternalAsync(JobContext context) return JobResult.Success; } - private IRemoteSource ConfigureRemoteElasticSource(RemoteSourceDescriptor rsd) + private void ConfigureRemoteElasticSource(RemoteSourceDescriptor rsd) { var elasticOptions = _configuration.Options.ElasticsearchToMigrate; if (elasticOptions is null) @@ -236,7 +249,7 @@ private IRemoteSource ConfigureRemoteElasticSource(RemoteSourceDescriptor rsd) if (!String.IsNullOrEmpty(elasticOptions.UserName) && !String.IsNullOrEmpty(elasticOptions.Password)) rsd.Username(elasticOptions.UserName).Password(elasticOptions.Password); - return rsd.Host(new Uri(elasticOptions.ServerUrl)); + rsd.Host(elasticOptions.ServerUrl); } } diff --git a/src/Exceptionless.Core/Migrations/001_UpdateIndexMappings.cs b/src/Exceptionless.Core/Migrations/001_UpdateIndexMappings.cs index c14657bb2d..a5ba073682 100644 --- a/src/Exceptionless.Core/Migrations/001_UpdateIndexMappings.cs +++ b/src/Exceptionless.Core/Migrations/001_UpdateIndexMappings.cs @@ -3,13 +3,14 @@ using Foundatio.Repositories.Elasticsearch.Extensions; using Foundatio.Repositories.Migrations; using Microsoft.Extensions.Logging; -using Nest; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.Mapping; namespace Exceptionless.Core.Migrations; public sealed class UpdateIndexMappings : MigrationBase { - private readonly IElasticClient _client; + private readonly ElasticsearchClient _client; private readonly ExceptionlessElasticConfiguration _config; public UpdateIndexMappings(ExceptionlessElasticConfiguration configuration, ILoggerFactory loggerFactory) : base(loggerFactory) @@ -26,56 +27,53 @@ public override async Task RunAsync(MigrationContext context) _logger.LogInformation("Start migration for adding index mappings..."); _logger.LogInformation("Updating Organization mappings..."); - var response = await _client.MapAsync(d => + var response = await _client.Indices.PutMappingAsync(d => { - d.Index(_config.Organizations.VersionedName); + d.Indices(_config.Organizations.VersionedName); d.Properties(p => p - .Date(f => f.Name(s => s.LastEventDateUtc)) - .Boolean(f => f.Name(s => s.IsDeleted)).FieldAlias(a => a.Path(p1 => p1.IsDeleted).Name("deleted"))); - - return d; + .Date(s => s.LastEventDateUtc) + .Boolean(s => s.IsDeleted) + .FieldAlias("deleted", new FieldAliasProperty { Path = "is_deleted" })); }); _logger.LogRequest(response); _logger.LogInformation("Setting Organization is_deleted=false..."); const string script = "ctx._source.is_deleted = false;"; await _config.Client.Indices.RefreshAsync(_config.Organizations.VersionedName); - var updateResponse = await _client.UpdateByQueryAsync(x => x.QueryOnQueryString("NOT _exists_:deleted").Script(s => s.Source(script).Lang(ScriptLang.Painless))); + var updateResponse = await _client.UpdateByQueryAsync(x => x.Query(q => q.QueryString(qs => qs.Query("NOT _exists_:deleted"))).Script(s => s.Source(script).Lang(ScriptLanguage.Painless))); _logger.LogRequest(updateResponse); _logger.LogInformation("Updating Project mappings..."); - response = await _client.MapAsync(d => + response = await _client.Indices.PutMappingAsync(d => { - d.Index(_config.Projects.VersionedName); + d.Indices(_config.Projects.VersionedName); d.Properties(p => p - .Date(f => f.Name(s => s.LastEventDateUtc)) - .Boolean(f => f.Name(s => s.IsDeleted)).FieldAlias(a => a.Path(p1 => p1.IsDeleted).Name("deleted"))); - - return d; + .Date(s => s.LastEventDateUtc) + .Boolean(s => s.IsDeleted) + .FieldAlias("deleted", new FieldAliasProperty { Path = "is_deleted" })); }); _logger.LogRequest(response); _logger.LogInformation("Setting Project is_deleted=false..."); await _config.Client.Indices.RefreshAsync(_config.Projects.VersionedName); - updateResponse = await _client.UpdateByQueryAsync(x => x.QueryOnQueryString("NOT _exists_:deleted").Script(s => s.Source(script).Lang(ScriptLang.Painless))); + updateResponse = await _client.UpdateByQueryAsync(x => x.Query(q => q.QueryString(qs => qs.Query("NOT _exists_:deleted"))).Script(s => s.Source(script).Lang(ScriptLanguage.Painless))); _logger.LogRequest(updateResponse); _logger.LogInformation("Updating Stack mappings..."); - response = await _client.MapAsync(d => + response = await _client.Indices.PutMappingAsync(d => { - d.Index(_config.Stacks.VersionedName); + d.Indices(_config.Stacks.VersionedName); d.Properties(p => p - .Keyword(f => f.Name(s => s.Status)) - .Date(f => f.Name(s => s.SnoozeUntilUtc)) - .Boolean(f => f.Name(s => s.IsDeleted)).FieldAlias(a => a.Path(p1 => p1.IsDeleted).Name("deleted"))); - - return d; + .Keyword(s => s.Status) + .Date(s => s.SnoozeUntilUtc) + .Boolean(s => s.IsDeleted) + .FieldAlias("deleted", new FieldAliasProperty { Path = "is_deleted" })); }); _logger.LogRequest(response); _logger.LogInformation("Setting Stack is_deleted=false..."); await _config.Client.Indices.RefreshAsync(_config.Stacks.VersionedName); - updateResponse = await _client.UpdateByQueryAsync(x => x.QueryOnQueryString("NOT _exists_:deleted").Script(s => s.Source(script).Lang(ScriptLang.Painless))); + updateResponse = await _client.UpdateByQueryAsync(x => x.Query(q => q.QueryString(qs => qs.Query("NOT _exists_:deleted"))).Script(s => s.Source(script).Lang(ScriptLanguage.Painless))); _logger.LogRequest(updateResponse); _logger.LogInformation("Finished adding mappings."); diff --git a/src/Exceptionless.Core/Migrations/002_SetStackStatus.cs b/src/Exceptionless.Core/Migrations/002_SetStackStatus.cs index cdf40112a6..4c2aa247cf 100644 --- a/src/Exceptionless.Core/Migrations/002_SetStackStatus.cs +++ b/src/Exceptionless.Core/Migrations/002_SetStackStatus.cs @@ -5,13 +5,14 @@ using Foundatio.Repositories.Elasticsearch.Extensions; using Foundatio.Repositories.Migrations; using Microsoft.Extensions.Logging; -using Nest; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.Core.ReindexRethrottle; namespace Exceptionless.Core.Migrations; public sealed class SetStackStatus : MigrationBase { - private readonly IElasticClient _client; + private readonly ElasticsearchClient _client; private readonly ExceptionlessElasticConfiguration _config; private readonly ICacheClient _cache; private readonly TimeProvider _timeProvider; @@ -37,9 +38,9 @@ public override async Task RunAsync(MigrationContext context) var sw = Stopwatch.StartNew(); const string script = "if (ctx._source.is_regressed == true) ctx._source.status = 'regressed'; else if (ctx._source.is_hidden == true) ctx._source.status = 'ignored'; else if (ctx._source.disable_notifications == true) ctx._source.status = 'ignored'; else if (ctx._source.is_fixed == true) ctx._source.status = 'fixed'; else if (ctx._source.containsKey('date_fixed')) ctx._source.status = 'fixed'; else ctx._source.status = 'open';"; var stackResponse = await _client.UpdateByQueryAsync(x => x - .QueryOnQueryString("NOT _exists_:status") - .Script(s => s.Source(script).Lang(ScriptLang.Painless)) - .Conflicts(Elasticsearch.Net.Conflicts.Proceed) + .Query(q => q.QueryString(qs => qs.Query("NOT _exists_:status"))) + .Script(s => s.Source(script).Lang(ScriptLanguage.Painless)) + .Conflicts(Conflicts.Proceed) .WaitForCompletion(false)); _logger.LogRequest(stackResponse, Microsoft.Extensions.Logging.LogLevel.Information); @@ -50,22 +51,22 @@ public override async Task RunAsync(MigrationContext context) do { attempts++; - var taskStatus = await _client.Tasks.GetTaskAsync(taskId); - var status = taskStatus.Task.Status; + var taskStatus = await _client.Tasks.GetAsync(taskId!.FullyQualifiedId); + var status = taskStatus.Task.Status as ReindexStatus; if (taskStatus.Completed) { // TODO: need to check to see if the task failed or completed successfully. Throw if it failed. - _logger.LogInformation("Script operation task ({TaskId}) completed: Created: {Created} Updated: {Updated} Deleted: {Deleted} Conflicts: {Conflicts} Total: {Total}", taskId, status.Created, status.Updated, status.Deleted, status.VersionConflicts, status.Total); - affectedRecords += status.Created + status.Updated + status.Deleted; + _logger.LogInformation("Script operation task ({TaskId}) completed: Created: {Created} Updated: {Updated} Deleted: {Deleted} Conflicts: {Conflicts} Total: {Total}", taskId, status?.Created, status?.Updated, status?.Deleted, status?.VersionConflicts, status?.Total); + affectedRecords += (status?.Created ?? 0) + (status?.Updated ?? 0) + (status?.Deleted ?? 0); break; } - _logger.LogInformation("Checking script operation task ({TaskId}) status: Created: {Created} Updated: {Updated} Deleted: {Deleted} Conflicts: {Conflicts} Total: {Total}", taskId, status.Created, status.Updated, status.Deleted, status.VersionConflicts, status.Total); + _logger.LogInformation("Checking script operation task ({TaskId}) status: Created: {Created} Updated: {Updated} Deleted: {Deleted} Conflicts: {Conflicts} Total: {Total}", taskId, status?.Created, status?.Updated, status?.Deleted, status?.VersionConflicts, status?.Total); var delay = TimeSpan.FromSeconds(attempts <= 5 ? 1 : 5); await Task.Delay(delay, _timeProvider); } while (true); - _logger.LogInformation("Finished adding stack status: Time={Duration:d\\.hh\\:mm} Completed={Completed:N0} Total={Total:N0} Errors={Errors:N0}", sw.Elapsed, affectedRecords, stackResponse.Total, stackResponse.Failures.Count); + _logger.LogInformation("Finished adding stack status: Time={Duration:d\\.hh\\:mm} Completed={Completed:N0} Total={Total:N0} Errors={Errors:N0}", sw.Elapsed, affectedRecords, stackResponse.Total, stackResponse.Failures?.Count ?? 0); _logger.LogInformation("Invalidating Stack Cache"); await _cache.RemoveByPrefixAsync(nameof(Stack)); diff --git a/src/Exceptionless.Core/Migrations/FixDuplicateStacks.cs b/src/Exceptionless.Core/Migrations/FixDuplicateStacks.cs index 224b5416cc..e08d21baca 100644 --- a/src/Exceptionless.Core/Migrations/FixDuplicateStacks.cs +++ b/src/Exceptionless.Core/Migrations/FixDuplicateStacks.cs @@ -8,14 +8,17 @@ using Foundatio.Repositories.Migrations; using Foundatio.Repositories.Models; using Microsoft.Extensions.Logging; -using Nest; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.Aggregations; +using Elastic.Clients.Elasticsearch.Core.ReindexRethrottle; +using Elastic.Clients.Elasticsearch.QueryDsl; using LogLevel = Microsoft.Extensions.Logging.LogLevel; namespace Exceptionless.Core.Migrations; public sealed class FixDuplicateStacks : MigrationBase { - private readonly IElasticClient _client; + private readonly ElasticsearchClient _client; private readonly ICacheClient _cache; private readonly IStackRepository _stackRepository; private readonly IEventRepository _eventRepository; @@ -39,12 +42,12 @@ public override async Task RunAsync(MigrationContext context) _logger.LogInformation("Getting duplicate stacks"); var duplicateStackAgg = await _client.SearchAsync(q => q - .QueryOnQueryString("is_deleted:false") + .Query(q => q.QueryString(qs => qs.Query("is_deleted:false"))) .Size(0) - .Aggregations(a => a.Terms("stacks", t => t.Field(f => f.DuplicateSignature).MinimumDocumentCount(2).Size(10000)))); + .AddAggregation("stacks", a => a.Terms(t => t.Field(f => f.DuplicateSignature).MinDocCount(2).Size(10000)))); _logger.LogRequest(duplicateStackAgg, LogLevel.Trace); - var buckets = duplicateStackAgg.Aggregations.Terms("stacks").Buckets; + var buckets = duplicateStackAgg.Aggregations?.GetStringTerms("stacks")?.Buckets ?? []; int total = buckets.Count; int processed = 0; int error = 0; @@ -62,7 +65,7 @@ public override async Task RunAsync(MigrationContext context) string? signature = null; try { - string[]? parts = duplicateSignature.Key.Split(':'); + string[]? parts = duplicateSignature.Key.ToString().Split(':'); if (parts.Length != 2) { _logger.LogError("Error parsing duplicate signature {DuplicateSignature}", duplicateSignature.Key); @@ -118,10 +121,10 @@ public override async Task RunAsync(MigrationContext context) { var response = await _client.UpdateByQueryAsync(u => u .Query(q => q.Bool(b => b.Must(m => m - .Terms(t => t.Field(f => f.StackId).Terms(duplicateStacks.Select(s => s.Id))) + .Terms(t => t.Field(f => f.StackId).Terms(new TermsQueryField(duplicateStacks.Select(s => (FieldValue)s.Id).ToList()))) ))) - .Script(s => s.Source($"ctx._source.stack_id = '{targetStack.Id}'").Lang(ScriptLang.Painless)) - .Conflicts(Elasticsearch.Net.Conflicts.Proceed) + .Script(s => s.Source($"ctx._source.stack_id = '{targetStack.Id}'").Lang(ScriptLanguage.Painless)) + .Conflicts(Conflicts.Proceed) .WaitForCompletion(false)); _logger.LogRequest(response, LogLevel.Trace); @@ -132,20 +135,20 @@ public override async Task RunAsync(MigrationContext context) do { attempts++; - var taskStatus = await _client.Tasks.GetTaskAsync(taskId); - var status = taskStatus.Task.Status; + var taskStatus = await _client.Tasks.GetAsync(taskId!.FullyQualifiedId); + var status = taskStatus.Task.Status as ReindexStatus; if (taskStatus.Completed) { // TODO: need to check to see if the task failed or completed successfully. Throw if it failed. if (_timeProvider.GetUtcNow().UtcDateTime.Subtract(taskStartedTime) > TimeSpan.FromSeconds(30)) - _logger.LogInformation("Script operation task ({TaskId}) completed: Created: {Created} Updated: {Updated} Deleted: {Deleted} Conflicts: {Conflicts} Total: {Total}", taskId, status.Created, status.Updated, status.Deleted, status.VersionConflicts, status.Total); + _logger.LogInformation("Script operation task ({TaskId}) completed: Created: {Created} Updated: {Updated} Deleted: {Deleted} Conflicts: {Conflicts} Total: {Total}", taskId, status?.Created, status?.Updated, status?.Deleted, status?.VersionConflicts, status?.Total); - affectedRecords += status.Created + status.Updated + status.Deleted; + affectedRecords += (status?.Created ?? 0) + (status?.Updated ?? 0) + (status?.Deleted ?? 0); break; } if (_timeProvider.GetUtcNow().UtcDateTime.Subtract(taskStartedTime) > TimeSpan.FromSeconds(30)) - _logger.LogInformation("Checking script operation task ({TaskId}) status: Created: {Created} Updated: {Updated} Deleted: {Deleted} Conflicts: {Conflicts} Total: {Total}", taskId, status.Created, status.Updated, status.Deleted, status.VersionConflicts, status.Total); + _logger.LogInformation("Checking script operation task ({TaskId}) status: Created: {Created} Updated: {Updated} Deleted: {Deleted} Conflicts: {Conflicts} Total: {Total}", taskId, status?.Created, status?.Updated, status?.Deleted, status?.VersionConflicts, status?.Total); var delay = TimeSpan.FromMilliseconds(50); if (attempts > 20) @@ -179,12 +182,12 @@ public override async Task RunAsync(MigrationContext context) await _client.Indices.RefreshAsync(_config.Stacks.VersionedName); duplicateStackAgg = await _client.SearchAsync(q => q - .QueryOnQueryString("is_deleted:false") + .Query(q => q.QueryString(qs => qs.Query("is_deleted:false"))) .Size(0) - .Aggregations(a => a.Terms("stacks", t => t.Field(f => f.DuplicateSignature).MinimumDocumentCount(2).Size(10000)))); + .AddAggregation("stacks", a => a.Terms(t => t.Field(f => f.DuplicateSignature).MinDocCount(2).Size(10000)))); _logger.LogRequest(duplicateStackAgg, LogLevel.Trace); - buckets = duplicateStackAgg.Aggregations.Terms("stacks").Buckets; + buckets = duplicateStackAgg.Aggregations?.GetStringTerms("stacks")?.Buckets ?? []; total += buckets.Count; batch++; diff --git a/src/Exceptionless.Core/Migrations/SetStackDuplicateSignature.cs b/src/Exceptionless.Core/Migrations/SetStackDuplicateSignature.cs index 435f3248ad..ff91daa7ec 100644 --- a/src/Exceptionless.Core/Migrations/SetStackDuplicateSignature.cs +++ b/src/Exceptionless.Core/Migrations/SetStackDuplicateSignature.cs @@ -5,13 +5,14 @@ using Foundatio.Repositories.Elasticsearch.Extensions; using Foundatio.Repositories.Migrations; using Microsoft.Extensions.Logging; -using Nest; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.Core.ReindexRethrottle; namespace Exceptionless.Core.Migrations; public sealed class SetStackDuplicateSignature : MigrationBase { - private readonly IElasticClient _client; + private readonly ElasticsearchClient _client; private readonly ExceptionlessElasticConfiguration _config; private readonly ICacheClient _cache; private readonly TimeProvider _timeProvider; @@ -33,12 +34,10 @@ public override async Task RunAsync(MigrationContext context) _logger.LogInformation("Done refreshing all indices"); _logger.LogInformation("Updating Stack mappings..."); - var response = await _client.MapAsync(d => + var response = await _client.Indices.PutMappingAsync(d => { - d.Index(_config.Stacks.VersionedName); - d.Properties(p => p.Keyword(f => f.Name(s => s.DuplicateSignature))); - - return d; + d.Indices(_config.Stacks.VersionedName); + d.Properties(p => p.Keyword(s => s.DuplicateSignature)); }); _logger.LogRequest(response); @@ -46,9 +45,9 @@ public override async Task RunAsync(MigrationContext context) var sw = Stopwatch.StartNew(); const string script = "ctx._source.duplicate_signature = ctx._source.project_id + ':' + ctx._source.signature_hash;"; var stackResponse = await _client.UpdateByQueryAsync(x => x - .QueryOnQueryString("NOT _exists_:duplicate_signature") - .Script(s => s.Source(script).Lang(ScriptLang.Painless)) - .Conflicts(Elasticsearch.Net.Conflicts.Proceed) + .Query(q => q.QueryString(qs => qs.Query("NOT _exists_:duplicate_signature"))) + .Script(s => s.Source(script).Lang(ScriptLanguage.Painless)) + .Conflicts(Conflicts.Proceed) .WaitForCompletion(false)); _logger.LogRequest(stackResponse, Microsoft.Extensions.Logging.LogLevel.Information); @@ -59,22 +58,22 @@ public override async Task RunAsync(MigrationContext context) do { attempts++; - var taskStatus = await _client.Tasks.GetTaskAsync(taskId); - var status = taskStatus.Task.Status; + var taskStatus = await _client.Tasks.GetAsync(taskId!.FullyQualifiedId); + var status = taskStatus.Task.Status as ReindexStatus; if (taskStatus.Completed) { // TODO: need to check to see if the task failed or completed successfully. Throw if it failed. - _logger.LogInformation("Script operation task ({TaskId}) completed: Created: {Created} Updated: {Updated} Deleted: {Deleted} Conflicts: {Conflicts} Total: {Total}", taskId, status.Created, status.Updated, status.Deleted, status.VersionConflicts, status.Total); - affectedRecords += status.Created + status.Updated + status.Deleted; + _logger.LogInformation("Script operation task ({TaskId}) completed: Created: {Created} Updated: {Updated} Deleted: {Deleted} Conflicts: {Conflicts} Total: {Total}", taskId, status?.Created, status?.Updated, status?.Deleted, status?.VersionConflicts, status?.Total); + affectedRecords += (status?.Created ?? 0) + (status?.Updated ?? 0) + (status?.Deleted ?? 0); break; } - _logger.LogInformation("Checking script operation task ({TaskId}) status: Created: {Created} Updated: {Updated} Deleted: {Deleted} Conflicts: {Conflicts} Total: {Total}", taskId, status.Created, status.Updated, status.Deleted, status.VersionConflicts, status.Total); + _logger.LogInformation("Checking script operation task ({TaskId}) status: Created: {Created} Updated: {Updated} Deleted: {Deleted} Conflicts: {Conflicts} Total: {Total}", taskId, status?.Created, status?.Updated, status?.Deleted, status?.VersionConflicts, status?.Total); var delay = TimeSpan.FromSeconds(attempts <= 5 ? 1 : 5); await Task.Delay(delay, _timeProvider); } while (true); - _logger.LogInformation("Finished adding stack duplicate signature: Time={Duration:d\\.hh\\:mm} Completed={Completed:N0} Total={Total:N0} Errors={Errors:N0}", sw.Elapsed, affectedRecords, stackResponse.Total, stackResponse.Failures.Count); + _logger.LogInformation("Finished adding stack duplicate signature: Time={Duration:d\\.hh\\:mm} Completed={Completed:N0} Total={Total:N0} Errors={Errors:N0}", sw.Elapsed, affectedRecords, stackResponse.Total, stackResponse.Failures?.Count ?? 0); _logger.LogInformation("Invalidating Stack Cache"); await _cache.RemoveByPrefixAsync(nameof(Stack)); diff --git a/src/Exceptionless.Core/Migrations/UpdateEventUsage.cs b/src/Exceptionless.Core/Migrations/UpdateEventUsage.cs index 700b113429..ed556d1863 100644 --- a/src/Exceptionless.Core/Migrations/UpdateEventUsage.cs +++ b/src/Exceptionless.Core/Migrations/UpdateEventUsage.cs @@ -8,7 +8,7 @@ using Foundatio.Repositories.Migrations; using Foundatio.Repositories.Models; using Microsoft.Extensions.Logging; -using Nest; +using Elastic.Clients.Elasticsearch; namespace Exceptionless.Core.Migrations; diff --git a/src/Exceptionless.Web/Controllers/AdminController.cs b/src/Exceptionless.Web/Controllers/AdminController.cs index 0549a57def..f1a6e436b1 100644 --- a/src/Exceptionless.Web/Controllers/AdminController.cs +++ b/src/Exceptionless.Web/Controllers/AdminController.cs @@ -295,37 +295,34 @@ await _workItemQueue.EnqueueAsync(new UpdateProjectNotificationSettingsWorkItem public async Task> GetElasticsearchInfoAsync() { var client = _configuration.Client; - var healthTask = client.Cluster.HealthAsync(); + var healthTask = client.Cluster.HealthAsync(r => r.Level(Elastic.Clients.Elasticsearch.Level.Indices)); var statsTask = client.Cluster.StatsAsync(); - var catIndicesTask = client.Cat.IndicesAsync(r => r.Bytes(Elasticsearch.Net.Bytes.B)); - var catShardsTask = client.Cat.ShardsAsync(); - await Task.WhenAll(healthTask, statsTask, catIndicesTask, catShardsTask); + var indicesStatsTask = client.Indices.StatsAsync(); + await Task.WhenAll(healthTask, statsTask, indicesStatsTask); var healthResponse = await healthTask; var statsResponse = await statsTask; - var catIndicesResponse = await catIndicesTask; - var catShardsResponse = await catShardsTask; + var indicesStatsResponse = await indicesStatsTask; - if (!healthResponse.IsValid || !statsResponse.IsValid || !catIndicesResponse.IsValid || !catShardsResponse.IsValid) + if (!healthResponse.IsValidResponse || !statsResponse.IsValidResponse || !indicesStatsResponse.IsValidResponse) return Problem(title: "Elasticsearch cluster information is unavailable."); - // Count unassigned shards per index - var unassignedByIndex = (catShardsResponse.Records ?? []) - .Where(s => string.Equals(s.State, "UNASSIGNED", StringComparison.OrdinalIgnoreCase)) - .GroupBy(s => s.Index ?? String.Empty, StringComparer.OrdinalIgnoreCase) - .ToDictionary(g => g.Key, g => g.Count(), StringComparer.OrdinalIgnoreCase); - - var indexDetails = (catIndicesResponse.Records ?? []) - .OrderByDescending(i => long.TryParse(i.StoreSize, out var s) ? s : 0) - .Select(i => new ElasticsearchIndexDetailResponse( - Index: i.Index, - Health: i.Health, - Status: i.Status, - Primary: int.TryParse(i.Primary, out var p) ? p : 0, - Replica: int.TryParse(i.Replica, out var r) ? r : 0, - DocsCount: long.TryParse(i.DocsCount, out var dc) ? dc : 0, - StoreSizeInBytes: long.TryParse(i.StoreSize, out var ss) ? ss : 0, - UnassignedShards: unassignedByIndex.GetValueOrDefault(i.Index ?? String.Empty, 0) + // Count unassigned shards per index from health response + var unassignedByIndex = (healthResponse.Indices ?? new Dictionary()) + .Where(kvp => kvp.Value.UnassignedShards > 0) + .ToDictionary(kvp => kvp.Key, kvp => kvp.Value.UnassignedShards, StringComparer.OrdinalIgnoreCase); + + var indexDetails = (indicesStatsResponse.Indices ?? new Dictionary()) + .OrderByDescending(kvp => kvp.Value.Total?.Store?.SizeInBytes ?? 0) + .Select(kvp => new ElasticsearchIndexDetailResponse( + Index: kvp.Key, + Health: kvp.Value.Health?.ToString().ToLowerInvariant(), + Status: kvp.Value.Status?.ToString().ToLowerInvariant(), + Primary: healthResponse.Indices?.GetValueOrDefault(kvp.Key)?.NumberOfShards ?? 0, + Replica: healthResponse.Indices?.GetValueOrDefault(kvp.Key)?.NumberOfReplicas ?? 0, + DocsCount: kvp.Value.Total?.Docs?.Count ?? 0, + StoreSizeInBytes: kvp.Value.Total?.Store?.SizeInBytes ?? 0, + UnassignedShards: unassignedByIndex.GetValueOrDefault(kvp.Key, 0) )) .ToArray(); @@ -342,7 +339,7 @@ public async Task> GetElasticsearchInfoA ), Indices: new ElasticsearchIndicesResponse( Count: statsResponse.Indices.Count, - DocsCount: statsResponse.Indices.Documents.Count, + DocsCount: statsResponse.Indices.Docs.Count, StoreSizeInBytes: statsResponse.Indices.Store.SizeInBytes ), IndexDetails: indexDetails @@ -355,43 +352,40 @@ public async Task> GetElasticsearch var client = _configuration.Client; try { - var repositoryResponse = await client.Cat.RepositoriesAsync(); - if (!repositoryResponse.IsValid) + var repositoryResponse = await client.Snapshot.GetRepositoryAsync(); + if (!repositoryResponse.IsValidResponse) return Problem(title: "Snapshot repository information is unavailable."); - if (!(repositoryResponse.Records?.Any() ?? false)) + if (repositoryResponse.Repositories is null || !repositoryResponse.Repositories.Any()) return Ok(new ElasticsearchSnapshotsResponse([], [])); - var repositoryNames = repositoryResponse.Records - .Where(r => !String.IsNullOrEmpty(r.Id)) - .Select(r => r.Id!) - .ToArray(); + var repositoryNames = repositoryResponse.Repositories.Select(r => r.Key).ToArray(); var snapshotTasks = repositoryNames .Select(async repositoryName => { - var snapshotResponse = await client.Cat.SnapshotsAsync(r => r.RepositoryName(repositoryName)); - if (!snapshotResponse.IsValid) + var snapshotResponse = await client.Snapshot.GetAsync(repositoryName, "*"); + if (!snapshotResponse.IsValidResponse) return ( RepositoryName: repositoryName, Snapshots: Array.Empty(), Error: $"Unable to retrieve snapshots for repository: {repositoryName}." ); - var snapshotRecords = snapshotResponse.Records?.ToArray() ?? []; + var snapshots = snapshotResponse.Snapshots?.ToArray() ?? []; return ( RepositoryName: repositoryName, - Snapshots: snapshotRecords.Select(s => new ElasticsearchSnapshotResponse( + Snapshots: snapshots.Select(s => new ElasticsearchSnapshotResponse( Repository: repositoryName, - Name: s.Id ?? String.Empty, - Status: s.Status ?? String.Empty, - StartTime: s.StartEpoch > 0 ? DateTimeOffset.FromUnixTimeSeconds(s.StartEpoch).UtcDateTime : null, - EndTime: s.EndEpoch > 0 ? DateTimeOffset.FromUnixTimeSeconds(s.EndEpoch).UtcDateTime : null, + Name: s.Snapshot, + Status: s.State ?? String.Empty, + StartTime: s.StartTime?.UtcDateTime, + EndTime: s.EndTime?.UtcDateTime, Duration: s.Duration?.ToString() ?? String.Empty, - IndicesCount: s.Indices, - SuccessfulShards: s.SuccessfulShards, - FailedShards: s.FailedShards, - TotalShards: s.TotalShards + IndicesCount: s.Indices?.Count ?? 0, + SuccessfulShards: s.Shards?.Successful ?? 0, + FailedShards: s.Shards?.Failed ?? 0, + TotalShards: s.Shards?.Total ?? 0 )).ToArray(), Error: (string?)null ); diff --git a/src/Exceptionless.Web/Controllers/EventController.cs b/src/Exceptionless.Web/Controllers/EventController.cs index a37be67be3..2f5a02bc8b 100644 --- a/src/Exceptionless.Web/Controllers/EventController.cs +++ b/src/Exceptionless.Web/Controllers/EventController.cs @@ -59,7 +59,7 @@ public EventController(IEventRepository repository, IValidator userDescriptionValidator, FormattingPluginManager formattingPluginManager, ICacheClient cacheClient, - JsonSerializerSettings jsonSerializerSettings, + ITextSerializer serializer, ApiMapper mapper, PersistentEventQueryValidator validator, AppOptions appOptions, @@ -312,7 +312,7 @@ private async Task>> GetInternalAsync( Date = e.Date, Data = summaryData.Data }; - }).ToList(), events.HasMore && !NextPageExceedsSkipLimit(page, limit), page, events.Total, events.Hits.FirstOrDefault()?.GetSortToken(), events.Hits.LastOrDefault()?.GetSortToken()); + }).ToList(), events.HasMore && !NextPageExceedsSkipLimit(page, limit), page, events.Total, events.Hits.FirstOrDefault()?.GetSortToken(_serializer), events.Hits.LastOrDefault()?.GetSortToken(_serializer)); case "stack_recent": case "stack_frequent": case "stack_new": @@ -358,7 +358,7 @@ private async Task>> GetInternalAsync( return OkWithResourceLinks(summaries.Take(limit).ToList(), summaries.Count > limit && !NextPageExceedsSkipLimit(resolvedPage, limit), resolvedPage, total); default: events = await GetEventsInternalAsync(sf, ti, filter, sort, page, limit, before, after); - return OkWithResourceLinks(events.Documents.ToArray(), events.HasMore && !NextPageExceedsSkipLimit(page, limit), page, events.Total, events.Hits.FirstOrDefault()?.GetSortToken(), events.Hits.LastOrDefault()?.GetSortToken()); + return OkWithResourceLinks(events.Documents.ToArray(), events.HasMore && !NextPageExceedsSkipLimit(page, limit), page, events.Total, events.Hits.FirstOrDefault()?.GetSortToken(_serializer), events.Hits.LastOrDefault()?.GetSortToken(_serializer)); } } catch (ApplicationException ex) @@ -420,7 +420,7 @@ private Task> GetEventsInternalAsync(AppFilter sf, .Index(ti.Range.UtcStart, ti.Range.UtcEnd), o => page.HasValue ? o.PageNumber(page).PageLimit(limit) - : o.SearchBeforeToken(before).SearchAfterToken(after).PageLimit(limit)); + : o.SearchBeforeToken(before, _serializer).SearchAfterToken(after, _serializer).PageLimit(limit)); } /// diff --git a/src/Exceptionless.Web/Controllers/ProjectController.cs b/src/Exceptionless.Web/Controllers/ProjectController.cs index ed2a9b1ef3..a37a3cb8d2 100644 --- a/src/Exceptionless.Web/Controllers/ProjectController.cs +++ b/src/Exceptionless.Web/Controllers/ProjectController.cs @@ -50,6 +50,7 @@ public ProjectController( SlackService slackService, ApiMapper mapper, IAppQueryValidator validator, + ITextSerializer serializer, AppOptions options, UsageService usageService, TimeProvider timeProvider, diff --git a/tests/Exceptionless.Tests/IntegrationTestsBase.cs b/tests/Exceptionless.Tests/IntegrationTestsBase.cs index 22a812e0c9..3f6ca2ff84 100644 --- a/tests/Exceptionless.Tests/IntegrationTestsBase.cs +++ b/tests/Exceptionless.Tests/IntegrationTestsBase.cs @@ -22,7 +22,8 @@ using Foundatio.Utility; using Foundatio.Xunit; using Microsoft.AspNetCore.TestHost; -using Nest; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.QueryDsl; using Xunit; using HttpMethod = System.Net.Http.HttpMethod; using LogLevel = Microsoft.Extensions.Logging.LogLevel; diff --git a/tests/Exceptionless.Tests/Migrations/FixDuplicateStacksMigrationTests.cs b/tests/Exceptionless.Tests/Migrations/FixDuplicateStacksMigrationTests.cs index 7267bb590b..4d7bb4406f 100644 --- a/tests/Exceptionless.Tests/Migrations/FixDuplicateStacksMigrationTests.cs +++ b/tests/Exceptionless.Tests/Migrations/FixDuplicateStacksMigrationTests.cs @@ -8,7 +8,8 @@ using Foundatio.Repositories.Migrations; using Foundatio.Repositories.Utility; using Foundatio.Utility; -using Nest; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.QueryDsl; using Xunit; namespace Exceptionless.Tests.Migrations; @@ -58,7 +59,7 @@ public async Task WillMergeDuplicatedStacks() await _eventRepository.AddAsync(_eventData.GenerateEvents(count: 100, stackId: originalStack.Id), o => o.ImmediateConsistency()); await _eventRepository.AddAsync(_eventData.GenerateEvents(count: 10, stackId: duplicateStack.Id), o => o.ImmediateConsistency()); - var results = await _stackRepository.FindAsync(q => q.ElasticFilter(Query.Term(s => s.DuplicateSignature, originalStack.DuplicateSignature))); + var results = await _stackRepository.FindAsync(q => q.ElasticFilter(new TermQuery { Field = Infer.Field(s => s.DuplicateSignature), Value = originalStack.DuplicateSignature })); Assert.Equal(2, results.Total); var migration = GetService(); @@ -67,7 +68,7 @@ public async Task WillMergeDuplicatedStacks() await RefreshDataAsync(); - results = await _stackRepository.FindAsync(q => q.ElasticFilter(Query.Term(s => s.DuplicateSignature, originalStack.DuplicateSignature))); + results = await _stackRepository.FindAsync(q => q.ElasticFilter(new TermQuery { Field = Infer.Field(s => s.DuplicateSignature), Value = originalStack.DuplicateSignature })); Assert.Single(results.Documents); var updatedOriginalStack = await _stackRepository.GetByIdAsync(originalStack.Id, o => o.IncludeSoftDeletes()); @@ -111,7 +112,7 @@ public async Task WillMergeToStackWithMostEvents() await _eventRepository.AddAsync(_eventData.GenerateEvents(count: 10, stackId: originalStack.Id), o => o.ImmediateConsistency()); await _eventRepository.AddAsync(_eventData.GenerateEvents(count: 100, stackId: biggerStack.Id), o => o.ImmediateConsistency()); - var results = await _stackRepository.FindAsync(q => q.ElasticFilter(Query.Term(s => s.DuplicateSignature, originalStack.DuplicateSignature))); + var results = await _stackRepository.FindAsync(q => q.ElasticFilter(new TermQuery { Field = Infer.Field(s => s.DuplicateSignature), Value = originalStack.DuplicateSignature })); Assert.Equal(2, results.Total); var migration = GetService(); @@ -120,7 +121,7 @@ public async Task WillMergeToStackWithMostEvents() await RefreshDataAsync(); - results = await _stackRepository.FindAsync(q => q.ElasticFilter(Query.Term(s => s.DuplicateSignature, originalStack.DuplicateSignature))); + results = await _stackRepository.FindAsync(q => q.ElasticFilter(new TermQuery { Field = Infer.Field(s => s.DuplicateSignature), Value = originalStack.DuplicateSignature })); Assert.Single(results.Documents); var updatedOriginalStack = await _stackRepository.GetByIdAsync(originalStack.Id, o => o.IncludeSoftDeletes()); @@ -160,7 +161,7 @@ public async Task WillNotMergeDuplicatedDeletedStacks() await _stackRepository.AddAsync(new[] { originalStack, duplicateStack }, o => o.ImmediateConsistency()); - var results = await _stackRepository.FindAsync(q => q.ElasticFilter(Query.Term(s => s.DuplicateSignature, originalStack.DuplicateSignature))); + var results = await _stackRepository.FindAsync(q => q.ElasticFilter(new TermQuery { Field = Infer.Field(s => s.DuplicateSignature), Value = originalStack.DuplicateSignature })); Assert.Single(results.Documents); var migration = GetService(); @@ -169,7 +170,7 @@ public async Task WillNotMergeDuplicatedDeletedStacks() await RefreshDataAsync(); - results = await _stackRepository.FindAsync(q => q.ElasticFilter(Query.Term(s => s.DuplicateSignature, originalStack.DuplicateSignature))); + results = await _stackRepository.FindAsync(q => q.ElasticFilter(new TermQuery { Field = Infer.Field(s => s.DuplicateSignature), Value = originalStack.DuplicateSignature })); Assert.Single(results.Documents); var updatedOriginalStack = await _stackRepository.GetByIdAsync(originalStack.Id, o => o.IncludeSoftDeletes()); diff --git a/tests/Exceptionless.Tests/Migrations/SetStackDuplicateSignatureMigrationTests.cs b/tests/Exceptionless.Tests/Migrations/SetStackDuplicateSignatureMigrationTests.cs index 5a5ea1cc70..c43aec006e 100644 --- a/tests/Exceptionless.Tests/Migrations/SetStackDuplicateSignatureMigrationTests.cs +++ b/tests/Exceptionless.Tests/Migrations/SetStackDuplicateSignatureMigrationTests.cs @@ -6,7 +6,8 @@ using Foundatio.Repositories; using Foundatio.Repositories.Migrations; using Foundatio.Utility; -using Nest; +using Elastic.Clients.Elasticsearch; +using Elastic.Clients.Elasticsearch.QueryDsl; using Xunit; namespace Exceptionless.Tests.Migrations; @@ -50,7 +51,7 @@ public async Task WillSetStackDuplicateSignature() Assert.NotEmpty(actualStack.SignatureHash); Assert.Equal($"{actualStack.ProjectId}:{actualStack.SignatureHash}", actualStack.DuplicateSignature); - var results = await _repository.FindAsync(q => q.ElasticFilter(Query.Term(s => s.DuplicateSignature, expectedDuplicateSignature))); + var results = await _repository.FindAsync(q => q.ElasticFilter(new TermQuery { Field = Infer.Field(s => s.DuplicateSignature), Value = expectedDuplicateSignature })); Assert.Single(results.Documents); } } From 3890cb47dd14876436180d65a75b4de227224cb2 Mon Sep 17 00:00:00 2001 From: Blake Niemyjski Date: Sun, 22 Mar 2026 11:19:53 -0500 Subject: [PATCH 7/7] fix: remove duplicate Id mapping in EventIndex SetupDefaults() already maps the Id property. The explicit Keyword(e => e.Id) mapping caused a duplicate key error in the new Elastic.Clients.Elasticsearch client. --- .../Repositories/Configuration/Indexes/EventIndex.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Exceptionless.Core/Repositories/Configuration/Indexes/EventIndex.cs b/src/Exceptionless.Core/Repositories/Configuration/Indexes/EventIndex.cs index ebc0076695..67e8a1c738 100644 --- a/src/Exceptionless.Core/Repositories/Configuration/Indexes/EventIndex.cs +++ b/src/Exceptionless.Core/Repositories/Configuration/Indexes/EventIndex.cs @@ -57,7 +57,6 @@ public override void ConfigureIndexMapping(TypeMappingDescriptor t.Match("*-s").Mapping(m => m.Keyword(s => s.IgnoreAbove(1024))))) .Properties(p => p .SetupDefaults() - .Keyword(e => e.Id) .Keyword(e => e.OrganizationId) .FieldAlias(Alias.OrganizationId, a => a.Path(f => f.OrganizationId)) .Keyword(e => e.ProjectId)