Migrate data handling from Serilog's LogEvent across to System.Text.Json.JsonObject, with the help of Seq.Syntax v2.0

src/Roastery/Data/Database.cs

@@ -1,6 +1,5 @@
 using System;
 using System.Collections.Generic;
-using System.Diagnostics;
 using System.Globalization;
 using System.Linq;
 using System.Reflection;

src/Roastery/Util/Distribution.cs

@@ -1,7 +1,6 @@
 using System;
 using System.Collections.Generic;
 using System.Runtime.CompilerServices;
-using System.Threading;
 
 namespace Roastery.Util;
 

src/Roastery/Web/RequestLoggingMiddleware.cs

@@ -1,6 +1,5 @@
 using System;
 using System.Diagnostics;
-using System.Diagnostics.Metrics;
 using System.Net;
 using System.Threading.Tasks;
 using Roastery.Metrics;

src/SeqCli/Api/EventEntityJson.cs

@@ -0,0 +1,104 @@
+// Copyright © Datalust Pty Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.Text;
+using System.Text.Json.Nodes;
+using Seq.Api.Model.Events;
+using Seq.Api.Model.Shared;
+using SeqCli.Data;
+using SeqCli.Output;
+
+namespace SeqCli.Api;
+
+/// <summary>
+/// Converts event entities into compact JSON format for further processing. This class is only necessary because
+/// <c>Seq.Api</c> doesn't yet provide a simple compact-JSON based result format for searches. Once we've filled
+/// that gap, this class, and <see cref="OutputFormat.WriteEventEntity"/> can be removed.
+/// </summary>
+static class EventEntityJson
+{
+    public static JsonObject ToEventJson(EventEntity evt)
+    {
+        var eventJson = new JsonObject
+        {
+            // Earlier versions relied on Serilog output formatting to show timestamps in local time; we'll need
+            // to consider adding some compensating mechanism to `Seq.Syntax`.
+            ["@t"] = DateTimeOffset.ParseExact(evt.Timestamp, "o", CultureInfo.InvariantCulture)
+                .ToLocalTime().ToString("o", CultureInfo.InvariantCulture)
+        };
+
+        if (evt.MessageTemplateTokens != null)
+            eventJson["@mt"] = ToMessageTemplateText(evt.MessageTemplateTokens);
+
+        if (!string.IsNullOrWhiteSpace(evt.Level) && evt.Level != "Information")
+            eventJson["@l"] = evt.Level;
+
+        if (!string.IsNullOrWhiteSpace(evt.Exception))
+            eventJson["@x"] = evt.Exception;
+
+        if (!string.IsNullOrWhiteSpace(evt.TraceId))
+            eventJson["@tr"] = evt.TraceId;
+
+        if (!string.IsNullOrWhiteSpace(evt.SpanId))
+            eventJson["@sp"] = evt.SpanId;
+
+        if (!string.IsNullOrWhiteSpace(evt.ParentId))
+            eventJson["@ps"] = evt.ParentId;
+
+        if (!string.IsNullOrWhiteSpace(evt.Start))
+            eventJson["@st"] = evt.Start;
+
+        if (!string.IsNullOrWhiteSpace(evt.SpanKind))
+            eventJson["@sk"] = evt.SpanKind;
+
+        if (evt.Resource?.Count > 0)
+            eventJson["@ra"] = ToPropertiesObject(evt.Resource);
+
+        if (evt.Scope?.Count > 0)
+            eventJson["@sa"] = ToPropertiesObject(evt.Scope);
+
+        if (evt.Properties != null)
+        {
+            foreach (var property in evt.Properties)
+                eventJson[EventJsonFormat.EscapeUserPropertyName(property.Name)] = ToSystemTextJson.FromApiValue(property.Value);
+        }
+
+        return eventJson;
+    }
+
+    static string ToMessageTemplateText(List<MessageTemplateTokenPart> tokens)
+    {
+        var text = new StringBuilder();
+        foreach (var token in tokens)
+        {
+            if (token.Text != null)
+                text.Append(token.Text.Replace("{", "{{").Replace("}", "}}"));
+            else
+                text.Append(token.RawText ?? $"{{{token.PropertyName}}}");
+        }
+
+        return text.ToString();
+    }
+
+    static JsonObject ToPropertiesObject(List<EventPropertyPart> properties)
+    {
+        var result = new JsonObject();
+        foreach (var property in properties)
+            result[property.Name] = ToSystemTextJson.FromApiValue(property.Value);
+        return result;
+    }
+}

src/SeqCli/Api/LevelMapping.cs

@@ -0,0 +1,99 @@
+// Copyright © Datalust and contributors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+using System;
+using System.Collections.Generic;
+using Seq.Api.Model.LogEvents;
+
+namespace SeqCli.Api;
+
+public static class LevelMapping
+{
+    static readonly Dictionary<string, string> LevelsByName =
+        new(StringComparer.OrdinalIgnoreCase)
+        {
+            ["t"] = "Trace",
+            ["tr"] = "Trace",
+            ["trc"] = "Trace",
+            ["trce"] = "Trace",
+            ["trace"] = "Trace",
+            ["v"] = "Verbose",
+            ["ver"] = "Verbose",
+            ["vrb"] = "Verbose",
+            ["verb"] = "Verbose",
+            ["verbose"] = "Verbose",
+            ["d"] = "Debug",
+            ["de"] = "Debug",
+            ["dbg"] = "Debug",
+            ["deb"] = "Debug",
+            ["dbug"] = "Debug",
+            ["debu"] = "Debug",
+            ["debug"] = "Debug",
+            ["i"] = "Information",
+            ["in"] = "Information",
+            ["inf"] = "Information",
+            ["info"] = "Information",
+            ["information"] = "Information",
+            ["notice"] = "Notice",
+            ["w"] = "Warning",
+            ["wa"] = "Warning",
+            ["war"] = "Warning",
+            ["wrn"] = "Warning",
+            ["warn"] = "Warning",
+            ["warning"] = "Warning",
+            ["e"] = "Error",
+            ["er"] = "Error",
+            ["err"] = "Error",
+            ["erro"] = "Error",
+            ["eror"] = "Error",
+            ["error"] = "Error",
+            ["f"] = "Fatal",
+            ["fa"] = "Fatal",
+            ["ftl"] = "Fatal",
+            ["fat"] = "Fatal",
+            ["fatl"] = "Fatal",
+            ["fatal"] = "Fatal",
+            ["c"] = "Critical",
+            ["cr"] = "Critical",
+            ["crt"] = "Critical",
+            ["cri"] = "Critical",
+            ["crit"] = "Critical",
+            ["critical"] = "Critical",
+            ["emerg"] = "Emergency",
+            ["alert"] = "Alert",
+            ["panic"] = "Panic"
+        };
+
+    // Intended only for use by ingest extraction patterns.
+    public static string ToFullLevelName(string level)
+    {
+        return LevelsByName.TryGetValue(level, out var m) ? m : level;
+    }
+
+    public static LogEventLevel ToSeqApiLogEventLevel(string level)
+    {
+        if (string.IsNullOrEmpty(level))
+            return LogEventLevel.Information;
+
+        return ToFullLevelName(level) switch
+        {
+            "Trace" or "Verbose" => LogEventLevel.Verbose,
+            "Debug" => LogEventLevel.Debug,
+            "Warning" => LogEventLevel.Warning,
+            "Error" => LogEventLevel.Error,
+            "Fatal" or "Critical" or "Emergency" or "Alert" or "Panic" => LogEventLevel.Fatal,
+            _ => LogEventLevel.Information
+        };
+    }
+}

src/SeqCli/Api/ToSystemTextJson.cs

@@ -0,0 +1,45 @@
+// Copyright © Datalust Pty Ltd
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+using System.Text.Json.Nodes;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using SeqCli.Data;
+
+namespace SeqCli.Api;
+
+static class ToSystemTextJson
+{
+    /// <summary>
+    /// Convert a value deserialized by the Seq API client into its `System.Text.Json` equivalent.
+    /// </summary>
+    public static JsonNode? FromApiValue(object? value)
+    {
+        return value switch
+        {
+            null => null,
+            JToken token => FromNewtonsoft(token),
+            _ => EventJsonFormat.CreateScalar(value)
+        };
+    }
+
+    /// Conversion helper for values retrieved through the Seq API client.
+    public static JsonNode? FromNewtonsoft(JToken token)
+    {
+        if (token is JValue { Value: null })
+            return null;
+
+        return JsonNode.Parse(token.ToString(Formatting.None));
+    }
+}

src/SeqCli/Apps/AppLoader.cs

@@ -34,7 +34,8 @@ class AppLoader : IDisposable
     [
         typeof(SeqApp).Assembly,
         typeof(Log).Assembly,
-        typeof(SerilogExpression).Assembly
+        // Seq.Syntax uses version-specific assembly names to improve our chances of successful loading.
+        typeof(SeqExpression).Assembly
     ];
 
     public AppLoader(string packageBinaryPath)

src/SeqCli/Apps/Hosting/AppContainer.cs

@@ -21,7 +21,7 @@
 using Newtonsoft.Json.Linq;
 using Seq.Apps;
 using Seq.Apps.LogEvents;
-using SeqCli.Mapping;
+using SeqCli.Api;
 using Serilog;
 using Serilog.Events;
 using Serilog.Formatting.Compact.Reader;
@@ -109,11 +109,11 @@ async Task SendTypedEventAsync(string clef)
         {
             if (_seqApp is ISubscribeTo<LogEventData> led)
             {
-                led.On(EventFormat.FromRaw(eventId, eventType, serilogEvent));
+                led.On(EventFormat.FromSerilogLogEvent(eventId, eventType, serilogEvent));
             }
             else if (_seqApp is ISubscribeToAsync<LogEventData> leda)
             {
-                await leda.OnAsync(EventFormat.FromRaw(eventId, eventType, serilogEvent));
+                await leda.OnAsync(EventFormat.FromSerilogLogEvent(eventId, eventType, serilogEvent));
             }
             else if (_seqApp is ISubscribeTo<LogEvent> sled)
             {
@@ -143,7 +143,8 @@ LogEvent ReadSerilogEvent(string clef, out string eventId, out uint eventType)
         if (jobject.TryGetValue("@l", out var levelToken))
         {
             jobject.Remove("@l");
-            jobject.Add("@l", new JValue(LevelMapping.ToSerilogLevel(levelToken.Value<string>()!).ToString()));
+            // The Seq.Api `LogEventLevel` enum intentionally matches the Serilog one.
+            jobject.Add("@l", new JValue(LevelMapping.ToSeqApiLogEventLevel(levelToken.Value<string>()!).ToString()));
         }
 
         SanitizeTraceIdentifiers(jobject);

src/SeqCli/Apps/Hosting/EventFormat.cs

@@ -24,7 +24,7 @@ namespace SeqCli.Apps.Hosting;
 
 static class EventFormat
 {
-    public static Event<LogEventData> FromRaw(string eventId, uint eventType, LogEvent raw)
+    public static Event<LogEventData> FromSerilogLogEvent(string eventId, uint eventType, LogEvent raw)
     {
         var properties = new Dictionary<string, object?>();
         foreach (var prop in raw.Properties)

src/SeqCli/Cli/Commands/Alert/CreateCommand.cs

@@ -17,12 +17,10 @@
 using System.Linq;
 using System.Threading.Tasks;
 using Seq.Api.Model.Alerting;
-using Seq.Api.Model.LogEvents;
 using Seq.Api.Model.Shared;
 using SeqCli.Api;
 using SeqCli.Cli.Features;
 using SeqCli.Config;
-using SeqCli.Mapping;
 using SeqCli.Signals;
 using SeqCli.Syntax;
 using SeqCli.Util;
@@ -178,7 +176,7 @@ protected override async Task<int> Run()
             alert.Having = _having;
 
         if (_notificationLevel != null)
-            alert.NotificationLevel = Enum.Parse<LogEventLevel>(LevelMapping.ToFullLevelName(_notificationLevel));
+            alert.NotificationLevel = LevelMapping.ToSeqApiLogEventLevel(_notificationLevel);
 
         if (_suppressionTime != null)
             alert.SuppressionTime = DurationMoniker.ToTimeSpan(_suppressionTime);

src/SeqCli/Cli/Commands/ApiKey/CreateCommand.cs

@@ -16,13 +16,11 @@
 using System.Linq;
 using System.Threading.Tasks;
 using Seq.Api;
-using Seq.Api.Model.LogEvents;
 using Seq.Api.Model.Security;
 using Seq.Api.Model.Shared;
 using SeqCli.Api;
 using SeqCli.Cli.Features;
 using SeqCli.Config;
-using SeqCli.Mapping;
 using SeqCli.Util;
 using Serilog;
 
@@ -125,7 +123,7 @@ protected override async Task<int> Run()
 
         if (_level != null)
         {
-            apiKey.InputSettings.MinimumLevel = Enum.Parse<LogEventLevel>(LevelMapping.ToFullLevelName(_level));
+            apiKey.InputSettings.MinimumLevel = LevelMapping.ToSeqApiLogEventLevel(_level);
         }
 
         apiKey.AssignedPermissions.Clear();