Added extended logging capability with metadata in addition to using …

APIDOCS.md

@@ -5282,6 +5282,7 @@ RESPONSE:
 		"logQueries": false,
 		"noStackTrace": false,
 		"useLocalTime": false,
+		"logQueryMetadata": false,
 		"logFolder": "logs",
 		"maxLogFileDays": 30,
 		"enableInMemoryStats": false,
@@ -5439,6 +5440,7 @@ WHERE:
 - `logQueries` (optional): Enable this option to log every query received by this DNS Server and the corresponding response answers into the log file.  Initial value is `false`.
 - `noStackTrace` (optional): Enable to log only short error messages instead of full exception stack trace.
 - `useLocalTime` (optional): Enable this option to use local time instead of UTC for logging.  Initial value is `false`.
+- `logQueryMetadata` (optional): Enable this option to pass extended metadata (blocking source, matched domain, block list URL, blocking group, and matched regex) to installed query logger apps. This option is node local and does not affect the log file, nor the Extended DNS Error and TXT blocking report sent to clients. When enabled, the `Tag` property of a blocked `DnsDatagram` response holds a `DnsServerResponseMetadata` object instead of a `DnsServerResponseType` value, so DNS Apps must read it using `DnsServerResponseTag.GetResponseType()`. Initial value is `false`.
 - `logFolder` (optional): The folder path on the server where the log files should be saved. The path can be relative to the DNS server config folder. Initial value is `logs`.
 - `maxLogFileDays` (optional): Max number of days to keep the log files. Log files older than the specified number of days will be deleted automatically. Recommended value is `365`. Set `0` to disable auto delete.
 - `enableInMemoryStats` (optional): Set this option to `true` to enable in-memory stats. When enabled, only Last Hour data will be available on Dashboard and no stats data will be stored on disk.

Apps/AdvancedBlockingApp/App.cs

@@ -611,14 +611,36 @@ string GetBlockingReport()
                 return blockingReport;
             }
 
+            DnsQueryLogMetadata GetLogMetadata()
+            {
+                Dictionary<string, string> values = new Dictionary<string, string>(6, StringComparer.OrdinalIgnoreCase)
+                {
+                    ["source"] = "advanced-blocking-app",
+                    ["group"] = group.Name
+                };
+
+                if (blockListUrl?.Uri is not null)
+                    values["blockListUrl"] = blockListUrl.Uri.AbsoluteUri;
+
+                if (blockedRegex is null)
+                    values["domain"] = blockedDomain ?? question.Name;
+                else
+                    values["regex"] = blockedRegex;
+
+                return new DnsQueryLogMetadata(values);
+            }
+
+            DnsQueryLogMetadata logMetadata = GetLogMetadata();
+            DnsServerResponseMetadata responseMetadata = new DnsServerResponseMetadata(DnsServerResponseType.Blocked, logMetadata);
+
             if (group.AllowTxtBlockingReport && (question.Type == DnsResourceRecordType.TXT))
             {
                 //return meta data
                 string blockingReport = GetBlockingReport();
 
                 DnsResourceRecord[] answer = [new DnsResourceRecord(question.Name, DnsResourceRecordType.TXT, question.Class, _blockingAnswerTtl, new DnsTXTRecordData(blockingReport))];
 
-                return Task.FromResult<DnsDatagram?>(new DnsDatagram(request.Identifier, true, DnsOpcode.StandardQuery, false, false, request.RecursionDesired, false, false, false, DnsResponseCode.NoError, request.Question, answer));
+                return Task.FromResult<DnsDatagram?>(new DnsDatagram(request.Identifier, true, DnsOpcode.StandardQuery, false, false, request.RecursionDesired, false, false, false, DnsResponseCode.NoError, request.Question, answer) { Tag = responseMetadata });
             }
             else
             {
@@ -703,7 +725,7 @@ string GetBlockingReport()
                     }
                 }
 
-                return Task.FromResult<DnsDatagram?>(new DnsDatagram(request.Identifier, true, DnsOpcode.StandardQuery, false, false, request.RecursionDesired, false, false, false, rcode, request.Question, answer, authority, null, request.EDNS is null ? ushort.MinValue : _dnsServer!.UdpPayloadSize, EDnsHeaderFlags.None, options));
+                return Task.FromResult<DnsDatagram?>(new DnsDatagram(request.Identifier, true, DnsOpcode.StandardQuery, false, false, request.RecursionDesired, false, false, false, rcode, request.Question, answer, authority, null, request.EDNS is null ? ushort.MinValue : _dnsServer!.UdpPayloadSize, EDnsHeaderFlags.None, options) { Tag = responseMetadata });
             }
         }
 

Apps/DnsRebindingProtectionApp/App.cs

@@ -177,7 +177,7 @@ public Task<DnsDatagram> PostProcessAsync(DnsDatagram request, IPEndPoint remote
             if (!_enableProtection || response.AuthoritativeAnswer)
                 return Task.FromResult(response);
 
-            if ((response.Tag is not null) && (response.Tag is DnsServerResponseType responseType) && (responseType == DnsServerResponseType.Blocked))
+            if (DnsServerResponseTag.GetResponseType(response.Tag) == DnsServerResponseType.Blocked)
                 return Task.FromResult(response); //allow blocked responses
 
             IPAddress remoteIP = remoteEP.Address;

Apps/LogExporterApp/App.cs

@@ -31,7 +31,7 @@ You should have received a copy of the GNU General Public License
 
 namespace LogExporter
 {
-    public sealed class App : IDnsApplication, IDnsQueryLogger
+    public sealed class App : IDnsApplication, IDnsQueryLoggerEx
     {
         #region variables
 
@@ -142,11 +142,16 @@ public Task InitializeAsync(IDnsServer dnsServer, string? config)
         }
 
         public Task InsertLogAsync(DateTime timestamp, DnsDatagram request, IPEndPoint remoteEP, DnsTransportProtocol protocol, DnsDatagram response)
+        {
+            return InsertLogAsync(timestamp, request, remoteEP, protocol, response, null);
+        }
+
+        public Task InsertLogAsync(DateTime timestamp, DnsDatagram request, IPEndPoint remoteEP, DnsTransportProtocol protocol, DnsDatagram response, DnsQueryLogMetadata? metadata)
         {
             if (_enableLogging)
             {
                 if (_queuedLogs.Count < _config!.MaxQueueSize)
-                    _queuedLogs.Enqueue(new LogEntry(timestamp, remoteEP, protocol, request, response, _config.EnableEdnsLogging));
+                    _queuedLogs.Enqueue(new LogEntry(timestamp, remoteEP, protocol, request, response, _config.EnableEdnsLogging, metadata));
             }
 
             return Task.CompletedTask;

Apps/LogExporterApp/LogEntry.cs

@@ -34,15 +34,17 @@ namespace LogExporter
 {
     public class LogEntry
     {
-        public LogEntry(DateTime timestamp, IPEndPoint remoteEP, DnsTransportProtocol protocol, DnsDatagram request, DnsDatagram response, bool ednsLogging = false)
+        public LogEntry(DateTime timestamp, IPEndPoint remoteEP, DnsTransportProtocol protocol, DnsDatagram request, DnsDatagram response, bool ednsLogging = false, DnsQueryLogMetadata? metadata = null)
         {
             // Assign timestamp and ensure it's in UTC
             Timestamp = timestamp.Kind == DateTimeKind.Utc ? timestamp : timestamp.ToUniversalTime();
 
             // Extract client information
             ClientIp = remoteEP.Address.ToString();
             Protocol = protocol;
-            ResponseType = response.Tag == null ? DnsServerResponseType.Recursive : (DnsServerResponseType)response.Tag;
+            ResponseType = DnsServerResponseTag.GetResponseType(response.Tag);
+            DnsQueryLogMetadata? logMetadata = metadata ?? DnsServerResponseTag.GetLogMetadata(response.Tag);
+            BlockingMetadata = logMetadata?.Values;
 
             if ((ResponseType == DnsServerResponseType.Recursive) && (response.Metadata is not null))
                 ResponseRtt = response.Metadata.RoundTripTime;
@@ -85,18 +87,27 @@ public LogEntry(DateTime timestamp, IPEndPoint remoteEP, DnsTransportProtocol pr
 
             foreach (EDnsOption extendedErrorLog in response.EDNS.Options.Where(o => o.Code == EDnsOptionCode.EXTENDED_DNS_ERROR))
             {
-                string[] extractedData = extendedErrorLog.Data.ToString().Replace("[", string.Empty).Replace("]", string.Empty).Split(":", StringSplitOptions.TrimEntries);
+                string[] extractedData = extendedErrorLog.Data.ToString().Replace("[", string.Empty).Replace("]", string.Empty).Split(":", 2, StringSplitOptions.TrimEntries);
+                string? errType = null;
+                string? message = null;
+
+                if (extractedData.Length > 0)
+                    errType = extractedData[0];
+
+                if (extractedData.Length > 1)
+                    message = extractedData[1];
 
                 EDNS.Add(new EDNSLog
                 {
-                    ErrType = extractedData[0],
-                    Message = extractedData[1]
+                    ErrType = errType,
+                    Message = message
                 });
             }
         }
 
         public List<DnsResourceRecord> Answers { get; private set; }
         public string ClientIp { get; private set; }
+        public IReadOnlyDictionary<string, string>? BlockingMetadata { get; private set; }
         public List<EDNSLog> EDNS { get; private set; }
         public DnsTransportProtocol Protocol { get; private set; }
         public DnsQuestion? Question { get; private set; }

Apps/LogExporterApp/Strategy/SyslogExportStrategy.cs

@@ -164,6 +164,15 @@ private static LogEvent Convert(LogEntry log)
                 }
             }
 
+            // Add blocking metadata
+            if ((log.BlockingMetadata is not null) && (log.BlockingMetadata.Count > 0))
+            {
+                foreach (KeyValuePair<string, string> item in log.BlockingMetadata)
+                    properties.Add(new LogEventProperty("blocking_" + item.Key, new ScalarValue(item.Value)));
+
+                properties.Add(new LogEventProperty("blockingMetadataSummary", new ScalarValue(string.Join(", ", log.BlockingMetadata.Select(kv => kv.Key + "=" + kv.Value)))));
+            }
+
             // Define the message template to match the original summary format
             const string templateText = "{questionsSummary}; RCODE: {rCode}; ANSWER: [{answersSummary}]";
 

Apps/QueryLogsMySqlApp/App.cs

@@ -34,7 +34,7 @@ You should have received a copy of the GNU General Public License
 
 namespace QueryLogsMySql
 {
-    public sealed class App : IDnsApplication, IDnsQueryLogger, IDnsQueryLogs
+    public sealed class App : IDnsApplication, IDnsQueryLoggerEx, IDnsQueryLogs
     {
         #region variables
 
@@ -236,14 +236,14 @@ private async Task BulkInsertLogsAsync(List<LogEntry> logs, StringBuilder sb)
                     await using (MySqlCommand command = connection.CreateCommand())
                     {
                         sb.Length = 0;
-                        sb.Append("INSERT INTO dns_logs (server, timestamp, client_ip, protocol, response_type, response_rtt, rcode, qname, qtype, qclass, answer) VALUES ");
+                        sb.Append("INSERT INTO dns_logs (server, timestamp, client_ip, protocol, response_type, response_rtt, rcode, qname, qtype, qclass, answer, blocking_metadata) VALUES ");
 
                         for (int i = 0; i < logs.Count; i++)
                         {
                             if (i == 0)
-                                sb.Append($"(@server{i}, @timestamp{i}, @client_ip{i}, @protocol{i}, @response_type{i}, @response_rtt{i}, @rcode{i}, @qname{i}, @qtype{i}, @qclass{i}, @answer{i})");
+                                sb.Append($"(@server{i}, @timestamp{i}, @client_ip{i}, @protocol{i}, @response_type{i}, @response_rtt{i}, @rcode{i}, @qname{i}, @qtype{i}, @qclass{i}, @answer{i}, @blocking_metadata{i})");
                             else
-                                sb.Append($", (@server{i}, @timestamp{i}, @client_ip{i}, @protocol{i}, @response_type{i}, @response_rtt{i}, @rcode{i}, @qname{i}, @qtype{i}, @qclass{i}, @answer{i})");
+                                sb.Append($", (@server{i}, @timestamp{i}, @client_ip{i}, @protocol{i}, @response_type{i}, @response_rtt{i}, @rcode{i}, @qname{i}, @qtype{i}, @qclass{i}, @answer{i}, @blocking_metadata{i})");
                         }
 
                         command.CommandText = sb.ToString();
@@ -263,18 +263,14 @@ private async Task BulkInsertLogsAsync(List<LogEntry> logs, StringBuilder sb)
                             MySqlParameter paramQtype = command.Parameters.Add("@qtype" + i, MySqlDbType.UInt16);
                             MySqlParameter paramQclass = command.Parameters.Add("@qclass" + i, MySqlDbType.Int16);
                             MySqlParameter paramAnswer = command.Parameters.Add("@answer" + i, MySqlDbType.VarChar);
+                            MySqlParameter paramBlockingMetadata = command.Parameters.Add("@blocking_metadata" + i, MySqlDbType.Text);
 
                             paramServer.Value = _dnsServer?.ServerDomain;
                             paramTimestamp.Value = log.Timestamp;
                             paramClientIp.Value = log.RemoteEP.Address.ToString();
                             paramProtocol.Value = (byte)log.Protocol;
 
-                            DnsServerResponseType responseType;
-
-                            if (log.Response.Tag == null)
-                                responseType = DnsServerResponseType.Recursive;
-                            else
-                                responseType = (DnsServerResponseType)log.Response.Tag;
+                            DnsServerResponseType responseType = DnsServerResponseTag.GetResponseType(log.Response.Tag);
 
                             paramResponseType.Value = (byte)responseType;
 
@@ -328,6 +324,11 @@ private async Task BulkInsertLogsAsync(List<LogEntry> logs, StringBuilder sb)
 
                                 paramAnswer.Value = answer;
                             }
+
+                            if (log.Metadata is null)
+                                paramBlockingMetadata.Value = DBNull.Value;
+                            else
+                                paramBlockingMetadata.Value = JsonSerializer.Serialize(log.Metadata.Values);
                         }
 
                         await command.ExecuteNonQueryAsync();
@@ -402,7 +403,8 @@ client_ip VARCHAR(39) NOT NULL,
     qname VARCHAR(255),
     qtype SMALLINT UNSIGNED,
     qclass SMALLINT,
-    answer VARCHAR(4000)
+    answer VARCHAR(4000),
+    blocking_metadata TEXT
 );
 ";
 
@@ -445,6 +447,18 @@ answer VARCHAR(4000)
                                 { }
                             }
 
+                            await using (MySqlCommand command = connection.CreateCommand())
+                            {
+                                command.CommandText = "ALTER TABLE dns_logs ADD blocking_metadata TEXT;";
+
+                                try
+                                {
+                                    await command.ExecuteNonQueryAsync();
+                                }
+                                catch
+                                { }
+                            }
+
                             await using (MySqlCommand command = connection.CreateCommand())
                             {
                                 command.CommandText = "CREATE INDEX index_server ON dns_logs (server);";
@@ -705,9 +719,14 @@ answer VARCHAR(4000)
         }
 
         public Task InsertLogAsync(DateTime timestamp, DnsDatagram request, IPEndPoint remoteEP, DnsTransportProtocol protocol, DnsDatagram response)
+        {
+            return InsertLogAsync(timestamp, request, remoteEP, protocol, response, null);
+        }
+
+        public Task InsertLogAsync(DateTime timestamp, DnsDatagram request, IPEndPoint remoteEP, DnsTransportProtocol protocol, DnsDatagram response, DnsQueryLogMetadata? metadata)
         {
             if (_enableLogging)
-                _channelWriter?.TryWrite(new LogEntry(timestamp, request, remoteEP, protocol, response));
+                _channelWriter?.TryWrite(new LogEntry(timestamp, request, remoteEP, protocol, response, metadata));
 
             return Task.CompletedTask;
         }
@@ -924,18 +943,20 @@ readonly struct LogEntry
             public readonly IPEndPoint RemoteEP;
             public readonly DnsTransportProtocol Protocol;
             public readonly DnsDatagram Response;
+            public readonly DnsQueryLogMetadata? Metadata;
 
             #endregion
 
             #region constructor
 
-            public LogEntry(DateTime timestamp, DnsDatagram request, IPEndPoint remoteEP, DnsTransportProtocol protocol, DnsDatagram response)
+            public LogEntry(DateTime timestamp, DnsDatagram request, IPEndPoint remoteEP, DnsTransportProtocol protocol, DnsDatagram response, DnsQueryLogMetadata? metadata = null)
             {
                 Timestamp = timestamp;
                 Request = request;
                 RemoteEP = remoteEP;
                 Protocol = protocol;
                 Response = response;
+                Metadata = metadata ?? DnsServerResponseTag.GetLogMetadata(response.Tag);
             }
 
             #endregion

Apps/QueryLogsPostgreSqlApp/App.cs

@@ -269,12 +269,7 @@ private async Task BulkInsertLogsAsync(List<LogEntry> logs, StringBuilder sb)
                             paramClientIp.Value = log.RemoteEP.Address.ToString();
                             paramProtocol.Value = (byte)log.Protocol;
 
-                            DnsServerResponseType responseType;
-
-                            if (log.Response.Tag == null)
-                                responseType = DnsServerResponseType.Recursive;
-                            else
-                                responseType = (DnsServerResponseType)log.Response.Tag;
+                            DnsServerResponseType responseType = DnsServerResponseTag.GetResponseType(log.Response.Tag);
 
                             paramResponseType.Value = (byte)responseType;