From 30f28c2c3765fcdfa0983119d029eac6b7cbecdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=98=BF=E6=96=8C?= <1142274323@qq.com> Date: Sat, 20 Jun 2026 16:25:18 +0800 Subject: [PATCH] Add DNS history fields and prompts --- README.md | 17 +++ dooked/include/cli_preprocessor.hpp | 7 ++ dooked/include/utils/io_utils.hpp | 162 +++++++++++++++++++++++++++- dooked/source/cli_preprocessor.cpp | 157 ++++++++++++++++++++++++++- dooked/source/main.cpp | 6 ++ 5 files changed, 342 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index f1a761c..28436fd 100644 --- a/README.md +++ b/README.md @@ -39,3 +39,20 @@ make ## Usage For comprehensive help, use `dooked --help` + +### History fields + +When a previous JSON result is used as input, dooked now preserves DNS records +that are no longer visible in the current run and stores: + +- `first-seen`: the first date the DNS record was observed +- `last-seen`: the most recent date the DNS record was observed +- `seen`: how many runs have observed the record + +Additional history prompts are available: + +- `--fs`: print records first seen in the current run +- `--ls `: print records not seen in the current run and not seen within + the requested number of days +- `--lsd `: print records not seen in the current run and not seen + since the requested date diff --git a/dooked/include/cli_preprocessor.hpp b/dooked/include/cli_preprocessor.hpp index 43fa1ba..102dd94 100644 --- a/dooked/include/cli_preprocessor.hpp +++ b/dooked/include/cli_preprocessor.hpp @@ -25,6 +25,9 @@ struct cli_args_t { int thread_count{}; int content_length{-1}; bool include_date{false}; + bool show_first_seen{false}; + int last_seen_days{-1}; + std::string last_seen_date{}; }; struct runtime_args_t { @@ -36,6 +39,10 @@ struct runtime_args_t { http_process_e http_request_time_{}; int thread_count{}; int content_length{-1}; + bool show_first_seen{false}; + int last_seen_days{-1}; + std::string last_seen_date{}; + std::string scan_date{}; }; void run_program(cli_args_t const &cli_args); diff --git a/dooked/include/utils/io_utils.hpp b/dooked/include/utils/io_utils.hpp index 829b09e..6a562d4 100644 --- a/dooked/include/utils/io_utils.hpp +++ b/dooked/include/utils/io_utils.hpp @@ -2,12 +2,18 @@ #include "utils/containers.hpp" #include "utils/probe_result.hpp" +#include +#include #include #include #include +#include #include #include +#include #include +#include +#include namespace dooked { @@ -30,6 +36,9 @@ struct json_data_t { int http_code{}; int content_length{}; dns_record_type_e type{}; + std::string first_seen{}; + std::string last_seen{}; + int seen{}; static json_data_t serialize(std::string const &d, int const len, int const http_code, @@ -42,6 +51,17 @@ struct json_data_t { data.ttl = json_object["ttl"].get(); data.content_length = len; data.http_code = http_code; + if (auto const iter = json_object.find("first-seen"); + iter != json_object.end()) { + data.first_seen = iter->second.get(); + } + if (auto const iter = json_object.find("last-seen"); + iter != json_object.end()) { + data.last_seen = iter->second.get(); + } + if (auto const iter = json_object.find("seen"); iter != json_object.end()) { + data.seen = iter->second.get(); + } return data; } }; @@ -54,10 +74,71 @@ struct jd_domain_comparator_t { namespace detail { +inline std::string lower_copy(std::string value) { + std::transform(value.begin(), value.end(), value.begin(), + [](unsigned char ch) { return (char)std::tolower(ch); }); + return value; +} + +inline std::string history_key(std::string const &domain_name, + dns_record_type_e const type, + std::string const &rdata) { + return lower_copy(domain_name) + "\n" + std::to_string((int)type) + "\n" + + lower_copy(rdata); +} + +inline std::string history_key(json_data_t const &record) { + return history_key(record.domain_name, record.type, record.rdata); +} + +inline std::string history_key(std::string const &domain_name, + probe_result_t const &record) { + return history_key(domain_name, record.type, record.rdata); +} + +inline json::object_t json_with_history(probe_result_t const &record, + std::string const &first_seen, + std::string const &last_seen, + int const seen) { + json record_json = record; + auto record_object = record_json.get(); + record_object["first-seen"] = first_seen; + record_object["last-seen"] = last_seen; + record_object["seen"] = seen; + return record_object; +} + +inline json::object_t json_from_history(json_data_t const &record, + std::string const &scan_date) { + auto first_seen = record.first_seen; + auto last_seen = record.last_seen; + auto seen = record.seen; + if (first_seen.empty()) { + first_seen = last_seen.empty() ? scan_date : last_seen; + } + if (last_seen.empty()) { + last_seen = first_seen.empty() ? scan_date : first_seen; + } + if (seen <= 0) { + seen = 1; + } + + json::object_t object; + object["ttl"] = record.ttl; + object["type"] = dns_record_type_to_str(record.type); + object["info"] = record.rdata; + object["first-seen"] = std::move(first_seen); + object["last-seen"] = std::move(last_seen); + object["seen"] = seen; + return object; +} + template void write_json_result_impl(map_container_t const &result_map, RtType const &rt_args) { - if (result_map.empty()) { + bool const has_previous = + rt_args.previous_data && !rt_args.previous_data->empty(); + if (result_map.empty() && !has_previous) { std::error_code ec{}; if (std::filesystem::exists(rt_args.output_filename) && !std::filesystem::remove(rt_args.output_filename, ec)) { @@ -66,11 +147,53 @@ void write_json_result_impl(map_container_t const &result_map, return; } + std::map previous_lookup; + std::map> previous_by_domain; + if (has_previous) { + for (auto const &record : *rt_args.previous_data) { + previous_lookup[history_key(record)] = record; + previous_by_domain[record.domain_name].push_back(record); + } + } + + auto const scan_date = + rt_args.scan_date.empty() ? std::string{"unknown"} : rt_args.scan_date; + json::array_t list; + std::set emitted_keys; for (auto const &result_pair : result_map.cresult()) { json::object_t internal_object; auto &http_result = result_pair.second.http_result_; - internal_object["dns_probe"] = result_pair.second.dns_result_list_; + json::array_t dns_probe_list; + for (auto const &record : result_pair.second.dns_result_list_) { + auto const key = history_key(result_pair.first, record); + emitted_keys.insert(key); + auto first_seen = scan_date; + auto seen = 1; + if (auto const iter = previous_lookup.find(key); + iter != previous_lookup.end()) { + first_seen = iter->second.first_seen.empty() + ? (iter->second.last_seen.empty() + ? scan_date + : iter->second.last_seen) + : iter->second.first_seen; + seen = (std::max)(iter->second.seen, 0) + 1; + } + dns_probe_list.push_back( + json_with_history(record, first_seen, scan_date, seen)); + } + if (auto const iter = previous_by_domain.find(result_pair.first); + iter != previous_by_domain.end()) { + for (auto const &record : iter->second) { + auto const key = history_key(record); + if (emitted_keys.find(key) == emitted_keys.end()) { + emitted_keys.insert(key); + dns_probe_list.push_back(json_from_history(record, scan_date)); + } + } + } + + internal_object["dns_probe"] = std::move(dns_probe_list); internal_object["content_length"] = http_result.content_length_; internal_object["http_code"] = http_result.http_status_; internal_object["code_string"] = code_string(http_result.http_status_); @@ -79,6 +202,41 @@ void write_json_result_impl(map_container_t const &result_map, object[result_pair.first] = internal_object; list.push_back(std::move(object)); } + if (has_previous) { + for (auto const &result_pair : previous_by_domain) { + auto const current_iter = result_map.cresult().find(result_pair.first); + if (current_iter != result_map.cresult().end()) { + continue; + } + + json::array_t dns_probe_list; + auto content_length = 0; + auto http_code = 0; + for (auto const &record : result_pair.second) { + auto const key = history_key(record); + if (emitted_keys.find(key) != emitted_keys.end()) { + continue; + } + emitted_keys.insert(key); + content_length = record.content_length; + http_code = record.http_code; + dns_probe_list.push_back(json_from_history(record, scan_date)); + } + if (dns_probe_list.empty()) { + continue; + } + + json::object_t internal_object; + internal_object["dns_probe"] = std::move(dns_probe_list); + internal_object["content_length"] = content_length; + internal_object["http_code"] = http_code; + internal_object["code_string"] = code_string(http_code); + + json::object_t object; + object[result_pair.first] = std::move(internal_object); + list.push_back(std::move(object)); + } + } json::object_t res_object; res_object["program"] = "dooked"; diff --git a/dooked/source/cli_preprocessor.cpp b/dooked/source/cli_preprocessor.cpp index c08d7fb..256767a 100644 --- a/dooked/source/cli_preprocessor.cpp +++ b/dooked/source/cli_preprocessor.cpp @@ -4,9 +4,15 @@ #include "utils/exceptions.hpp" #include "utils/random_utils.hpp" #include "utils/string_utils.hpp" +#include #include #include +#include +#include +#include +#include #include +#include #include // defined (and assigned to) in main.cpp @@ -18,6 +24,140 @@ namespace dooked { namespace net = boost::asio; using namespace fmt::v7::literals; +std::string lower_copy(std::string value) { + std::transform(value.begin(), value.end(), value.begin(), + [](unsigned char ch) { return (char)std::tolower(ch); }); + return value; +} + +std::string history_key(std::string const &domain_name, + dns_record_type_e const type, + std::string const &rdata) { + return lower_copy(domain_name) + "\n" + std::to_string((int)type) + "\n" + + lower_copy(rdata); +} + +std::string history_key(json_data_t const &record) { + return history_key(record.domain_name, record.type, record.rdata); +} + +std::string history_key(std::string const &domain_name, + probe_result_t const &record) { + return history_key(domain_name, record.type, record.rdata); +} + +std::map +build_history_lookup(std::optional> const &history) { + std::map lookup; + if (!history) { + return lookup; + } + for (auto const &record : *history) { + lookup[history_key(record)] = record; + } + return lookup; +} + +std::set +build_current_keys(map_container_t const ¤t_result) { + std::set keys; + for (auto const &result_pair : current_result.cresult()) { + for (auto const &record : result_pair.second.dns_result_list_) { + keys.insert(history_key(result_pair.first, record)); + } + } + return keys; +} + +bool parse_date(std::string const &input, char const *format, + std::time_t &output) { + std::tm tm{}; + std::istringstream ss(input); + ss >> std::get_time(&tm, format); + if (ss.fail()) { + return false; + } + tm.tm_hour = 0; + tm.tm_min = 0; + tm.tm_sec = 0; + output = std::mktime(&tm); + return output != (std::time_t)-1; +} + +bool parse_date(std::string const &input, std::time_t &output) { + return parse_date(input, "%Y-%m-%d", output) || + parse_date(input, "%m/%d/%Y", output) || + parse_date(input, "%m-%d-%Y", output); +} + +bool older_than_days(std::string const &last_seen, int const days, + std::time_t const now) { + if (days < 0) { + return false; + } + std::time_t last_seen_time{}; + if (!parse_date(last_seen, last_seen_time)) { + return true; + } + double const seconds = std::difftime(now, last_seen_time); + return seconds >= static_cast(days) * 24 * 60 * 60; +} + +bool older_than_date(std::string const &last_seen, std::string const &date) { + if (date.empty()) { + return false; + } + std::time_t last_seen_time{}; + std::time_t cutoff_time{}; + if (!parse_date(last_seen, last_seen_time) || !parse_date(date, cutoff_time)) { + return true; + } + return last_seen_time < cutoff_time; +} + +void report_history_flags(runtime_args_t const &rt_args, + map_container_t const ¤t_result) { + auto const previous_lookup = build_history_lookup(rt_args.previous_data); + + if (rt_args.show_first_seen) { + for (auto const &result_pair : current_result.cresult()) { + for (auto const &record : result_pair.second.dns_result_list_) { + auto const key = history_key(result_pair.first, record); + if (previous_lookup.find(key) == previous_lookup.end()) { + spdlog::info("[FIRST-SEEN][{}][{}] `{}`", result_pair.first, + dns_record_type_to_str(record.type), record.rdata); + } + } + } + } + + if (rt_args.last_seen_days < 0 && rt_args.last_seen_date.empty()) { + return; + } + + if (!rt_args.previous_data) { + return; + } + + auto const current_keys = build_current_keys(current_result); + auto const now = std::time(nullptr); + for (auto const &record : *rt_args.previous_data) { + if (current_keys.find(history_key(record)) != current_keys.end()) { + continue; + } + bool const older_than_requested_days = + older_than_days(record.last_seen, rt_args.last_seen_days, now); + bool const older_than_requested_date = + older_than_date(record.last_seen, rt_args.last_seen_date); + if (older_than_requested_days || older_than_requested_date) { + spdlog::info("[LAST-SEEN][{}][{}] `{}` last seen `{}`", + record.domain_name, dns_record_type_to_str(record.type), + record.rdata, + record.last_seen.empty() ? "unknown" : record.last_seen); + } + } +} + void compare_http_result(int const base_cl, json_data_t const &prev_http_result, http_response_t const ¤t_result) { auto const current_req_cl = current_result.content_length_; @@ -354,6 +494,7 @@ void start_name_checking(runtime_args_t &&rt_args) { spdlog::info("Writing JSON output"); } write_json_result(result_map, rt_args); + report_history_flags(rt_args, result_map); // compare old with new result -- only if we had previous record if (rt_args.previous_data) { @@ -477,6 +618,12 @@ void run_program(cli_args_t const &cli_args) { static_cast(cli_args.post_http_request); rt_args.thread_count = cli_args.thread_count; rt_args.content_length = cli_args.content_length; + rt_args.show_first_seen = cli_args.show_first_seen; + rt_args.last_seen_days = cli_args.last_seen_days; + rt_args.last_seen_date = cli_args.last_seen_date; + if (!timet_to_string(rt_args.scan_date, std::time(nullptr), "%Y-%m-%d")) { + rt_args.scan_date = {}; + } return start_name_checking(std::move(rt_args)); } @@ -492,14 +639,14 @@ void report_error(char const *format, int const value, bool const boolean_value, void print_banner() { auto const header = R"sep(" -·▄▄▄▄ ▄ •▄ ▄▄▄ .·▄▄▄▄ -██▪ ██ ▪ ▪ █▌▄▌▪▀▄.▀·██▪ ██ +·▄▄▄▄ ▄ •▄ ▄▄▄ .·▄▄▄▄ +██▪ ██ ▪ ▪ █▌▄▌▪▀▄.▀·██▪ ██ ▐█· ▐█▌ ▄█▀▄ ▄█▀▄ ▐▀▀▄·▐▀▀▪▄▐█· ▐█▌ -██. ██ ▐█▌.▐▌▐█▌.▐▌▐█.█▌▐█▄▄▌██. ██ -▀▀▀▀▀• ▀█▄▀▪ ▀█▄▀▪·▀ ▀ ▀▀▀ ▀▀▀▀▀• +██. ██ ▐█▌.▐▌▐█▌.▐▌▐█.█▌▐█▄▄▌██. ██ +▀▀▀▀▀• ▀█▄▀▪ ▀█▄▀▪·▀ ▀ ▀▀▀ ▀▀▀▀▀• DNS and Target History Local Storage -Made with ❥ by codingo (https://twitter.com/codingo_) +Made with ❥ by codingo (https://twitter.com/codingo_) )sep"; fprintf(stdout, "%s", header); } diff --git a/dooked/source/main.cpp b/dooked/source/main.cpp index cf29460..1bbfe51 100644 --- a/dooked/source/main.cpp +++ b/dooked/source/main.cpp @@ -41,6 +41,12 @@ int main(int argc, char **argv) { "defers http request until after all DNS requests have been completed"); app.add_flag("--compare-cl", compare_cl, "compare content-length of HTTP requests"); + app.add_flag("--fs", cli_args.show_first_seen, + "show DNS records first seen in this run"); + app.add_option("--ls", cli_args.last_seen_days, + "show DNS records not seen for at least N days"); + app.add_option("--lsd", cli_args.last_seen_date, + "show DNS records not seen since MM/DD/YYYY"); app.add_flag("--nbc", no_bytes_count, "in case `content-length` is missing in an HTTP header field,"