diff --git a/api/debuggerapi.h b/api/debuggerapi.h index 0ff27f12..63d59351 100644 --- a/api/debuggerapi.h +++ b/api/debuggerapi.h @@ -983,4 +983,70 @@ namespace BinaryNinjaDebuggerAPI { std::string GetWinDbgInstalledVersion(const std::string& installPath = ""); std::string GetWinDbgLatestVersion(); + // A memory-mapped report of the Windows API calls extracted from a TTD trace. + // + // Thin wrapper over the core reader: opening is a header validation, and rows are + // decoded out of the mapping only when asked for. RunQuery filters the whole report + // on the core side and hands back matching row indices in one crossing, which is + // what keeps filtering millions of calls affordable through the FFI. + struct TTDApiCallParam + { + std::string name; + std::string type; + std::string kind; + uint64_t value = 0; + std::string str; + std::vector flags; + std::vector bytes; + uint64_t bytesTotal = 0; // the buffer's real length when `bytes` was capped + uint64_t deref = 0; + bool hasDeref = false; + bool out = false; + bool atReturn = false; + }; + + struct TTDApiCall + { + uint64_t seq = 0; + uint64_t tid = 0; + uint64_t positionSequence = 0; + uint64_t positionSteps = 0; + std::string module; + std::string api; + uint64_t ret = 0; + uint64_t returnAddress = 0; + std::string paramSummary; + bool decoded = false; + std::vector params; + }; + + class TTDBehaviorReport + { + BNTTDBehaviorReport* m_object = nullptr; + + public: + TTDBehaviorReport() = default; + ~TTDBehaviorReport(); + TTDBehaviorReport(const TTDBehaviorReport&) = delete; + TTDBehaviorReport& operator=(const TTDBehaviorReport&) = delete; + + bool Open(const std::string& path, std::string& error); + void Close(); + bool IsOpen() const { return m_object != nullptr; } + + uint64_t GetCallCount() const; + uint64_t GetDecodedCount() const; + uint64_t GetProcessId() const; + uint64_t GetMaxSequence() const; + uint32_t GetMaxPositionChars() const; + std::string GetTracePath() const; + std::string GetArchitecture() const; + + // Row indices of every call matching `query`. + std::vector RunQuery(const std::string& query) const; + + // `withParams` is the expensive half, so a table can skip it. + bool GetCall(uint64_t index, TTDApiCall& out, bool withParams) const; + }; + }; // namespace BinaryNinjaDebuggerAPI diff --git a/api/ffi.h b/api/ffi.h index 61bfc333..06a52ffc 100644 --- a/api/ffi.h +++ b/api/ffi.h @@ -879,6 +879,73 @@ extern "C" DEBUGGER_FFI_API char* BNDebuggerGetWinDbgInstalledVersion(const char* installPath); DEBUGGER_FFI_API char* BNDebuggerGetWinDbgLatestVersion(void); + // TTD behavior reports: the Windows API calls extracted from a TTD trace. + // + // The boundary is drawn so that no loop over the call list crosses it. + // BNTTDBehaviorRunQuery evaluates a whole filter core-side and returns the matching + // row indices in one call, which measures within noise of filtering in-process; + // crossing per row instead costs 30-50%. Callers then fetch only the rows they show. + typedef struct BNTTDBehaviorReport BNTTDBehaviorReport; + + typedef struct BNTTDApiCallParam + { + char* m_name; + char* m_type; + char* m_kind; + uint64_t m_value; + char* m_str; // empty when the parameter is not a string + char** m_flags; // symbolic names for enum/flag parameters + size_t m_flagCount; + uint8_t* m_bytes; // captured buffer contents, possibly only a prefix + size_t m_byteCount; + uint64_t m_bytesTotal; // the buffer's real length when capped, else 0 + uint64_t m_deref; + bool m_hasDeref; + bool m_out; + bool m_atReturn; // re-read at the call's return position + } BNTTDApiCallParam; + + typedef struct BNTTDApiCall + { + uint64_t m_seq; + uint64_t m_tid; + uint64_t m_positionSequence; + uint64_t m_positionSteps; + char* m_module; + char* m_api; + uint64_t m_ret; + uint64_t m_returnAddress; // the instruction after the CALL, i.e. the call site + char* m_paramSummary; + bool m_decoded; + BNTTDApiCallParam* m_params; + size_t m_paramCount; + } BNTTDApiCall; + + // Returns NULL on failure, with *errorMessage set to a string the caller frees with + // BNDebuggerFreeString. + DEBUGGER_FFI_API BNTTDBehaviorReport* BNTTDBehaviorOpenReport(const char* path, char** errorMessage); + DEBUGGER_FFI_API void BNTTDBehaviorCloseReport(BNTTDBehaviorReport* report); + + DEBUGGER_FFI_API uint64_t BNTTDBehaviorGetCallCount(BNTTDBehaviorReport* report); + DEBUGGER_FFI_API uint64_t BNTTDBehaviorGetDecodedCount(BNTTDBehaviorReport* report); + DEBUGGER_FFI_API uint64_t BNTTDBehaviorGetProcessId(BNTTDBehaviorReport* report); + DEBUGGER_FFI_API uint64_t BNTTDBehaviorGetMaxSequence(BNTTDBehaviorReport* report); + DEBUGGER_FFI_API uint32_t BNTTDBehaviorGetMaxPositionChars(BNTTDBehaviorReport* report); + // Both must be freed with BNDebuggerFreeString. + DEBUGGER_FFI_API char* BNTTDBehaviorGetTracePath(BNTTDBehaviorReport* report); + DEBUGGER_FFI_API char* BNTTDBehaviorGetArchitecture(BNTTDBehaviorReport* report); + + // Row indices of every call matching `query`. Free with BNTTDBehaviorFreeQueryResult. + DEBUGGER_FFI_API uint64_t* BNTTDBehaviorRunQuery( + BNTTDBehaviorReport* report, const char* query, size_t* count); + DEBUGGER_FFI_API void BNTTDBehaviorFreeQueryResult(uint64_t* result); + + // Fills `out`, which the caller frees with BNTTDBehaviorFreeCall. Decoding the + // parameters is the expensive half, so a table can pass false and a detail view true. + DEBUGGER_FFI_API bool BNTTDBehaviorGetCall( + BNTTDBehaviorReport* report, uint64_t index, bool withParams, BNTTDApiCall* out); + DEBUGGER_FFI_API void BNTTDBehaviorFreeCall(BNTTDApiCall* call); + #ifdef __cplusplus } #endif diff --git a/api/python/__init__.py b/api/python/__init__.py index f60836d7..a963e9a0 100644 --- a/api/python/__init__.py +++ b/api/python/__init__.py @@ -25,8 +25,10 @@ from .debuggercontroller import * from .debugadaptertype import * from .debugger_enums import * + from .ttdbehavior import * else: if Settings().get_bool('corePlugins.debugger') and (os.environ.get('BN_DISABLE_CORE_DEBUGGER') is None): from .debuggercontroller import * from .debugadaptertype import * from .debugger_enums import * + from .ttdbehavior import * diff --git a/api/python/ttdbehavior.py b/api/python/ttdbehavior.py new file mode 100644 index 00000000..5ba92965 --- /dev/null +++ b/api/python/ttdbehavior.py @@ -0,0 +1,487 @@ +# Copyright 2020-2026 Vector 35 Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Query the Windows API calls extracted from a TTD trace. + +A report is produced by the external extractor (``ttdcapa-extract -b out.ttdb``) and is +memory-mapped rather than parsed, so opening one is effectively free no matter how many +calls it holds. + + >>> from debugger.ttdbehavior import TTDBehaviorReport + >>> with TTDBehaviorReport("NetacLockFile01.ttdb") as report: + ... print(report.call_count) + ... for call in report.query("module:kernel32 api:WriteFile ret:!0"): + ... print(call.position, call.api, call.param_summary) + +``query`` filters entirely inside the core, returning the matching row indices in a +single crossing of the FFI; rows are then decoded only as you touch them. Iterating a +whole multi-million-call report one call at a time from Python is possible but is the +slow way round -- narrow with a query first. + +Calls come back with their parameters decoded. On a query matching millions of rows that +is worth turning off with ``with_params=False``, which leaves ``param_summary`` but skips +building the structured list. + +``extract`` produces a report from a trace, which requires Windows -- see its docstring. +""" + +import ctypes +import os +import subprocess +from dataclasses import dataclass, field +from typing import Callable, Iterator, List, Optional, Sequence + +import binaryninja +from binaryninja.settings import Settings + +from . import _debuggercore as dbgcore + +__all__ = [ + "TTDBehaviorReport", + "TTDApiCall", + "TTDApiCallParam", + "extract", + "extractor_path", + "ttd_replay_directory", +] + + +@dataclass +class TTDApiCallParam: + """One decoded parameter. Which fields are meaningful depends on what the Win32 + metadata knew about the parameter.""" + + name: str = "" + type: str = "" + kind: str = "" + value: int = 0 + string: str = "" + flags: List[str] = field(default_factory=list) + data: bytes = b"" + #: The buffer's real length when ``data`` is only a prefix, else 0. + data_total: int = 0 + deref: Optional[int] = None + out: bool = False + #: The value was re-read at the call's return position, which is what makes an + #: [Out] parameter renderable at all. + at_return: bool = False + + @property + def truncated(self) -> bool: + return self.data_total > len(self.data) + + +@dataclass +class TTDApiCall: + """One Windows API call observed during the trace.""" + + index: int = 0 + tid: int = 0 + position_sequence: int = 0 + position_steps: int = 0 + module: str = "" + api: str = "" + ret: int = 0 + #: The instruction after the CALL, i.e. the call site -- which tells you whether the + #: sample made this call itself or something made it on the sample's behalf. + return_address: int = 0 + param_summary: str = "" + #: True when a real signature backed the parameters, rather than the four-register + #: heuristic fallback. + decoded: bool = False + params: List[TTDApiCallParam] = field(default_factory=list) + + @property + def position(self) -> str: + """The TTD position, in the ``Sequence:Steps`` form WinDbg and the debugger use.""" + return "%X:%X" % (self.position_sequence, self.position_steps) + + def __repr__(self) -> str: + return " 0x%x>" % ( + self.position, self.module, self.api, self.param_summary, self.ret) + + +class TTDBehaviorReport: + """A memory-mapped report of the API calls in a TTD trace.""" + + def __init__(self, path: str): + error = ctypes.c_char_p() + handle = dbgcore.BNTTDBehaviorOpenReport(path, ctypes.byref(error)) + if not handle: + message = dbgcore.pyNativeStr(error.value) if error.value else "unknown error" + if error.value: + dbgcore.BNDebuggerFreeString(error) + raise IOError("cannot open %s: %s" % (path, message)) + self.handle = handle + self.path = path + + def __del__(self): + self.close() + + def __enter__(self) -> "TTDBehaviorReport": + return self + + def __exit__(self, exc_type, exc_value, traceback) -> None: + self.close() + + def close(self) -> None: + handle = getattr(self, "handle", None) + if handle: + dbgcore.BNTTDBehaviorCloseReport(handle) + self.handle = None + + def __len__(self) -> int: + return self.call_count + + def __getitem__(self, index: int) -> TTDApiCall: + call = self.get_call(index) + if call is None: + raise IndexError(index) + return call + + def __iter__(self) -> Iterator[TTDApiCall]: + for index in range(self.call_count): + call = self.get_call(index) + if call is not None: + yield call + + def __repr__(self) -> str: + return "" % (self.path, self.call_count, self.architecture) + + @property + def call_count(self) -> int: + return dbgcore.BNTTDBehaviorGetCallCount(self.handle) + + @property + def decoded_count(self) -> int: + """Calls whose parameters came from a real signature rather than the heuristic.""" + return dbgcore.BNTTDBehaviorGetDecodedCount(self.handle) + + @property + def pid(self) -> int: + return dbgcore.BNTTDBehaviorGetProcessId(self.handle) + + @property + def trace_path(self) -> str: + return dbgcore.BNTTDBehaviorGetTracePath(self.handle) or "" + + @property + def architecture(self) -> str: + """``"x86"`` or ``"x64"``, from the traced process rather than the host.""" + return dbgcore.BNTTDBehaviorGetArchitecture(self.handle) or "" + + def query_indices(self, query: str) -> List[int]: + """Row indices matching ``query``, evaluated in one crossing of the FFI. + + The filter runs entirely in the core, so this costs about what filtering in + native code costs -- the boundary is not in the loop. + """ + count = ctypes.c_ulonglong() + result = dbgcore.BNTTDBehaviorRunQuery(self.handle, query, ctypes.byref(count)) + if not result: + return [] + try: + return [int(result[i]) for i in range(count.value)] + finally: + dbgcore.BNTTDBehaviorFreeQueryResult(result) + + def query(self, query: str, with_params: bool = True) -> Iterator[TTDApiCall]: + """Calls matching ``query``, decoded lazily as you iterate. + + Terms are ANDed; a bare word matches anywhere including buffer contents, and + ``field:value`` narrows:: + + module:kernel32 api:WriteFile only kernel32's, not ntdll's + api:Reg* ret:!0 registry calls that failed + retaddr:0x400000-0x500000 calls the sample made itself + tid:4 "c:\\\\windows" thread 4, mentioning that path + + Numeric fields (``tid``, ``ret``, ``retaddr``) take ``!=``, ``>``, ``>=``, + ``<``, ``<=`` and ``a-b`` ranges, decimal or ``0x`` hex. + + Each call arrives with its full parameter list. Pass ``with_params=False`` to skip + that when you only need ``param_summary`` -- it is about 4x cheaper per call and + holds far less memory, which starts to matter once a query matches millions of + rows rather than hundreds. + """ + for index in self.query_indices(query): + call = self.get_call(index, with_params) + if call is not None: + yield call + + def get_call(self, index: int, with_params: bool = True) -> Optional[TTDApiCall]: + raw = dbgcore.BNTTDApiCall() + if not dbgcore.BNTTDBehaviorGetCall(self.handle, index, with_params, ctypes.byref(raw)): + return None + try: + call = TTDApiCall( + index=raw.m_seq, + tid=raw.m_tid, + position_sequence=raw.m_positionSequence, + position_steps=raw.m_positionSteps, + module=raw.m_module, + api=raw.m_api, + ret=raw.m_ret, + return_address=raw.m_returnAddress, + param_summary=raw.m_paramSummary, + decoded=raw.m_decoded, + ) + for i in range(raw.m_paramCount): + call.params.append(self._convert_param(raw.m_params[i])) + return call + finally: + dbgcore.BNTTDBehaviorFreeCall(ctypes.byref(raw)) + + @staticmethod + def _convert_param(raw) -> TTDApiCallParam: + param = TTDApiCallParam( + name=raw.m_name, + type=raw.m_type, + kind=raw.m_kind, + value=raw.m_value, + string=raw.m_str, + data_total=raw.m_bytesTotal, + deref=raw.m_deref if raw.m_hasDeref else None, + out=raw.m_out, + at_return=raw.m_atReturn, + ) + for f in range(raw.m_flagCount): + param.flags.append(dbgcore.pyNativeStr(raw.m_flags[f])) + if raw.m_byteCount: + param.data = bytes(bytearray(raw.m_bytes[i] for i in range(raw.m_byteCount))) + return param + + +_EXTRACTOR_SETTING = "debugger.ttdBehaviorExtractorPath" +_MAX_BUFFER_SETTING = "debugger.ttdBehaviorMaxBuffer" + + +def _setting(key: str, getter: str): + """A setting's value, or None if the debugger did not register it. + + The TTD settings only exist on Windows, so reading them unguarded would raise on the + platforms where extraction is unavailable anyway. + """ + settings = Settings() + if not settings.contains(key): + return None + return getattr(settings, getter)(key) + + +def _plugin_root() -> str: + return ( + binaryninja.user_plugin_path() + if os.environ.get("BN_STANDALONE_DEBUGGER") is not None + else binaryninja.bundled_plugin_path() + ) + + +def extractor_path() -> Optional[str]: + """The extractor the debugger would run, or None if there is not one. + + A path configured in ``debugger.ttdBehaviorExtractorPath`` wins outright, so someone + running their own build always gets it; a configured path that is not a file raises + rather than quietly falling back, since reporting results from a different binary than + you think you are running is worse than failing. + """ + configured = _setting(_EXTRACTOR_SETTING, "get_string") + if configured: + if os.path.isfile(configured): + return configured + raise RuntimeError( + "%s is set to %r, which is not a file. Clear the setting to use the extractor " + "shipped with the debugger." % (_EXTRACTOR_SETTING, configured)) + + bundled = os.path.join(_plugin_root(), "ttd-extract", "ttdcapa-extract.exe") + return bundled if os.path.isfile(bundled) else None + + +def ttd_replay_directory() -> Optional[str]: + """Where ``TTDReplay.dll`` lives, or None if it cannot be found. + + Those DLLs are Microsoft's and ship with WinDbg rather than with us, so the extractor + is pointed at them instead of carrying copies. None is not fatal: the extractor then + falls back to the ordinary DLL search order. + """ + def has_replay(directory: Optional[str]) -> bool: + return bool(directory) and os.path.isfile(os.path.join(directory, "TTDReplay.dll")) + + # A configured DbgEng path is authoritative: if it is set and wrong, fall through to + # the extractor's own search rather than silently using a different engine. + configured = _setting("debugger.x64dbgEngPath", "get_string") + if configured: + path = os.path.join(configured, "TTD") + return path if has_replay(path) else None + + bundled = os.path.join(_plugin_root(), "dbgeng", "amd64", "TTD") + return bundled if has_replay(bundled) else None + + +def extract( + trace: str, + output: Optional[str] = None, + *, + extractor: Optional[str] = None, + ttd_dlls: Optional[str] = None, + max_buffer: Optional[int] = None, + progress: Optional[Callable[[str, float, int], bool]] = None, +) -> TTDBehaviorReport: + """Sweep a TTD trace for the Windows API calls it made and open the resulting report. + + **Windows only.** The sweep replays the trace through Microsoft's TTD engine, which + exists nowhere else. Reading a report has no such restriction, so one produced here + opens on any platform. + + >>> from debugger.ttdbehavior import extract + >>> report = extract(r"C:\\traces\\sample.run") + >>> print(report.call_count) + + The report is written beside the trace as ``.ttdb`` unless ``output`` says + otherwise. An existing file at that path is overwritten. + + ``progress`` is called as ``progress(phase, percent, calls)``, where ``phase`` is + ``"sweep"`` or ``"write"``. It fires a few times a second during the sweep, and once + when each phase begins -- writing the report is a large part of the wall clock but + reports no percentage of its own, so that transition is the only notice of it. + Returning False cancels: the extractor stops replaying and writes what it has, so a + cancelled run still yields a complete, valid report of a shorter prefix of the trace. + + >>> def show(phase, percent, calls): + ... print(f"{phase} {percent}% {calls} calls") + ... return calls < 1_000_000 # stop once we have a million + >>> report = extract("sample.run", progress=show) + + ``extractor``, ``ttd_dlls`` and ``max_buffer`` default to the same values the sidebar + uses -- respectively ``debugger.ttdBehaviorExtractorPath`` or the bundled copy, the + replay DLLs beside the configured DbgEng, and ``debugger.ttdBehaviorMaxBuffer``. + + Raises RuntimeError if no extractor can be found or the sweep fails, and IOError if + the trace does not exist. + """ + trace = os.path.abspath(trace) + if not os.path.isfile(trace): + raise IOError("no such trace: %s" % trace) + + output = os.path.abspath(output if output else os.path.splitext(trace)[0] + ".ttdb") + + if extractor is None: + extractor = extractor_path() + if extractor is None: + raise RuntimeError( + "no TTD behavior extractor found. It ships with the debugger on Windows; on " + "other platforms reports can be opened but not produced. Set %s to point at a " + "build of your own." % _EXTRACTOR_SETTING) + if not os.path.isfile(extractor): + raise RuntimeError("not a TTD behavior extractor: %s" % extractor) + + command = [extractor, trace, "-b", output, "--progress"] + + if ttd_dlls is None: + ttd_dlls = ttd_replay_directory() + if ttd_dlls: + command += ["--ttd-dlls", ttd_dlls] + + if max_buffer is None: + max_buffer = _setting(_MAX_BUFFER_SETTING, "get_integer") + if max_buffer is not None: + command += ["--max-buffer", str(int(max_buffer))] + + if progress is not None: + command.append("--cancel-on-stdin") + + process = subprocess.Popen( + command, + stdin=subprocess.PIPE if progress is not None else subprocess.DEVNULL, + stdout=subprocess.DEVNULL, + stderr=subprocess.PIPE, + text=True, + bufsize=1, + # Otherwise a script flashes up a console window on every call. + creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0), + ) + + phase = "sweep" + percent = 0.0 + calls = 0 + cancelled = False + # Only the tail is kept: a failing run can be chatty, and the last lines are the ones + # that say why. + tail: List[str] = [] + + def report_progress() -> None: + nonlocal cancelled + if progress is None or cancelled: + return + if progress(phase, percent, calls) is not False: + return + cancelled = True + try: + process.stdin.write("cancel\n") + process.stdin.flush() + except OSError: + pass # already gone; the sweep is ending anyway + + for line in process.stderr: + line = line.strip() + if not line: + continue + + if line.startswith("[phase]"): + parts = line.split(None, 1) + if len(parts) > 1: + phase = parts[1] + # Announced rather than waited on: the extractor only emits [progress] + # during the sweep, so without this a caller would never learn that + # writing had started, and writing is a large part of the wall clock. + report_progress() + continue + + if line.startswith("[progress]"): + parts = line.split() + try: + # The percentage is fractional, e.g. "[progress] 5.10 160". + percent, calls = float(parts[1]), int(parts[2]) + except (IndexError, ValueError): + continue + report_progress() + continue + + tail.append(line) + del tail[:-20] + + process.wait() + if process.stdin: + process.stdin.close() + + if process.returncode != 0: + raise RuntimeError( + "extraction failed (exit %d):\n%s" % (process.returncode, "\n".join(tail))) + + return TTDBehaviorReport(output) + + +def modules(report: TTDBehaviorReport) -> Sequence[str]: + """Every distinct module seen in the report, in first-call order. + + Convenience for the common "what did this touch?" question; walks the whole report, + so narrow with a query first if you only care about part of it. + """ + seen = [] + known = set() + for call in report: + if call.module not in known: + known.add(call.module) + seen.append(call.module) + return seen diff --git a/api/ttdbehavior.cpp b/api/ttdbehavior.cpp new file mode 100644 index 00000000..1a947b0f --- /dev/null +++ b/api/ttdbehavior.cpp @@ -0,0 +1,168 @@ +/* +Copyright 2020-2026 Vector 35 Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +#include "debuggerapi.h" + +using namespace BinaryNinjaDebuggerAPI; + +namespace { + // Every char* the FFI hands back is ours to free. + std::string TakeString(char* raw) + { + if (raw == nullptr) + return std::string(); + std::string result(raw); + BNDebuggerFreeString(raw); + return result; + } +} // namespace + + +TTDBehaviorReport::~TTDBehaviorReport() +{ + Close(); +} + + +bool TTDBehaviorReport::Open(const std::string& path, std::string& error) +{ + Close(); + char* errorMessage = nullptr; + m_object = BNTTDBehaviorOpenReport(path.c_str(), &errorMessage); + if (m_object == nullptr) + { + error = TakeString(errorMessage); + return false; + } + return true; +} + + +void TTDBehaviorReport::Close() +{ + if (m_object != nullptr) + { + BNTTDBehaviorCloseReport(m_object); + m_object = nullptr; + } +} + + +uint64_t TTDBehaviorReport::GetCallCount() const +{ + return m_object ? BNTTDBehaviorGetCallCount(m_object) : 0; +} + + +uint64_t TTDBehaviorReport::GetDecodedCount() const +{ + return m_object ? BNTTDBehaviorGetDecodedCount(m_object) : 0; +} + + +uint64_t TTDBehaviorReport::GetProcessId() const +{ + return m_object ? BNTTDBehaviorGetProcessId(m_object) : 0; +} + + +uint64_t TTDBehaviorReport::GetMaxSequence() const +{ + return m_object ? BNTTDBehaviorGetMaxSequence(m_object) : 0; +} + + +uint32_t TTDBehaviorReport::GetMaxPositionChars() const +{ + return m_object ? BNTTDBehaviorGetMaxPositionChars(m_object) : 0; +} + + +std::string TTDBehaviorReport::GetTracePath() const +{ + return m_object ? TakeString(BNTTDBehaviorGetTracePath(m_object)) : std::string(); +} + + +std::string TTDBehaviorReport::GetArchitecture() const +{ + return m_object ? TakeString(BNTTDBehaviorGetArchitecture(m_object)) : std::string(); +} + + +std::vector TTDBehaviorReport::RunQuery(const std::string& query) const +{ + std::vector result; + if (m_object == nullptr) + return result; + + size_t count = 0; + uint64_t* rows = BNTTDBehaviorRunQuery(m_object, query.c_str(), &count); + if (rows == nullptr) + return result; + result.assign(rows, rows + count); + BNTTDBehaviorFreeQueryResult(rows); + return result; +} + + +bool TTDBehaviorReport::GetCall(uint64_t index, TTDApiCall& out, bool withParams) const +{ + if (m_object == nullptr) + return false; + + BNTTDApiCall raw {}; + if (!BNTTDBehaviorGetCall(m_object, index, withParams, &raw)) + return false; + + out.seq = raw.m_seq; + out.tid = raw.m_tid; + out.positionSequence = raw.m_positionSequence; + out.positionSteps = raw.m_positionSteps; + out.module = raw.m_module ? raw.m_module : ""; + out.api = raw.m_api ? raw.m_api : ""; + out.ret = raw.m_ret; + out.returnAddress = raw.m_returnAddress; + out.paramSummary = raw.m_paramSummary ? raw.m_paramSummary : ""; + out.decoded = raw.m_decoded; + + out.params.clear(); + out.params.reserve(raw.m_paramCount); + for (size_t i = 0; i < raw.m_paramCount; ++i) + { + const BNTTDApiCallParam& src = raw.m_params[i]; + TTDApiCallParam param; + param.name = src.m_name ? src.m_name : ""; + param.type = src.m_type ? src.m_type : ""; + param.kind = src.m_kind ? src.m_kind : ""; + param.value = src.m_value; + param.str = src.m_str ? src.m_str : ""; + for (size_t f = 0; f < src.m_flagCount; ++f) + param.flags.emplace_back(src.m_flags[f] ? src.m_flags[f] : ""); + if (src.m_byteCount != 0 && src.m_bytes != nullptr) + param.bytes.assign(src.m_bytes, src.m_bytes + src.m_byteCount); + param.bytesTotal = src.m_bytesTotal; + param.deref = src.m_deref; + param.hasDeref = src.m_hasDeref; + param.out = src.m_out; + param.atReturn = src.m_atReturn; + out.params.push_back(std::move(param)); + } + + // The strings and arrays above were copied, so the FFI's allocation goes back now. + BNTTDBehaviorFreeCall(&raw); + return true; +} diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index 840a6f36..2d68e4e8 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -219,6 +219,17 @@ if (WIN32) "${PROJECT_SOURCE_DIR}/adapters/dbgeng/" ${LIBRARY_OUTPUT_DIRECTORY_PATH}/dbgeng ) + + # The TTD Behavior sidebar shells out to this rather than linking it in. Built by + # CI from Vector35/ttd-capa; see ttd-extract/README.md, whose footer records the exact + # commit. The executable reads win32-index.bin from beside itself, so the folder is + # copied whole. + add_custom_command(TARGET debuggercore PRE_LINK + COMMAND ${CMAKE_COMMAND} -E echo "Copying TTD behavior extractor" + COMMAND ${CMAKE_COMMAND} -E copy_directory + "${PROJECT_SOURCE_DIR}/ttd-extract/" + ${LIBRARY_OUTPUT_DIRECTORY_PATH}/ttd-extract + ) endif() if(APPLE) diff --git a/core/debugger.cpp b/core/debugger.cpp index d37ef548..4e4c85a0 100644 --- a/core/debugger.cpp +++ b/core/debugger.cpp @@ -126,6 +126,28 @@ static void RegisterSettings() "enumDescriptions" : ["Windows Native - lightweight native Windows debug API adapter", "DbgEng - Windows Debugger Engine (WinDbg) based adapter"], "ignore" : ["SettingsProjectScope", "SettingsResourceScope"] })"); + + // Registered here rather than in the UI because extraction is driven from the Python + // API too, and a headless script has to be able to configure it. + settings->RegisterSetting("debugger.ttdBehaviorExtractorPath", + R"({ + "title" : "TTD Behavior Extractor Path", + "type" : "string", + "default" : "", + "description" : "Path of the executable that extracts Windows API calls from a TTD trace. Leave empty to use the one shipped with the debugger; set it to run a build of your own.", + "ignore" : ["SettingsProjectScope", "SettingsResourceScope"] + })"); + + settings->RegisterSetting("debugger.ttdBehaviorMaxBuffer", + R"({ + "title" : "TTD Behavior Buffer Capture Limit", + "type" : "number", + "default" : 65536, + "minValue" : 0, + "maxValue" : 16777216, + "description" : "Bytes to keep from any one buffer parameter when extracting API calls from a TTD trace. Buffers longer than this are captured as a prefix and marked as truncated. Buffer parameters are a small fraction of the calls in a typical trace, so this has far less effect on report size than it appears.", + "ignore" : ["SettingsProjectScope", "SettingsResourceScope"] + })"); #endif settings->RegisterSetting("debugger.stackVariableAnnotations", diff --git a/core/ffi.cpp b/core/ffi.cpp index f6fd65f6..71879fb8 100644 --- a/core/ffi.cpp +++ b/core/ffi.cpp @@ -20,6 +20,7 @@ limitations under the License. #include "highlevelilinstruction.h" #include "debuggercontroller.h" #include "debuggercommon.h" +#include "ttdbehavior.h" #include "../api/ffi.h" #include #include @@ -2287,3 +2288,190 @@ char* BNDebuggerGetWinDbgLatestVersion(void) } #endif // WIN32 + + +// --- TTD behavior reports --------------------------------------------------------- +// +// Plain open/close rather than the reference-counted handle the controller uses: a +// report is not shared between subsystems, and each consumer maps its own view. + +static TTDBehaviorReport* AsReport(BNTTDBehaviorReport* handle) +{ + return reinterpret_cast(handle); +} + + +BNTTDBehaviorReport* BNTTDBehaviorOpenReport(const char* path, char** errorMessage) +{ + auto* report = new TTDBehaviorReport(); + std::string error; + if (!report->Open(path ? path : "", error)) + { + delete report; + if (errorMessage) + *errorMessage = BNDebuggerAllocString(error.c_str()); + return nullptr; + } + if (errorMessage) + *errorMessage = nullptr; + return reinterpret_cast(report); +} + + +void BNTTDBehaviorCloseReport(BNTTDBehaviorReport* report) +{ + delete AsReport(report); +} + + +uint64_t BNTTDBehaviorGetCallCount(BNTTDBehaviorReport* report) +{ + return report ? AsReport(report)->GetCallCount() : 0; +} + + +uint64_t BNTTDBehaviorGetDecodedCount(BNTTDBehaviorReport* report) +{ + return report ? AsReport(report)->GetDecodedCount() : 0; +} + + +uint64_t BNTTDBehaviorGetProcessId(BNTTDBehaviorReport* report) +{ + return report ? AsReport(report)->GetProcessId() : 0; +} + + +uint64_t BNTTDBehaviorGetMaxSequence(BNTTDBehaviorReport* report) +{ + return report ? AsReport(report)->GetMaxSequence() : 0; +} + + +uint32_t BNTTDBehaviorGetMaxPositionChars(BNTTDBehaviorReport* report) +{ + return report ? AsReport(report)->GetMaxPositionChars() : 0; +} + + +char* BNTTDBehaviorGetTracePath(BNTTDBehaviorReport* report) +{ + return BNDebuggerAllocString(report ? AsReport(report)->GetTracePath().c_str() : ""); +} + + +char* BNTTDBehaviorGetArchitecture(BNTTDBehaviorReport* report) +{ + return BNDebuggerAllocString(report ? AsReport(report)->GetArchitecture().c_str() : ""); +} + + +uint64_t* BNTTDBehaviorRunQuery(BNTTDBehaviorReport* report, const char* query, size_t* count) +{ + *count = 0; + if (!report) + return nullptr; + + std::vector matches = AsReport(report)->RunQuery(query ? query : ""); + *count = matches.size(); + if (matches.empty()) + return nullptr; + + auto* result = new uint64_t[matches.size()]; + std::memcpy(result, matches.data(), matches.size() * sizeof(uint64_t)); + return result; +} + + +void BNTTDBehaviorFreeQueryResult(uint64_t* result) +{ + delete[] result; +} + + +bool BNTTDBehaviorGetCall(BNTTDBehaviorReport* report, uint64_t index, bool withParams, BNTTDApiCall* out) +{ + if (!report || !out) + return false; + + TTDApiCall call; + if (!AsReport(report)->GetCall(index, call, withParams)) + return false; + + out->m_seq = call.seq; + out->m_tid = call.tid; + out->m_positionSequence = call.positionSequence; + out->m_positionSteps = call.positionSteps; + out->m_module = BNDebuggerAllocString(call.module.c_str()); + out->m_api = BNDebuggerAllocString(call.api.c_str()); + out->m_ret = call.ret; + out->m_returnAddress = call.returnAddress; + out->m_paramSummary = BNDebuggerAllocString(call.paramSummary.c_str()); + out->m_decoded = call.decoded; + out->m_paramCount = call.params.size(); + out->m_params = nullptr; + + if (!call.params.empty()) + { + out->m_params = new BNTTDApiCallParam[call.params.size()]; + for (size_t i = 0; i < call.params.size(); ++i) + { + const TTDApiCallParam& src = call.params[i]; + BNTTDApiCallParam& dst = out->m_params[i]; + dst.m_name = BNDebuggerAllocString(src.name.c_str()); + dst.m_type = BNDebuggerAllocString(src.type.c_str()); + dst.m_kind = BNDebuggerAllocString(src.kind.c_str()); + dst.m_value = src.value; + dst.m_str = BNDebuggerAllocString(src.str.c_str()); + dst.m_flagCount = src.flags.size(); + dst.m_flags = nullptr; + if (!src.flags.empty()) + { + dst.m_flags = new char*[src.flags.size()]; + for (size_t f = 0; f < src.flags.size(); ++f) + dst.m_flags[f] = BNDebuggerAllocString(src.flags[f].c_str()); + } + dst.m_byteCount = src.bytes.size(); + dst.m_bytes = nullptr; + if (!src.bytes.empty()) + { + dst.m_bytes = new uint8_t[src.bytes.size()]; + std::memcpy(dst.m_bytes, src.bytes.data(), src.bytes.size()); + } + dst.m_bytesTotal = src.bytesTotal; + dst.m_deref = src.deref; + dst.m_hasDeref = src.hasDeref; + dst.m_out = src.out; + dst.m_atReturn = src.atReturn; + } + } + return true; +} + + +void BNTTDBehaviorFreeCall(BNTTDApiCall* call) +{ + if (!call) + return; + BNDebuggerFreeString(call->m_module); + BNDebuggerFreeString(call->m_api); + BNDebuggerFreeString(call->m_paramSummary); + for (size_t i = 0; i < call->m_paramCount; ++i) + { + BNTTDApiCallParam& p = call->m_params[i]; + BNDebuggerFreeString(p.m_name); + BNDebuggerFreeString(p.m_type); + BNDebuggerFreeString(p.m_kind); + BNDebuggerFreeString(p.m_str); + for (size_t f = 0; f < p.m_flagCount; ++f) + BNDebuggerFreeString(p.m_flags[f]); + delete[] p.m_flags; + delete[] p.m_bytes; + } + delete[] call->m_params; + call->m_params = nullptr; + call->m_paramCount = 0; + call->m_module = nullptr; + call->m_api = nullptr; + call->m_paramSummary = nullptr; +} diff --git a/core/ttd-extract/README.md b/core/ttd-extract/README.md new file mode 100644 index 00000000..2168081f --- /dev/null +++ b/core/ttd-extract/README.md @@ -0,0 +1,291 @@ +# `ttdcapa-extract` + +Sweeps a Time Travel Debugging trace and records every Windows API call it made: when it +happened, which function was called, what its arguments were, and what it returned. +Arguments are decoded against Microsoft's Win32 API metadata where a signature exists, so +parameters come back named and typed rather than as four guessed registers. + +It writes either of two things, or both: + +- **`-o report.json`** -- the JSON report [capa](https://github.com/mandiant/capa) consumes. +- **`-b report.ttdb`** -- a compact binary layout meant to be memory-mapped and browsed. + On a 3.4M-call trace it takes 3.6s to write against ~56s for the JSON, and loading it is + a memory map rather than a 17s parse. The format is specified [below](#the-ttdb-format). + +## Usage + +``` +ttdcapa-extract [-o ] [-b ] [options] +ttdcapa-extract --dump-sig +``` + +### Output + +| Option | Effect | +| --- | --- | +| `-o`, `--output ` | write the JSON report | +| `-b`, `--binary ` | write the binary report | +| `--sample ` | hash this file and record its MD5/SHA1/SHA256 in the report | + +Giving neither `-o` nor `-b` sweeps the trace and writes nothing, which is only useful for +timing. + +### Finding the replay engine + +| Option | Effect | +| --- | --- | +| `--ttd-dlls ` | where `TTDReplay.dll` and `TTDReplayCPU.dll` live | + +Those DLLs are Microsoft's, shipped with WinDbg, and are not redistributed with this tool. +Point at the copy WinDbg already installed -- normally its `amd64 td` folder: + +```powershell +Join-Path (Get-AppxPackage Microsoft.WinDbg).InstallLocation 'amd64 td' +``` + +Without the flag the usual DLL search order applies, so copies placed beside the +executable work too. `TTDReplay.dll` is delay-loaded, which is what lets the location be +chosen at runtime; a missing engine is reported rather than crashing the process. + +### Argument decoding + +| Option | Effect | +| --- | --- | +| `--win32-index ` | use a specific `win32-index.bin` instead of the one beside the executable | +| `--no-metadata` | ignore the metadata entirely and use the register/stack heuristic everywhere | +| `--with-stack-args` | for calls with *no* signature, also grab four stack slots past the register arguments. Calls that have a signature always capture their true arity regardless | +| `--max-buffer N` | bytes to keep from any one captured buffer (default 65536). Buffers cut short at this limit carry their real length, so a consumer can tell a short buffer from a truncated one | +| `--max-calls N` | stop recording after N calls (default 0, unlimited). The sweep still runs to the end | +| `--dump-sig ` | print one function's decoded signature and exit. Needs only the metadata index, not a trace or the replay DLLs, which makes it a good smoke test | + +### Driving it from another program + +| Option | Effect | +| --- | --- | +| `--progress` | emit progress on stderr | +| `--cancel-on-stdin` | stop the sweep when a line reading `cancel` arrives on stdin | + +`--progress` writes `[phase] sweep` and `[phase] write` markers and, during the sweep, +`[progress] ` a few times a second. The phases matter because the sweep is +only part of the wall clock: on a 3.4M-call trace it takes about 30s while serialising the +report takes comparable time, so a caller tracking only the sweep would sit at 100% looking +wedged. Percentages are clamped to be non-decreasing, since positions read from different +threads are not perfectly ordered. + +`--cancel-on-stdin` interrupts the replay and then writes the report as usual: everything +recorded up to that point is kept, so a cancelled run yields a complete, valid report of a +shorter prefix of the trace. + +## Notes on what the data can and cannot tell you + +**A missing string argument is not evidence the argument was null.** The sweep reads guest +memory through the `IThreadView` the replay engine hands to a callback, and the SDK fixes +those reads to `QueryMemoryPolicy::ThreadLocal` -- a fast lookup explicitly allowed to +ignore memory observed by other threads or by this thread at another time. It returns +nothing for roughly a quarter of string parameters even though the data is in the trace; +reading the same address at the same position through a cursor returns the whole string. +The parameter's raw pointer value is still recorded, and is usually valid. + +**Calls without a signature have guessed arguments.** Coverage is the public Windows SDK, +so `ntdll` internals and CRT helpers fall back to capturing argument registers (x64) or +stack slots (x86). Their values are positional, unnamed, and the count is a guess. The +binary format flags these per call. + +## The `.ttdb` format + +Designed to be **memory-mapped and read in place**: nothing is parsed on open and nothing +is allocated per call, so opening a 3.4-million-call report costs a header validation. The +reference implementations are `ttd/src/binreport.cpp` (writer) and, in the Binary Ninja +debugger, `core/ttdbehavior.cpp` (reader). + +Format version 2. All integers are **little-endian**. All *file offsets* are byte offsets +from the start of the file, so a mapped view needs no fixups; *region offsets* (into the +string table or the blob) are relative to that region's start, and are noted as such. + +### Layout + +``` ++----------------------------+ 0 +| header | 128 bytes ++----------------------------+ callsOff +| call records | callCount * 48 bytes, in sweep order ++----------------------------+ paramsOff +| parameter records | variable length, referenced by call records ++----------------------------+ stringsOff +| string table | stringsSize bytes, NUL-terminated UTF-8 ++----------------------------+ blobOff +| blob | blobSize bytes: search text, strings, buffers ++----------------------------+ end of file +``` + +The split exists because the three kinds of data have different access patterns. Call +records are fixed size so row *N* is a multiply. The string table is deduplicated, which is +where the compression really comes from -- a trace has millions of calls but only thousands +of distinct module and function names, so every name in a 3.4M-call report fits in about +100 KB. Everything variable-length and unique to one call goes in the blob. + +### Header (128 bytes) + +| Offset | Type | Field | Notes | +| ---: | --- | --- | --- | +| 0 | `char[8]` | magic | `TTDBEHV1` | +| 8 | `u32` | version | 2 | +| 12 | `u32` | arch | 0 = x64, 1 = x86 | +| 16 | `u64` | callCount | number of call records | +| 24 | `u64` | paramBytes | size in bytes of the parameter region | +| 32 | `u64` | pid | process id of the traced process | +| 40 | `u64` | callsOff | file offset of the call records | +| 48 | `u64` | paramsOff | file offset of the parameter records | +| 56 | `u64` | stringsOff | file offset of the string table | +| 64 | `u64` | stringsSize | size in bytes of the string table | +| 72 | `u64` | blobOff | file offset of the blob | +| 80 | `u64` | blobSize | size in bytes of the blob | +| 88 | `u64` | decodedCount | calls whose parameters came from a real signature | +| 96 | `u64` | maxSeq | highest sequence number, i.e. `callCount - 1` | +| 104 | `u32` | tracePathStr | string-table offset of the source `.run` path | +| 108 | `u32` | sampleNameStr | string-table offset of the main module's name | +| 112 | `u32` | maxPositionChars | widest rendered position, for column sizing | +| 116 | | *(reserved)* | zero | + +`arch` describes the **traced process**, taken from the main module's PE format, not the +machine that recorded the trace. A reader should reject a file whose magic does not match, +whose version it does not know, or whose regions do not fit inside the file. + +### Call record (48 bytes) + +| Offset | Type | Field | Notes | +| ---: | --- | --- | --- | +| 0 | `u64` | ret | the call's return value | +| 8 | `u32` | tid | TTD unique thread id | +| 12 | `u32` | positionSequence | TTD position, sequence part | +| 16 | `u32` | positionSteps | TTD position, steps part | +| 20 | `u32` | moduleStr | string-table offset of the module name, e.g. `kernel32` | +| 24 | `u32` | apiStr | string-table offset of the function name, e.g. `CreateFileW` | +| 28 | `u32` | paramOff | offset into the parameter region; 0 when there are none | +| 32 | `u32` | searchOff | offset into the blob of this call's search text | +| 36 | `u16` | paramCount | low 15 bits; bit 15 (`0x8000`) set means *decoded* | +| 38 | `u16` | searchLen | length in bytes of the search text | +| 40 | `u64` | returnAddress | the instruction after the CALL, i.e. the call site | + +**There is no sequence number field.** Recorded calls are numbered densely from zero in +sweep order, so a record's index *is* its sequence number. + +**Position** is conventionally rendered `%X:%X` of sequence and steps -- `45905:15A5` -- +which is the form WinDbg and the Binary Ninja debugger accept for time travel. + +**`returnAddress`** identifies the caller, which is how you distinguish a call the sample +made itself from one a system DLL made on its behalf. + +**The decoded bit** matters for interpreting the parameters. When set, the extractor had a +real signature for the function from Microsoft's Win32 metadata, so the parameters have +correct arity, names, and types. When clear, it fell back to capturing argument registers +(x64) or stack slots (x86) heuristically: the values are positional, unnamed, and the count +is a guess. A decoded call may legitimately have zero parameters, which is why the flag is +separate from the count. + +### Parameter record (variable length) + +Parameters for one call are stored consecutively starting at `paramsOff + paramOff`, and +are decoded in sequence -- there is no index, so reading parameter *k* means walking the +*k* before it. That is deliberate: a viewer only decodes parameters for rows a user +actually looks at. + +Fixed part, 18 bytes: + +| Offset | Type | Field | +| ---: | --- | --- | +| 0 | `u8` | kind | +| 1 | `u8` | bits | +| 2 | `u32` | nameStr (string-table offset; 0 when unnamed) | +| 6 | `u32` | typeStr (string-table offset; 0 when untyped) | +| 10 | `u64` | value (the raw register or stack value) | + +Then, in this order, only the parts whose bit is set in `bits`: + +| Bit | Name | Adds | +| ---: | --- | --- | +| 0x01 | Out | *(nothing; marks an `[Out]` parameter)* | +| 0x02 | AtReturn | *(nothing; value was re-read at the call's return)* | +| 0x04 | HasDeref | `u64` deref -- the pointed-to value | +| 0x08 | HasStr | `u32` strOff, `u32` strLen -- blob offset and length | +| 0x10 | HasBytes | `u32` bytesOff, `u32` bytesLen, `u64` bytesTotal | +| 0x20 | HasFlags | `u32` flagsOff, `u32` flagsLen -- `|`-joined names in the blob | + +So a parameter's size is 18 bytes plus 8 for HasDeref, 8 for HasStr, 16 for HasBytes and 8 +for HasFlags, in that order. + +**`bytesTotal`** is the buffer's real length when only a prefix was captured (the capture +limit defaults to 64 KiB); `bytesLen` is what is actually in the file. `bytesTotal > +bytesLen` means truncated. + +**`AtReturn`** marks values re-read at the call's return position rather than at the call. +This is what makes `[Out]` parameters renderable at all -- at the moment of the call they +have not been written yet -- and it is the one thing a time-travel trace gives you that a +live debugger cannot easily. + +### Parameter kinds + +`kind` is an index into this list. It describes how the value should be interpreted, not +its C type, which is in `typeStr`. + +| # | Name | Meaning | +| ---: | --- | --- | +| 0 | *(unknown)* | unclassified, or a heuristic capture with no signature | +| 1 | `int` | plain scalar | +| 2 | `bool` | | +| 3 | `handle` | opaque, pointer-sized, never dereferenced | +| 4 | `enum` | scalar with a symbolic value table; see HasFlags | +| 5 | `float` | | +| 6 | `double` | | +| 7 | `str` | `char*`, NUL-terminated | +| 8 | `wstr` | `wchar_t*`, NUL-terminated | +| 9 | `strbuf` | `char[]`, length from a sibling parameter | +| 10 | `wstrbuf` | `wchar_t[]` | +| 11 | `buf` | `void*`/byte array | +| 12 | `int*` | pointer to a scalar; see HasDeref | +| 13 | `struct*` | pointer to a struct or union, not expanded | +| 14 | `fnptr` | | +| 15 | `guid` | pointer to a 16-byte GUID, rendered into HasStr | +| 16 | `ptr` | opaque pointer | +| 17 | `str*` | `char**`, an out-parameter receiving an allocated string | +| 18 | `wstr*` | `wchar_t**` | + +### String table + +A run of NUL-terminated UTF-8 strings, deduplicated. A string-table offset is relative to +`stringsOff`. **Offset 0 is the empty string**, so 0 doubles as "absent". + +Only repeated text lives here: module names, function names, parameter names, parameter +type names. + +### Blob + +Raw bytes, referenced by `(offset, length)` pairs relative to `blobOff`. It holds three +things, interleaved in whatever order the writer emitted them: + +- **Search text**, one per call, referenced by `searchOff`/`searchLen`. This is a + lowercased rendering of `module!api` followed by each parameter as it would be displayed, + including printable runs extracted from captured buffers. It exists so a text filter is a + substring scan over mapped pages with nothing to build first. +- **Parameter strings**, referenced by HasStr. +- **Captured buffer contents**, referenced by HasBytes. Raw bytes, not hex. + +### Reading a report + +Validate the magic and version, check the regions fit in the file, and map it. Then: + +- **Row *N***: read 48 bytes at `callsOff + N * 48`. +- **Its module and function**: NUL-terminated strings at `stringsOff + moduleStr` and + `stringsOff + apiStr`. +- **Its parameters**: walk `paramCount & 0x7FFF` records from `paramsOff + paramOff`, + decoding each per the bits. +- **Filtering**: for a text match, `memmem` the needle in the `searchLen` bytes at + `blobOff + searchOff`. For a match on module or function, resolve the name against the + string table once to a set of offsets, then compare `moduleStr`/`apiStr` as integers -- + that is far cheaper than a string search, which is why scoped queries are faster than + plain text ones. + +--- + +Built by CI from [Vector35/ttd-capa](https://github.com/Vector35/ttd-capa) +at `6cd27a2078a1ddd2149c710beef294502fcd1d72` (`ci-flat-artifact`) on 2026-07-30. diff --git a/core/ttd-extract/ttdcapa-extract.exe b/core/ttd-extract/ttdcapa-extract.exe new file mode 100644 index 00000000..f73fb7b9 Binary files /dev/null and b/core/ttd-extract/ttdcapa-extract.exe differ diff --git a/core/ttd-extract/win32-index.bin b/core/ttd-extract/win32-index.bin new file mode 100644 index 00000000..9df5b243 Binary files /dev/null and b/core/ttd-extract/win32-index.bin differ diff --git a/core/ttdbehavior.cpp b/core/ttdbehavior.cpp new file mode 100644 index 00000000..201c2ffd --- /dev/null +++ b/core/ttdbehavior.cpp @@ -0,0 +1,708 @@ +/* +Copyright 2020-2026 Vector 35 Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +#include "ttdbehavior.h" + +#include +#include +#include +#include +#include + +#ifdef WIN32 + #ifndef WIN32_LEAN_AND_MEAN + #define WIN32_LEAN_AND_MEAN + #endif + #include +#else + #include + #include + #include + #include +#endif + +using namespace BinaryNinjaDebugger; + +namespace { + // Mirrors ttd-capa's ttd/src/binreport.hpp. Any change there needs one here. + constexpr char kMagic[8] = { 'T', 'T', 'D', 'B', 'E', 'H', 'V', '1' }; + constexpr uint32_t kVersion = 2; // 2 added returnAddress to the call record + constexpr uint64_t kHeaderSize = 128; + constexpr uint64_t kCallRecordSize = 48; + constexpr uint16_t kDecodedFlag = 0x8000; + + enum ParamBits : uint8_t + { + ParamOut = 0x01, + ParamAtReturn = 0x02, + ParamHasDeref = 0x04, + ParamHasStr = 0x08, + ParamHasBytes = 0x10, + ParamHasFlags = 0x20, + }; + + // The extractor writes the parameter kind as a raw byte; these are the display names, + // indexed by that value. Order matches win32meta.hpp's ArgKind. + const char* const kKindNames[] = { "", "int", "bool", "handle", "enum", "float", "double", "str", + "wstr", "strbuf", "wstrbuf", "buf", "int*", "struct*", "fnptr", "guid", "ptr", "str*", "wstr*" }; + + template + T Read(const uint8_t* p) + { + T v {}; + std::memcpy(&v, p, sizeof(T)); + return v; + } + + std::string ToLower(std::string s) + { + std::transform(s.begin(), s.end(), s.begin(), + [](unsigned char c) { return static_cast(std::tolower(c)); }); + return s; + } + + std::string FormatHex(uint64_t value) + { + char buf[32]; + std::snprintf(buf, sizeof(buf), "0x%llx", static_cast(value)); + return buf; + } + + + // Bytes a table cell can show before it stops being a glance. + constexpr size_t kMaxPreviewBytes = 32; + + std::string PreviewBytes(const std::vector& bytes) + { + static const char* digits = "0123456789abcdef"; + // Not std::min: windows.h defines a min macro and this file includes it. + size_t shown = bytes.size() < kMaxPreviewBytes ? bytes.size() : kMaxPreviewBytes; + std::string out; + out.reserve(shown * 3 + 4); + for (size_t i = 0; i < shown; ++i) + { + if (i != 0) + out.push_back(' '); + out.push_back(digits[bytes[i] >> 4]); + out.push_back(digits[bytes[i] & 0x0f]); + } + if (bytes.size() > shown) + out += " \xe2\x80\xa6"; // ellipsis + return out; + } + + // Decimal, or hex when prefixed with 0x. + bool ParseNumber(const std::string& text, uint64_t& out) + { + if (text.empty()) + return false; + try + { + size_t consumed = 0; + bool hex = text.size() > 2 && text[0] == '0' && (text[1] == 'x' || text[1] == 'X'); + out = std::stoull(hex ? text.substr(2) : text, &consumed, hex ? 16 : 10); + return consumed == (hex ? text.size() - 2 : text.size()); + } + catch (...) + { + return false; + } + } + + // `!0`, `>0x1000`, `>=5`, `<10`, `<=10`, `0x400000-0x500000`, or a plain value. + bool ParseNumeric(const std::string& text, TTDBehaviorQuery::Numeric& out) + { + std::string value = text; + if (value.rfind("!", 0) == 0) + { + out.op = TTDBehaviorQuery::Numeric::NotEqual; + value = value.substr(1); + } + else if (value.rfind(">=", 0) == 0) + { + out.op = TTDBehaviorQuery::Numeric::GreaterEqual; + value = value.substr(2); + } + else if (value.rfind("<=", 0) == 0) + { + out.op = TTDBehaviorQuery::Numeric::LessEqual; + value = value.substr(2); + } + else if (value.rfind(">", 0) == 0) + { + out.op = TTDBehaviorQuery::Numeric::Greater; + value = value.substr(1); + } + else if (value.rfind("<", 0) == 0) + { + out.op = TTDBehaviorQuery::Numeric::Less; + value = value.substr(1); + } + else + { + // A '-' separating two numbers is a range. Searched past any 0x so the hex + // digits of a lone value are not mistaken for one. + size_t from = (value.size() > 2 && value[0] == '0' && (value[1] == 'x' || value[1] == 'X')) ? 2 : 1; + size_t dash = value.find('-', from); + if (dash != std::string::npos) + { + if (!ParseNumber(value.substr(0, dash), out.a) || !ParseNumber(value.substr(dash + 1), out.b)) + return false; + out.op = TTDBehaviorQuery::Numeric::Range; + return true; + } + } + return ParseNumber(value, out.a); + } + + // Split on whitespace, honouring double quotes so a phrase can contain spaces. + std::vector SplitTerms(const std::string& text) + { + std::vector terms; + std::string current; + bool inQuotes = false; + for (char c : text) + { + if (c == '"') + inQuotes = !inQuotes; + else if (std::isspace(static_cast(c)) && !inQuotes) + { + if (!current.empty()) + terms.push_back(current); + current.clear(); + } + else + current += c; + } + if (!current.empty()) + terms.push_back(current); + return terms; + } +} // namespace + + +bool TTDBehaviorQuery::Numeric::Test(uint64_t value) const +{ + switch (op) + { + case NotEqual: + return value != a; + case Greater: + return value > a; + case GreaterEqual: + return value >= a; + case Less: + return value < a; + case LessEqual: + return value <= a; + case Range: + return value >= a && value <= b; + case Equal: + default: + return value == a; + } +} + + +void TTDBehaviorQuery::Parse(const std::string& text) +{ + m_terms.clear(); + for (const std::string& raw : SplitTerms(text)) + { + Term term; + size_t colon = raw.find(':'); + std::string field = colon != std::string::npos && colon > 0 ? ToLower(raw.substr(0, colon)) : std::string(); + std::string value = colon != std::string::npos && colon > 0 ? raw.substr(colon + 1) : raw; + + // An unknown prefix is not an error: "c:\windows" should search for that text, + // not complain about a field called "c". + bool numericField = false; + if (field == "module" || field == "mod") + term.field = Field::Module; + else if (field == "api" || field == "func" || field == "function") + term.field = Field::Api; + else if (field == "tid" || field == "thread") + { + term.field = Field::Tid; + numericField = true; + } + else if (field == "ret" || field == "return") + { + term.field = Field::Ret; + numericField = true; + } + else if (field == "retaddr" || field == "caller" || field == "from") + { + term.field = Field::RetAddr; + numericField = true; + } + else + { + term.field = Field::Text; + value = raw; + } + + if (numericField) + { + if (!ParseNumeric(value, term.numeric)) + { + // Unparseable number: treat the whole thing as text so the row set does + // not silently become everything. + term.field = Field::Text; + term.text = ToLower(raw); + m_terms.push_back(std::move(term)); + continue; + } + } + else + { + if (!value.empty() && value.back() == '*') + { + term.prefix = true; + value.pop_back(); + } + if (value.empty()) + continue; + term.text = ToLower(value); + } + m_terms.push_back(std::move(term)); + } +} + + +TTDBehaviorReport::~TTDBehaviorReport() +{ + Close(); +} + + +void TTDBehaviorReport::Close() +{ +#ifdef WIN32 + if (m_map != nullptr) + ::UnmapViewOfFile(const_cast(m_map)); + if (m_mappingHandle != nullptr) + ::CloseHandle(m_mappingHandle); + if (m_fileHandle != nullptr && m_fileHandle != INVALID_HANDLE_VALUE) + ::CloseHandle(m_fileHandle); +#else + if (m_map != nullptr) + ::munmap(const_cast(m_map), static_cast(m_mapSize)); + if (m_fileHandle != nullptr) + ::close(static_cast(reinterpret_cast(m_fileHandle)) - 1); +#endif + m_map = nullptr; + m_mappingHandle = nullptr; + m_fileHandle = nullptr; + m_mapSize = 0; + m_callCount = 0; + m_decodedCount = 0; + m_maxSeq = 0; + m_pid = 0; + m_maxPositionChars = 0; + m_path.clear(); + m_tracePath.clear(); + m_arch.clear(); +} + + +bool TTDBehaviorReport::Open(const std::string& path, std::string& error) +{ + Close(); + + const uint8_t* map = nullptr; + uint64_t size = 0; + +#ifdef WIN32 + HANDLE file = ::CreateFileA( + path.c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, nullptr); + if (file == INVALID_HANDLE_VALUE) + { + error = "cannot open " + path; + return false; + } + LARGE_INTEGER fileSize {}; + if (!::GetFileSizeEx(file, &fileSize) || static_cast(fileSize.QuadPart) < kHeaderSize) + { + ::CloseHandle(file); + error = "report is too small to contain a header"; + return false; + } + HANDLE mapping = ::CreateFileMappingA(file, nullptr, PAGE_READONLY, 0, 0, nullptr); + if (mapping == nullptr) + { + ::CloseHandle(file); + error = "cannot create a mapping for " + path; + return false; + } + map = static_cast(::MapViewOfFile(mapping, FILE_MAP_READ, 0, 0, 0)); + if (map == nullptr) + { + ::CloseHandle(mapping); + ::CloseHandle(file); + error = "cannot map " + path; + return false; + } + size = static_cast(fileSize.QuadPart); + m_fileHandle = file; + m_mappingHandle = mapping; +#else + int fd = ::open(path.c_str(), O_RDONLY); + if (fd < 0) + { + error = "cannot open " + path; + return false; + } + struct stat st {}; + if (::fstat(fd, &st) != 0 || static_cast(st.st_size) < kHeaderSize) + { + ::close(fd); + error = "report is too small to contain a header"; + return false; + } + void* addr = ::mmap(nullptr, static_cast(st.st_size), PROT_READ, MAP_PRIVATE, fd, 0); + if (addr == MAP_FAILED) + { + ::close(fd); + error = "cannot map " + path; + return false; + } + map = static_cast(addr); + size = static_cast(st.st_size); + // Stored offset by one so that fd 0 is distinguishable from "no handle". + m_fileHandle = reinterpret_cast(static_cast(fd) + 1); +#endif + + m_map = map; + m_mapSize = size; + + if (std::memcmp(map, kMagic, sizeof(kMagic)) != 0) + { + Close(); + error = "not a TTD behavior report"; + return false; + } + uint32_t version = Read(map + 8); + if (version != kVersion) + { + Close(); + error = "report format version " + std::to_string(version) + " is not supported (expected " + + std::to_string(kVersion) + "); re-extract it"; + return false; + } + + uint32_t archId = Read(map + 12); + m_callCount = Read(map + 16); + m_pid = Read(map + 32); + m_callsOff = Read(map + 40); + m_paramsOff = Read(map + 48); + m_stringsOff = Read(map + 56); + m_stringsSize = Read(map + 64); + m_blobOff = Read(map + 72); + m_blobSize = Read(map + 80); + m_decodedCount = Read(map + 88); + m_maxSeq = Read(map + 96); + uint32_t tracePathStr = Read(map + 104); + m_maxPositionChars = Read(map + 112); + + // Refuse a file whose regions do not fit rather than trusting offsets from disk. + auto withinFile = [size](uint64_t off, uint64_t len) { + return off <= size && len <= size - off; + }; + if (!withinFile(m_callsOff, m_callCount * kCallRecordSize) || !withinFile(m_stringsOff, m_stringsSize) + || !withinFile(m_blobOff, m_blobSize)) + { + Close(); + error = "report header describes regions outside the file; it may be truncated"; + return false; + } + + m_path = path; + m_arch = archId == 1 ? "x86" : "x64"; + m_tracePath = MappedString(tracePathStr); + return true; +} + + +const uint8_t* TTDBehaviorReport::CallRecord(uint64_t index) const +{ + return m_map + m_callsOff + index * kCallRecordSize; +} + + +std::string TTDBehaviorReport::MappedString(uint32_t offset) const +{ + if (m_map == nullptr || offset >= m_stringsSize) + return std::string(); + const char* start = reinterpret_cast(m_map + m_stringsOff + offset); + size_t maxLen = static_cast(m_stringsSize - offset); + size_t len = 0; + while (len < maxLen && start[len] != '\0') + ++len; + return std::string(start, len); +} + + +void TTDBehaviorReport::ForEachString(const std::function& fn) const +{ + if (m_map == nullptr) + return; + const char* base = reinterpret_cast(m_map + m_stringsOff); + uint64_t offset = 0; + while (offset < m_stringsSize) + { + size_t length = 0; + while (offset + length < m_stringsSize && base[offset + length] != '\0') + ++length; + if (length != 0) + fn(static_cast(offset), base + offset, length); + offset += length + 1; + } +} + + +void TTDBehaviorReport::ResolveQuery(TTDBehaviorQuery& query) const +{ + bool needsStrings = false; + for (TTDBehaviorQuery::Term& t : query.GetTerms()) + { + t.stringOffsets.clear(); + if (t.field == TTDBehaviorQuery::Field::Module || t.field == TTDBehaviorQuery::Field::Api) + needsStrings = true; + } + if (!needsStrings || m_map == nullptr) + return; + + // One pass over the string table -- a few thousand entries, not a few million rows -- + // collecting the offsets each module/api term accepts. Matching a row afterwards is + // then a lookup in a handful of integers. + ForEachString([&query](uint32_t offset, const char* text, size_t length) { + std::string lowered = ToLower(std::string(text, length)); + for (TTDBehaviorQuery::Term& t : query.GetTerms()) + { + if (t.field != TTDBehaviorQuery::Field::Module && t.field != TTDBehaviorQuery::Field::Api) + continue; + bool hit = t.prefix ? lowered.rfind(t.text, 0) == 0 : lowered == t.text; + if (hit) + t.stringOffsets.push_back(offset); + } + }); + + for (TTDBehaviorQuery::Term& t : query.GetTerms()) + std::sort(t.stringOffsets.begin(), t.stringOffsets.end()); +} + + +bool TTDBehaviorReport::Matches(uint64_t index, const TTDBehaviorQuery& query) const +{ + if (query.IsEmpty()) + return true; + if (m_map == nullptr || index >= m_callCount) + return false; + + const uint8_t* rec = CallRecord(index); + for (const TTDBehaviorQuery::Term& t : query.GetTerms()) + { + bool ok = true; + switch (t.field) + { + case TTDBehaviorQuery::Field::Text: + { + uint32_t searchOff = Read(rec + 32); + uint16_t searchLen = Read(rec + 38); + // searchOff is relative to the blob region, so it is bounded against the + // region's size rather than its position in the file. + if (static_cast(searchOff) + searchLen > m_blobSize) + return false; + std::string_view hay(reinterpret_cast(m_map + m_blobOff + searchOff), searchLen); + ok = hay.find(t.text) != std::string_view::npos; + break; + } + case TTDBehaviorQuery::Field::Module: + case TTDBehaviorQuery::Field::Api: + { + uint32_t offset = Read(rec + (t.field == TTDBehaviorQuery::Field::Module ? 20 : 24)); + ok = std::binary_search(t.stringOffsets.begin(), t.stringOffsets.end(), offset); + break; + } + case TTDBehaviorQuery::Field::Tid: + ok = t.numeric.Test(Read(rec + 8)); + break; + case TTDBehaviorQuery::Field::Ret: + ok = t.numeric.Test(Read(rec + 0)); + break; + case TTDBehaviorQuery::Field::RetAddr: + ok = t.numeric.Test(Read(rec + 40)); + break; + } + if (!ok) + return false; + } + return true; +} + + +std::vector TTDBehaviorReport::RunQuery(TTDBehaviorQuery& query) const +{ + std::vector result; + if (m_map == nullptr) + return result; + ResolveQuery(query); + if (query.IsEmpty()) + { + result.resize(static_cast(m_callCount)); + for (uint64_t i = 0; i < m_callCount; ++i) + result[static_cast(i)] = i; + return result; + } + for (uint64_t i = 0; i < m_callCount; ++i) + { + if (Matches(i, query)) + result.push_back(i); + } + return result; +} + + +std::vector TTDBehaviorReport::RunQuery(const std::string& query) const +{ + TTDBehaviorQuery parsed; + parsed.Parse(query); + return RunQuery(parsed); +} + + +bool TTDBehaviorReport::GetCall(uint64_t index, TTDApiCall& out, bool withParams) const +{ + if (m_map == nullptr || index >= m_callCount) + return false; + + const uint8_t* rec = CallRecord(index); + out.params.clear(); + out.seq = index; // recorded calls are numbered densely, so the row index is the seq + out.ret = Read(rec + 0); + out.tid = Read(rec + 8); + out.positionSequence = Read(rec + 12); + out.positionSteps = Read(rec + 16); + out.module = MappedString(Read(rec + 20)); + out.api = MappedString(Read(rec + 24)); + out.returnAddress = Read(rec + 40); + + uint32_t paramOff = Read(rec + 28); + uint16_t rawCount = Read(rec + 36); + out.decoded = (rawCount & kDecodedFlag) != 0; + int paramCount = rawCount & ~kDecodedFlag; + + // The parameter region is variable-length, so a call's parameters are decoded in + // sequence from its offset. Only ever done for rows something actually asked about. + const uint8_t* p = m_map + m_paramsOff + paramOff; + const uint8_t* blob = m_map + m_blobOff; + std::string summary; + for (int i = 0; i < paramCount; ++i) + { + uint8_t kind = *p++; + uint8_t bits = *p++; + uint32_t nameStr = Read(p); + p += 4; + uint32_t typeStr = Read(p); + p += 4; + + TTDApiCallParam param; + param.name = MappedString(nameStr); + param.type = MappedString(typeStr); + param.kind = kind < (sizeof(kKindNames) / sizeof(kKindNames[0])) ? kKindNames[kind] : ""; + param.value = Read(p); + p += 8; + param.out = (bits & ParamOut) != 0; + param.atReturn = (bits & ParamAtReturn) != 0; + + if (bits & ParamHasDeref) + { + param.hasDeref = true; + param.deref = Read(p); + p += 8; + } + if (bits & ParamHasStr) + { + uint32_t off = Read(p); + uint32_t len = Read(p + 4); + p += 8; + param.str.assign(reinterpret_cast(blob + off), len); + } + if (bits & ParamHasBytes) + { + uint32_t off = Read(p); + uint32_t len = Read(p + 4); + param.bytesTotal = Read(p + 8); + p += 16; + param.bytes.assign(blob + off, blob + off + len); + } + if (bits & ParamHasFlags) + { + uint32_t off = Read(p); + uint32_t len = Read(p + 4); + p += 8; + std::string joined(reinterpret_cast(blob + off), len); + size_t start = 0; + while (start <= joined.size()) + { + size_t bar = joined.find('|', start); + if (bar == std::string::npos) + { + if (start < joined.size()) + param.flags.push_back(joined.substr(start)); + break; + } + param.flags.push_back(joined.substr(start, bar - start)); + start = bar + 1; + } + } + + // Rendered the way it reads best given what the decoder recovered: a resolved + // string beats symbolic flags, which beat a raw value. + std::string rendered; + if (!param.str.empty()) + rendered = "\"" + param.str + "\""; + else if (!param.flags.empty()) + { + for (size_t f = 0; f < param.flags.size(); ++f) + { + if (f != 0) + rendered += "|"; + rendered += param.flags[f]; + } + } + else if (!param.bytes.empty()) + { + // Worth the width: the head of a buffer tells you at a glance what a + // WriteFile actually wrote, where the pointer alone tells you nothing. + rendered = FormatHex(param.value) + " -> [" + PreviewBytes(param.bytes) + "]"; + } + else if (param.hasDeref) + rendered = FormatHex(param.value) + " -> " + FormatHex(param.deref); + else + rendered = FormatHex(param.value); + + if (!summary.empty()) + summary += ", "; + summary += out.decoded && !param.name.empty() ? param.name + "=" + rendered : rendered; + + if (withParams) + out.params.push_back(std::move(param)); + } + out.paramSummary = std::move(summary); + return true; +} diff --git a/core/ttdbehavior.h b/core/ttdbehavior.h new file mode 100644 index 00000000..da312a6f --- /dev/null +++ b/core/ttdbehavior.h @@ -0,0 +1,188 @@ +/* +Copyright 2020-2026 Vector 35 Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +#pragma once + +// Reader and query engine for the Windows API calls extracted from a TTD trace. +// +// The report is written by an external extractor in a compact layout designed to be +// memory-mapped and read in place -- see ttd-capa's ttd/src/binreport.hpp for the format. +// Nothing is parsed on open and nothing is allocated per call: a 3.4M-call report costs +// a header validation instead of the ~17s and ~1.4GB that parsing the equivalent JSON +// did. Records are decoded out of the mapping only when something asks for a row. +// +// This lives in core rather than the UI so that queries are available over the FFI, and +// therefore from Python. The boundary is drawn so that no loop over the call list ever +// crosses it: RunQuery evaluates the whole filter here and hands back matching row +// indices in one call, which measures within noise of doing it in-process, where +// crossing per row costs 30-50%. + +#include +#include +#include +#include + +namespace BinaryNinjaDebugger { + + // One decoded parameter of an API call. Which fields are meaningful depends on what + // the Win32 metadata said about the parameter; `has*` flags say which. + struct TTDApiCallParam + { + std::string name; + std::string type; + std::string kind; + uint64_t value = 0; + std::string str; // resolved ANSI/UTF-16 string + std::vector flags; // symbolic names for enum/flag parameters + std::vector bytes; // captured buffer contents, possibly a prefix + uint64_t bytesTotal = 0; // real length when `bytes` was capped, else 0 + uint64_t deref = 0; // pointee, for int*-like parameters + bool hasDeref = false; + bool out = false; + bool atReturn = false; // re-read at the call's return position + }; + + + struct TTDApiCall + { + uint64_t seq = 0; + uint64_t tid = 0; + uint64_t positionSequence = 0; + uint64_t positionSteps = 0; + std::string module; + std::string api; + uint64_t ret = 0; + uint64_t returnAddress = 0; // the instruction after the CALL, i.e. the call site + std::vector params; + bool decoded = false; // a real signature backed this call's parameters + std::string paramSummary; // single-line rendering, for a table cell + }; + + + // A parsed filter expression: whitespace-separated terms, ANDed. A bare word (or + // "quoted phrase") matches anywhere in a call's searchable text; `field:value` + // narrows to one field. + // + // module:kernel32 api:WriteFile only kernel32's, not ntdll's + // api:Reg* ret:!0 registry calls that failed + // retaddr:0x400000-0x500000 calls the sample made itself + // + // module and api resolve against the report's deduplicated string table once per + // query, so evaluating them per row is an integer compare rather than a search. + class TTDBehaviorQuery + { + public: + struct Numeric + { + enum Op { Equal, NotEqual, Greater, GreaterEqual, Less, LessEqual, Range }; + Op op = Equal; + uint64_t a = 0; + uint64_t b = 0; + bool Test(uint64_t value) const; + }; + + enum class Field { Text, Module, Api, Tid, Ret, RetAddr }; + + struct Term + { + Field field = Field::Text; + std::string text; + bool prefix = false; + Numeric numeric; + std::vector stringOffsets; + }; + + // Never fails: anything that does not look like a field term is treated as text, + // so a half-typed query narrows rather than erroring. + void Parse(const std::string& text); + bool IsEmpty() const { return m_terms.empty(); } + const std::vector& GetTerms() const { return m_terms; } + std::vector& GetTerms() { return m_terms; } + + private: + std::vector m_terms; + }; + + + class TTDBehaviorReport + { + public: + TTDBehaviorReport() = default; + ~TTDBehaviorReport(); + TTDBehaviorReport(const TTDBehaviorReport&) = delete; + TTDBehaviorReport& operator=(const TTDBehaviorReport&) = delete; + + // Maps the report at `path`. Returns false with `error` set; the object is left + // closed and safe to destroy. + bool Open(const std::string& path, std::string& error); + void Close(); + bool IsOpen() const { return m_map != nullptr; } + + const std::string& GetPath() const { return m_path; } + const std::string& GetTracePath() const { return m_tracePath; } + const std::string& GetArchitecture() const { return m_arch; } + uint64_t GetProcessId() const { return m_pid; } + uint64_t GetCallCount() const { return m_callCount; } + uint64_t GetDecodedCount() const { return m_decodedCount; } + uint64_t GetMaxSequence() const { return m_maxSeq; } + uint32_t GetMaxPositionChars() const { return m_maxPositionChars; } + + // Decode one call. `withParams` is the expensive half, so a table can skip it and + // only a detail view asks for it. + bool GetCall(uint64_t index, TTDApiCall& out, bool withParams) const; + + // Evaluate `query` over every call and return the matching row indices. One call + // does the whole filter, which is what keeps this affordable across the FFI. + std::vector RunQuery(const std::string& query) const; + + // Same, for a query already parsed and resolved against this report. + std::vector RunQuery(TTDBehaviorQuery& query) const; + + // Bind a query's module/api terms to this report's string table. + void ResolveQuery(TTDBehaviorQuery& query) const; + + bool Matches(uint64_t index, const TTDBehaviorQuery& query) const; + + private: + const uint8_t* CallRecord(uint64_t index) const; + std::string MappedString(uint32_t offset) const; + void ForEachString(const std::function& fn) const; + + std::string m_path; + std::string m_tracePath; + std::string m_arch; + + // Platform handles for the mapping, opaque here to keep windows.h out of the + // header. + void* m_fileHandle = nullptr; + void* m_mappingHandle = nullptr; + const uint8_t* m_map = nullptr; + uint64_t m_mapSize = 0; + + uint64_t m_callCount = 0; + uint64_t m_decodedCount = 0; + uint64_t m_maxSeq = 0; + uint64_t m_pid = 0; + uint32_t m_maxPositionChars = 0; + uint64_t m_callsOff = 0; + uint64_t m_paramsOff = 0; + uint64_t m_stringsOff = 0; + uint64_t m_stringsSize = 0; + uint64_t m_blobOff = 0; + uint64_t m_blobSize = 0; + }; + +} // namespace BinaryNinjaDebugger diff --git a/debuggerui.qrc b/debuggerui.qrc index 4040671a..7bc275ca 100644 --- a/debuggerui.qrc +++ b/debuggerui.qrc @@ -26,6 +26,7 @@ icons/ttd-calls.png icons/ttd-timestamp.png icons/ttd-events.png + icons/ttd-analysis.png icons/ttd-back.png icons/ttd-forward.png diff --git a/docs/guide/index.md b/docs/guide/index.md index 78e2209c..67958315 100644 --- a/docs/guide/index.md +++ b/docs/guide/index.md @@ -621,6 +621,11 @@ See [Time Travel Debugging Guide (Windows)](dbgeng-ttd.md) See [Time Travel Debugging Guide (Linux)](gdbrsp-ttd.md) +### TTD Behavior + +See [TTD Behavior Guide](ttd-behavior.md) for extracting and querying the Windows API calls +a TTD trace made. + ### Windows Kernel Debugging See [Windows Kernel Debugging Guide](windows-kd.md) diff --git a/docs/guide/ttd-behavior.md b/docs/guide/ttd-behavior.md new file mode 100644 index 00000000..f3cb9abe --- /dev/null +++ b/docs/guide/ttd-behavior.md @@ -0,0 +1,213 @@ +# TTD Behavior + +While TTD already makes reverse engineering so much easier, we still want better ways to +help us analyze the program. The **TTD Behavior** feature sweeps a trace once and records +every Windows API call the process made -- when it happened, which function was called, +what its arguments were, and what it returned -- into a report you can then query, sort +through, and jump into. + + + +Arguments are decoded against Microsoft's Win32 API metadata, so they come back named and +typed rather than as four guessed registers: + +``` +kernel32!CreateFileW(lpFileName: "C:\Users\me\AppData\Local\Temp\x.dat", + dwDesiredAccess: GENERIC_WRITE, + dwCreationDisposition: CREATE_ALWAYS, ...) -> 0xf4 +``` + +## Platform support + +| | Windows | Linux | macOS | +| --- | --- | --- | --- | +| Extracting a report from a trace | Yes | No | No | +| Opening and querying a report | Yes | Yes | Yes | + +Extraction replays the trace through Microsoft's TTD engine, which only exists on Windows, +so the **Extract...** button is hidden elsewhere. Reading a report is just a memory-mapped +file with no Windows dependency at all, so a report extracted on Windows can be copied to a +Linux or macOS machine and opened there normally, in the UI or from Python. + +## Extracting a report + +1. Open the **TTD Behavior** sidebar. +2. Click **Extract...**. +3. If the current session is already replaying a trace, that trace is used. Otherwise you + are asked for a `.run` file. + +The report is written next to the trace as `.ttdb`. If the trace sits somewhere +unwritable you are asked where to put it instead. + +Extraction runs in the background. A progress bar shows elapsed and estimated remaining +time, and **Cancel** stops the sweep while keeping everything recorded so far -- a +cancelled run produces a complete, valid report covering the portion of the trace that has +been processed. + +The extraction is quite performant, and generally faster than running multiple TTD Calls +queries in the old way. + +## Loading a report + +**Load Report...** opens any `.ttdb` file. Loading is a memory map rather than a parse, so +it is effectively instant regardless of report size. + +## Reading the table + +| Column | Meaning | +| --- | --- | +| `#` | index of the call within the report | +| `Position` | TTD position, `sequence:step` | +| `TID` | thread that made the call | +| `Module` | module the function was resolved in, e.g. `kernel32` | +| `Function` | function name | +| `Parameters` | decoded arguments | +| `Return` | return value | +| `Return Address` | the call site -- useful for isolating calls the sample made itself | + +Selecting a row shows the full detail below the table, including a hex dump of any buffer +argument. Buffer arguments also appear inline in the table as `0x1eefd98 -> [4d 5a 90 00 +…]`, so you can see at a glance what a `WriteFile` actually wrote. + +**Double-clicking a row time travels to that call and navigates to the instruction pointer +at that position**, which requires an active TTD session on the same trace. + +## Querying + +The filter box takes terms combined with AND. A bare word matches anywhere in the row, +including buffer contents. + +| Term | Matches | +| --- | --- | +| `module:kernel32` | exact module, so `ntdll!WriteFile` is excluded | +| `api:WriteFile` | exact function name | +| `api:Reg*` | trailing `*` matches a prefix | +| `tid:4` | thread id | +| `ret:!0` | return value; also `>` `>=` `<` `<=` and `10-20` ranges | +| `retaddr:0x400000-0x500000` | call site within a range | +| `"c:\windows"` | quoted phrase | + +Values may be decimal or `0x` hex. An unrecognised prefix is treated as text, so +`foo:bar` searches for that literal string rather than erroring. + +Queries run inside the core over the whole report; a full scan of a 3.4M-call report takes +60-190 ms, so results appear as you type. + +Each tab holds its own query. Use **+** to open another, which lets you keep one view of +`api:CreateFileW` beside another of `module:ws2_32` rather than retyping. + +## Settings + +| Setting | Default | Effect | +| --- | --- | --- | +| `debugger.ttdBehaviorExtractorPath` | empty | Path to the extractor executable. Empty uses the one shipped with the debugger; set it to run a build of your own. | +| `debugger.ttdBehaviorMaxBuffer` | 65536 | Bytes kept from any one buffer argument. Longer buffers are captured as a prefix and marked truncated. | + +## The Python API + +Reports can be opened and queried from Python on any platform, and extracted from Python +on Windows. + +```python +from debugger.ttdbehavior import TTDBehaviorReport + +with TTDBehaviorReport(r"C:\traces\sample.ttdb") as report: + print(f"{report.call_count} calls from pid {report.pid}, {report.architecture}") + + for call in report.query("api:CreateFileW"): + name = call.params[0].string if call.params else "?" + print(f"{call.position} {name} -> {call.ret:#x}") +``` + +### Extracting + +`extract()` runs the same sweep the **Extract...** button does, with the same defaults -- +the configured or bundled extractor, WinDbg's replay DLLs, and +`debugger.ttdBehaviorMaxBuffer`. It writes the report beside the trace and returns it +open: + +```python +from debugger.ttdbehavior import extract + +report = extract(r"C:\traces\sample.run") +print(f"{report.call_count} calls -> {report.path}") +``` + +Pass `progress` to follow along. It is called a few times a second with the phase +(`"sweep"` or `"write"`), a percentage, and the running call count; returning `False` +cancels, which stops the replay and writes what has been recorded so far: + +```python +def show(phase, percent, calls): + print(f"{phase} {percent}% ({calls} calls)") + return calls < 1_000_000 # stop once we have a million + +report = extract(r"C:\traces\sample.run", progress=show) +``` + +`output`, `extractor`, `ttd_dlls` and `max_buffer` override the corresponding defaults. +Off Windows, `extract()` raises `RuntimeError` -- there is no replay engine to sweep with. + +`query()` filters entirely inside the core and returns matching rows, decoding each one +only as you reach it. Iterating a multi-million-call report one call at a time is possible +but slow -- narrow with a query first. + +Calls come back with parameters decoded. On a query matching millions of rows that is +worth turning off: + +```python +for call in report.query("", with_params=False): + print(call.param_summary) # still populated; the structured list is skipped +``` + +Useful members: + +| | | +| --- | --- | +| `report.call_count`, `report.decoded_count` | totals; `decoded_count` counts calls that had a real signature | +| `report.pid`, `report.trace_path`, `report.architecture` | provenance | +| `report.path` | the `.ttdb` file this was opened from | +| `report.query_indices(q)` | just the row indices | +| `report.get_call(i)` | one call by index; `report[i]` also works | +| `call.module`, `call.api`, `call.tid`, `call.ret`, `call.return_address` | the call | +| `call.position` | `sequence:step` as a string | +| `call.decoded` | whether a real signature backed the parameters | +| `call.params` | list of `TTDApiCallParam` | +| `param.name`, `.type`, `.kind`, `.value` | one argument | +| `param.string`, `.flags`, `.data` | its resolved string, symbolic flags, or captured buffer | +| `param.truncated` | whether `data` is a prefix of a larger buffer | + +## What the data can and cannot tell you + +**An accessible buffer is not guaranteed to be extracted.** Due to the specific way the +extractor works (and some complex TTD shenanigans), buffers and strings that are indeed +readable may not always be properly recorded. This is a known limitation. When that +happens, time travelling to the timestamp yourself will read them fine. + +**Calls without a signature have guessed arguments.** Metadata coverage is the public +Windows SDK, so `ntdll` internals and CRT helpers fall back to capturing argument registers +(x64) or stack slots (x86). Those values are positional, unnamed, and the count is a guess. +Such calls are flagged in the report and counted by `decoded_count`. + +**Buffers are captured up to a limit.** See `debugger.ttdBehaviorMaxBuffer` above. A +truncated buffer still records its true length, so a short buffer is distinguishable from a +cut-off one. + +## Running the extractor directly + +The extractor ships with the debugger at `ttd-extract/ttdcapa-extract.exe` under the plugin +directory, alongside the `win32-index.bin` metadata it reads from beside itself. It is a +normal command-line tool and can be run without Binary Ninja: + +``` +ttdcapa-extract.exe trace.run -b report.ttdb --ttd-dlls \amd64\ttd +``` + +`TTDReplay.dll` and `TTDReplayCPU.dll` are Microsoft's, ship with WinDbg, and are not +redistributed with the debugger -- `--ttd-dlls` points at the copy WinDbg installed. Full +option reference and the `.ttdb` format specification are in `ttd-extract/README.md`. + +The extractor is built by CI from +[Vector35/ttd-capa](https://github.com/Vector35/ttd-capa), a fork of +[HullaBrian/ttd-capa](https://github.com/HullaBrian/ttd-capa). The README's footer records +the exact commit the shipped binary was built from. diff --git a/docs/img/debugger/ttd_behavior_widget.png b/docs/img/debugger/ttd_behavior_widget.png new file mode 100644 index 00000000..5b64d9cf Binary files /dev/null and b/docs/img/debugger/ttd_behavior_widget.png differ diff --git a/ui/ttdbehaviorwidget.cpp b/ui/ttdbehaviorwidget.cpp new file mode 100644 index 00000000..e09ee12f --- /dev/null +++ b/ui/ttdbehaviorwidget.cpp @@ -0,0 +1,1176 @@ +/* +Copyright 2020-2026 Vector 35 Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Shows the Windows API calls a TTD trace made, with their parameters decoded against +// Microsoft's Win32 API metadata: real arity, resolved strings, symbolic flag names, +// captured buffers, and [Out] parameters re-read at the call's return position. +// +// The sweep itself is done out-of-process by an extractor built on the TTD replay SDK +// (https://github.com/Vector35/ttd-capa), which writes a .ttdb report that the core +// memory-maps. Extraction takes minutes on a large trace, so it runs in the background +// rather than on the critical path of a debug session. + +#include "ttdbehaviorwidget.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "fontsettings.h" +#include "theme.h" + +using namespace BinaryNinja; +using namespace BinaryNinjaDebuggerAPI; + +// Settings key holding the path of the extractor executable. Registered by +// GlobalDebuggerUI::InitializeUI(). +static const char* kExtractorSetting = "debugger.ttdBehaviorExtractorPath"; + +// The extractor captures up to 64 KiB per buffer, which is 4096 lines of hex dump -- +// more than the detail pane can usefully show or lay out quickly. Clip the rendering, +// not the data: the full bytes stay in the report. +static const int kMaxHexDumpBytes = 4096; + + +namespace { + // QJsonValue stores anything that does not fit in a qint64 as a double, so pointer-sized + // sentinels such as 0xFFFFFFFFFFFFFFFE arrive rounded. Clamp instead of letting the + // conversion be undefined. + uint64_t jsonToUInt64(const QJsonValue& value) + { + if (value.isDouble()) + { + double d = value.toDouble(); + qint64 i = value.toInteger(0); + if (static_cast(i) == d) + return static_cast(i); + if (!std::isfinite(d) || d <= 0.0) + return 0; + if (d >= 18446744073709551616.0) + return UINT64_MAX; + return static_cast(d); + } + return 0; + } + + + QString formatHex(uint64_t value) + { + return QString("0x%1").arg(value, 0, 16); + } + + + QString escapeString(const QString& str) + { + QString out; + out.reserve(str.size()); + for (QChar c : str) + { + if (c == '\\') + out += "\\\\"; + else if (c == '"') + out += "\\\""; + else if (c == '\n') + out += "\\n"; + else if (c == '\r') + out += "\\r"; + else if (c == '\t') + out += "\\t"; + else if (c.unicode() < 0x20) + out += QString("\\x%1").arg(static_cast(c.unicode()), 2, 16, QChar('0')); + else + out += c; + } + return out; + } + + + QString formatPosition(const TTDApiCall& call) + { + return QString("%1:%2").arg(call.positionSequence, 0, 16).arg(call.positionSteps, 0, 16).toUpper(); + } + + + QByteArray toByteArray(const std::vector& bytes) + { + return QByteArray(reinterpret_cast(bytes.data()), static_cast(bytes.size())); + } + + + // Where TTDReplay.dll and TTDReplayCPU.dll live. They are Microsoft's, shipped with + // WinDbg rather than with us, so the extractor takes their location instead of + // requiring copies beside itself. This is the same folder ttdrecord.cpp uses to find + // the recorder. Returns empty when it cannot be found, in which case the extractor + // falls back to its own directory. + QString ttdReplayDirectory() + { + auto hasReplay = [](const QString& dir) { + return !dir.isEmpty() && QFileInfo::exists(QDir(dir).filePath("TTDReplay.dll")); + }; + + // A configured DbgEng path is authoritative: if it is set and wrong, fall through + // to the extractor's own search rather than silently using a different engine. + std::string configured = Settings::Instance()->Get("debugger.x64dbgEngPath"); + if (!configured.empty()) + { + QString path = QDir(QString::fromStdString(configured)).filePath("TTD"); + return hasReplay(path) ? path : QString(); + } + + std::string pluginRoot = getenv("BN_STANDALONE_DEBUGGER") != nullptr + ? GetUserPluginDirectory() + : GetBundledPluginDirectory(); + QString bundled = QDir(QString::fromStdString(pluginRoot)).filePath("dbgeng/amd64/TTD"); + return hasReplay(bundled) ? bundled : QString(); + } + + + QString hexDump(const QByteArray& bytes) + { + QString out; + for (int offset = 0; offset < bytes.size(); offset += 16) + { + QByteArray line = bytes.mid(offset, 16); + QString ascii; + for (char c : line) + ascii += (c >= 0x20 && c < 0x7f) ? QChar(c) : QChar('.'); + out += QString(" %1 %2 %3\n") + .arg(offset, 4, 16, QChar('0')) + .arg(QString::fromLatin1(line.toHex(' ')), -47) + .arg(ascii); + } + return out; + } +} // namespace + + +TTDBehaviorCallModel::TTDBehaviorCallModel(QObject* parent) : QAbstractTableModel(parent) {} + + +void TTDBehaviorCallModel::setReport(std::shared_ptr report) +{ + beginResetModel(); + m_report = std::move(report); + rebuildVisible(); + endResetModel(); +} + + +void TTDBehaviorCallModel::setFilter(const QString& text) +{ + if (text.trimmed() == m_filterText) + return; + + beginResetModel(); + m_filterText = text.trimmed(); + rebuildVisible(); + endResetModel(); +} + + +void TTDBehaviorCallModel::rebuildVisible() +{ + m_cachedIndex = static_cast(-1); + m_visible.clear(); + m_visible.shrink_to_fit(); + m_filtered = !m_filterText.isEmpty(); + if (!m_filtered || !m_report || !m_report->IsOpen()) + return; + + // One call filters the whole report on the core side and returns the matching rows. + // Measured at 8-70ms over 3.4M calls depending on the query, against 30-50% more if + // the boundary were crossed per row. + m_visible = m_report->RunQuery(m_filterText.toStdString()); +} + + +const TTDApiCall* TTDBehaviorCallModel::callAt(int row, bool withParams) const +{ + if (!m_report || !m_report->IsOpen() || row < 0) + return nullptr; + size_t index = static_cast(row); + if (m_filtered) + { + if (index >= m_visible.size()) + return nullptr; + index = static_cast(m_visible[index]); + } + if (index >= m_report->GetCallCount()) + return nullptr; + + // No stored objects to point at, so decode into a one-row cache. data() is called + // once per column, so without this a row would be decoded seven times per repaint. + if (m_cachedIndex != index || (withParams && !m_cachedHasParams)) + { + m_cached = TTDApiCall(); + if (!m_report->GetCall(index, m_cached, withParams)) + return nullptr; + m_cachedIndex = index; + m_cachedHasParams = withParams; + } + return &m_cached; +} + + +int TTDBehaviorCallModel::rowCount(const QModelIndex& parent) const +{ + if (parent.isValid() || !m_report) + return 0; + return static_cast(m_filtered ? m_visible.size() : m_report->GetCallCount()); +} + + +int TTDBehaviorCallModel::columnCount(const QModelIndex& parent) const +{ + if (parent.isValid()) + return 0; + return ColumnCount; +} + + +QVariant TTDBehaviorCallModel::data(const QModelIndex& index, int role) const +{ + const TTDApiCall* call = callAt(index.row()); + if (!call) + return QVariant(); + + if (role == Qt::ToolTipRole) + return QString("%1!%2(%3)").arg(call->module, call->api, call->paramSummary); + + if (role != Qt::DisplayRole) + return QVariant(); + + switch (index.column()) + { + case SeqColumn: + return QString::number(call->seq); + case PositionColumn: + return QString("%1:%2").arg(call->positionSequence, 0, 16).arg(call->positionSteps, 0, 16).toUpper(); + case ThreadColumn: + return QString::number(call->tid); + case ModuleColumn: + return QString::fromStdString(call->module); + case ApiColumn: + return QString::fromStdString(call->api); + case ParametersColumn: + return QString::fromStdString(call->paramSummary); + case ReturnColumn: + return formatHex(call->ret); + case ReturnAddressColumn: + return formatHex(call->returnAddress); + default: + return QVariant(); + } +} + + +QVariant TTDBehaviorCallModel::headerData(int section, Qt::Orientation orientation, int role) const +{ + if (orientation != Qt::Horizontal || role != Qt::DisplayRole) + return QVariant(); + + switch (section) + { + case SeqColumn: + return "#"; + case PositionColumn: + return "Position"; + case ThreadColumn: + return "TID"; + case ModuleColumn: + return "Module"; + case ApiColumn: + return "Function"; + case ParametersColumn: + return "Parameters"; + case ReturnColumn: + return "Return"; + case ReturnAddressColumn: + return "Return Address"; + default: + return QVariant(); + } +} + + +TTDBehaviorQueryWidget::TTDBehaviorQueryWidget(QWidget* parent, BinaryViewRef data) : + QWidget(parent), m_data(data) +{ + m_controller = DebuggerController::GetController(data); + m_report = std::make_shared(); + setupUI(); + updateStatus(); +} + + +void TTDBehaviorQueryWidget::setupUI() +{ + auto* layout = new QVBoxLayout(); + layout->setContentsMargins(0, 0, 0, 0); + layout->setSpacing(4); + + m_filterEdit = new QLineEdit(); + m_filterEdit->setPlaceholderText("Filter, e.g. module:kernel32 api:WriteFile ret:!0"); + // The syntax is the only part of this that is not self-evident, so spell it out + // where someone hovering the box will find it. + m_filterEdit->setToolTip(QStringList { + "Terms are combined with AND. A bare word matches anywhere, including buffer contents.", + "", + " module:kernel32 exact module, so ntdll!WriteFile is excluded", + " api:WriteFile exact function name", + " api:Reg* trailing * matches a prefix", + " tid:4 thread id", + " ret:!0 ret:>0x1000 return value; also >= <= < and 10-20 ranges", + " retaddr:0x400000-0x500000 call site, e.g. calls the sample made itself", + " \"c:\\windows\" quoted phrase", + "", + "Values may be decimal or 0x hex. An unrecognised prefix is treated as text.", + }.join('\n')); + m_filterEdit->setClearButtonEnabled(true); + m_filterEdit->setContentsMargins(4, 4, 4, 0); + layout->addWidget(m_filterEdit); + + m_model = new TTDBehaviorCallModel(this); + + // A full scan of a 3.4M-call report measures 60-190ms, so filtering can run + // synchronously and appear immediate. The short debounce is only there to coalesce + // a burst of keystrokes into one pass; it is below the threshold where waiting is + // perceptible, so it costs nothing on smaller reports. + m_filterTimer = new QTimer(this); + m_filterTimer->setSingleShot(true); + m_filterTimer->setInterval(150); + + m_table = new QTableView(); + m_table->setModel(m_model); + m_table->setSelectionBehavior(QAbstractItemView::SelectRows); + m_table->setSelectionMode(QAbstractItemView::ExtendedSelection); + m_table->setAlternatingRowColors(true); + m_table->setSortingEnabled(false); + m_table->setWordWrap(false); + m_table->setContextMenuPolicy(Qt::CustomContextMenu); + m_table->verticalHeader()->setVisible(false); + m_table->verticalHeader()->setDefaultSectionSize(QFontMetrics(getMonospaceFont(this)).height() + 4); + m_table->horizontalHeader()->setStretchLastSection(false); + m_table->setFont(getMonospaceFont(this)); + + m_detail = new QTextEdit(); + m_detail->setReadOnly(true); + m_detail->setLineWrapMode(QTextEdit::NoWrap); + m_detail->setFont(getMonospaceFont(this)); + m_detail->setPlaceholderText("Select a call to see its decoded parameters"); + + auto* splitter = new QSplitter(Qt::Vertical); + splitter->addWidget(m_table); + splitter->addWidget(m_detail); + splitter->setStretchFactor(0, 3); + splitter->setStretchFactor(1, 1); + layout->addWidget(splitter, 1); + + m_statusLabel = new QLabel(); + m_statusLabel->setContentsMargins(4, 0, 4, 4); + layout->addWidget(m_statusLabel); + + setLayout(layout); + + connect(m_filterEdit, &QLineEdit::textChanged, this, &TTDBehaviorQueryWidget::onFilterTextEdited); + // Enter applies immediately rather than waiting out the debounce. + connect(m_filterEdit, &QLineEdit::returnPressed, this, &TTDBehaviorQueryWidget::applyFilter); + connect(m_filterTimer, &QTimer::timeout, this, &TTDBehaviorQueryWidget::applyFilter); + connect(m_table, &QTableView::doubleClicked, this, &TTDBehaviorQueryWidget::onDoubleClicked); + connect(m_table, &QTableView::customContextMenuRequested, this, &TTDBehaviorQueryWidget::onContextMenu); + connect(m_table->selectionModel(), &QItemSelectionModel::selectionChanged, this, + &TTDBehaviorQueryWidget::onSelectionChanged); +} + + +QString TTDBehaviorQueryWidget::filterText() const +{ + return m_filterEdit->text(); +} + + +void TTDBehaviorQueryWidget::setFilterText(const QString& text) +{ + m_filterEdit->setText(text); + applyFilter(); +} + + +void TTDBehaviorQueryWidget::setTimings(double writeSeconds, double loadSeconds) +{ + m_writeSeconds = writeSeconds; + m_loadSeconds = loadSeconds; + updateStatus(); +} + + +void TTDBehaviorQueryWidget::setReport(std::shared_ptr report, const QString& path) +{ + m_report = std::move(report); + m_reportPath = path; + m_model->setReport(m_report); + m_detail->clear(); + m_table->resizeColumnsToContents(); + // The parameter rendering is long enough that fitting it would push every other + // column off screen; cap it and let the detail pane carry the full value. + if (m_table->columnWidth(TTDBehaviorCallModel::ParametersColumn) > 600) + m_table->setColumnWidth(TTDBehaviorCallModel::ParametersColumn, 600); + + // resizeColumnsToContents() samples only the leading rows, which is fine for the + // columns whose content is uniform but clips these two: the last call's index has + // far more digits than the first thousand, and positions grow as the trace advances. + // Size them from the widest value actually present. + QFontMetrics metrics(m_table->font()); + const int padding = 16; + int seqWidth = metrics.horizontalAdvance(QString::number(m_report->GetMaxSequence())) + padding; + int positionWidth = metrics.horizontalAdvance(QString(m_report->GetMaxPositionChars(), 'M')) + padding; + if (seqWidth > m_table->columnWidth(TTDBehaviorCallModel::SeqColumn)) + m_table->setColumnWidth(TTDBehaviorCallModel::SeqColumn, seqWidth); + if (positionWidth > m_table->columnWidth(TTDBehaviorCallModel::PositionColumn)) + m_table->setColumnWidth(TTDBehaviorCallModel::PositionColumn, positionWidth); + + updateStatus(); +} + + +TTDBehaviorWidget::TTDBehaviorWidget(BinaryViewRef data) : SidebarWidget("TTD Behavior"), m_data(data) +{ + m_controller = DebuggerController::GetController(data); + m_report = std::make_shared(); + setupUI(); +} + + +TTDBehaviorWidget::~TTDBehaviorWidget() +{ + if (m_extractProcess && m_extractProcess->state() != QProcess::NotRunning) + { + m_extractProcess->kill(); + m_extractProcess->waitForFinished(1000); + } +} + + +void TTDBehaviorWidget::setupUI() +{ + auto* layout = new QVBoxLayout(); + layout->setContentsMargins(0, 0, 0, 0); + layout->setSpacing(4); + + // Loading and extracting act on the report as a whole, so they live above the tabs + // rather than being repeated in each one. + auto* toolbar = new QHBoxLayout(); + toolbar->setContentsMargins(4, 4, 4, 0); + + m_loadButton = new QPushButton("Load Report..."); + m_loadButton->setToolTip("Load a report of API calls extracted from a TTD trace"); + toolbar->addWidget(m_loadButton); + + m_extractButton = new QPushButton("Extract..."); + m_extractButton->setToolTip("Run the extractor over a TTD trace and load the result"); + toolbar->addWidget(m_extractButton); +#ifndef WIN32 + // Extraction replays the trace through WinDbg TTD, which only works on Windows + m_extractButton->setVisible(false); +#endif + toolbar->addStretch(1); + + layout->addLayout(toolbar); + + m_tabWidget = new QTabWidget(); + m_tabWidget->setTabsClosable(true); + + m_newTabButton = new QToolButton(); + m_newTabButton->setText("+"); + m_newTabButton->setAutoRaise(true); + m_newTabButton->setToolTip("New query tab"); + m_tabWidget->setCornerWidget(m_newTabButton, Qt::TopRightCorner); + + layout->addWidget(m_tabWidget, 1); + + // Progress row: hidden until something long-running starts. Shared, because a load + // or an extraction affects every tab. + m_progressRow = new QWidget(); + auto* progressLayout = new QHBoxLayout(m_progressRow); + progressLayout->setContentsMargins(4, 0, 4, 0); + m_progressBar = new QProgressBar(); + m_progressBar->setRange(0, 100); + m_progressBar->setTextVisible(false); + progressLayout->addWidget(m_progressBar, 1); + m_progressLabel = new QLabel(); + progressLayout->addWidget(m_progressLabel); + m_cancelButton = new QPushButton("Cancel"); + m_cancelButton->setToolTip("Stop the extraction and keep the calls recorded so far"); + progressLayout->addWidget(m_cancelButton); + m_progressRow->setVisible(false); + layout->addWidget(m_progressRow); + + setLayout(layout); + + connect(m_loadButton, &QPushButton::clicked, this, &TTDBehaviorWidget::onLoadClicked); + connect(m_extractButton, &QPushButton::clicked, this, &TTDBehaviorWidget::onExtractClicked); + connect(m_cancelButton, &QPushButton::clicked, this, &TTDBehaviorWidget::onCancelClicked); + connect(m_newTabButton, &QToolButton::clicked, this, &TTDBehaviorWidget::createNewTab); + connect(m_tabWidget, &QTabWidget::tabCloseRequested, this, &TTDBehaviorWidget::closeTab); + + createNewTab(); +} + + +TTDBehaviorQueryWidget* TTDBehaviorWidget::currentQuery() const +{ + return qobject_cast(m_tabWidget->currentWidget()); +} + + +void TTDBehaviorWidget::createNewTab() +{ + // Seed from the current tab, so refining a query is a matter of opening a tab and + // editing rather than retyping it. + QString seed; + if (TTDBehaviorQueryWidget* current = currentQuery()) + seed = current->filterText(); + + auto* query = new TTDBehaviorQueryWidget(this, m_data); + int index = m_tabWidget->addTab(query, QString("Query %1").arg(m_tabWidget->count() + 1)); + + query->setReport(m_report, m_reportPath); + query->setTimings(m_lastWriteSeconds, m_lastLoadSeconds); + if (!seed.isEmpty()) + query->setFilterText(seed); + + // Label the tab with what it is asking, which is far more use than "Query 3" once + // there are several. + connect(query, &TTDBehaviorQueryWidget::filterApplied, this, [this, query](const QString& text) { + int at = m_tabWidget->indexOf(query); + if (at < 0) + return; + QString label = text.trimmed(); + if (label.isEmpty()) + label = QString("Query %1").arg(at + 1); + else if (label.size() > 24) + label = label.left(24) + QString::fromUtf8("\xe2\x80\xa6"); + m_tabWidget->setTabText(at, label); + m_tabWidget->setTabToolTip(at, text.trimmed()); + }); + + m_tabWidget->setCurrentIndex(index); +} + + +void TTDBehaviorWidget::closeTab(int index) +{ + // Keep at least one, so the widget is never an empty frame. + if (m_tabWidget->count() <= 1) + return; + QWidget* widget = m_tabWidget->widget(index); + m_tabWidget->removeTab(index); + widget->deleteLater(); +} + + +void TTDBehaviorQueryWidget::updateStatus() +{ + if (m_report->GetCallCount() == 0) + { + m_statusLabel->setText("No report loaded"); + return; + } + + int shown = m_model->rowCount(); + QString name = QFileInfo(m_reportPath).fileName(); + QString text = QString("%1: %2 of %3 calls shown, %4 with decoded parameters") + .arg(name) + .arg(shown) + .arg(m_report->GetCallCount()) + .arg(m_report->GetDecodedCount()); + // Where the time actually went, since that is the thing worth knowing about a format. + if (m_writeSeconds > 0.0) + text += QString(" | saved in %1s").arg(m_writeSeconds, 0, 'f', 1); + if (m_loadSeconds >= 0.0) + text += QString("%1loaded in %2s") + .arg(m_writeSeconds > 0.0 ? ", " : " | ") + .arg(m_loadSeconds, 0, 'f', 2); + m_statusLabel->setText(text); +} + + +namespace { + // "1m 24s" / "12s" -- short enough to sit in a status row. + QString formatDuration(qint64 milliseconds) + { + qint64 seconds = milliseconds / 1000; + if (seconds < 60) + return QString("%1s").arg(seconds); + return QString("%1m %2s").arg(seconds / 60).arg(seconds % 60); + } +} // namespace + + +void TTDBehaviorWidget::beginOperation(const QString& what, bool cancellable) +{ + m_operationTimer.start(); + m_progressBar->setRange(0, 100); + m_progressBar->setValue(0); + m_progressLabel->setText(what); + m_cancelButton->setVisible(cancellable); + m_cancelButton->setEnabled(cancellable); + m_progressRow->setVisible(true); + m_loadButton->setEnabled(false); + m_extractButton->setEnabled(false); +} + + +void TTDBehaviorWidget::endOperation() +{ + m_progressRow->setVisible(false); + m_loadButton->setEnabled(true); + m_extractButton->setEnabled(true); +} + + +void TTDBehaviorWidget::setProgress(double percent, const QString& detail) +{ + qint64 elapsed = m_operationTimer.elapsed(); + QString text = detail; + text += QString(" %1 elapsed").arg(formatDuration(elapsed)); + + if (percent > 0.0) + { + m_progressBar->setRange(0, 100); + m_progressBar->setValue(static_cast(percent)); + // Linear extrapolation from the work done so far. Crude, but the sweep rate is + // steady enough for it to be useful, and anything cleverer would still be a + // guess. + if (percent >= 1.0 && percent < 100.0) + { + qint64 remaining = static_cast(elapsed * (100.0 - percent) / percent); + text += QString(", ~%1 left").arg(formatDuration(remaining)); + } + } + else + { + // No measurable progress to report: a busy indicator beats a bar stuck at zero. + m_progressBar->setRange(0, 0); + } + + m_progressLabel->setText(text); +} + + +void TTDBehaviorWidget::consumeExtractorStderr() +{ + if (!m_extractProcess) + return; + + m_stderrTail += m_extractProcess->readAllStandardError(); + int newline = -1; + while ((newline = m_stderrTail.indexOf('\n')) >= 0) + { + QString line = QString::fromLocal8Bit(m_stderrTail.left(newline)).trimmed(); + m_stderrTail.remove(0, newline + 1); + + if (line.startsWith("[progress]")) + { + QStringList parts = line.split(' ', Qt::SkipEmptyParts); + if (parts.size() >= 3) + { + bool ok = false; + double percent = parts[1].toDouble(&ok); + if (ok) + setProgress(percent, QString("Sweeping trace: %1 calls").arg(parts[2])); + } + } + else if (line.startsWith("[timing] write")) + { + QStringList parts = line.split(' ', Qt::SkipEmptyParts); + if (parts.size() >= 3) + m_lastWriteSeconds = parts[2].toDouble(); + } + else if (line == "[phase] strings") + { + setProgress(0.0, "Recovering strings the sweep could not read"); + m_cancelButton->setEnabled(false); + } + else if (line == "[phase] write") + { + // The report can be hundreds of MB; serialising it takes comparable time to + // the sweep, and there is no progress to be had from it. + setProgress(0.0, "Writing report"); + m_cancelButton->setEnabled(false); + } + } +} + + +void TTDBehaviorWidget::loadReport(const QString& path) +{ + beginOperation(QString("Loading %1").arg(QFileInfo(path).fileName()), false); + setProgress(0.0, "Parsing report"); + + // Parsing a 650MB report takes tens of seconds; on the UI thread that is a freeze. + // The report is self-contained, so build it on a worker and hand the finished object + // back. QPointer guards the widget being destroyed while the worker runs. + QPointer self(this); + std::thread([self, path]() { + QElapsedTimer timer; + timer.start(); + auto report = std::make_shared(); + std::string loadError; + bool ok = report->Open(path.toStdString(), loadError); + QString error = QString::fromStdString(loadError); + double loadSeconds = timer.elapsed() / 1000.0; + + // Back to the UI thread to install it. + QMetaObject::invokeMethod( + QCoreApplication::instance(), + [self, report, error, ok, loadSeconds, path]() { + if (!self) + return; + self->endOperation(); + if (!ok) + { + QMessageBox::warning(self, "TTD Behavior", error); + return; + } + self->m_lastLoadSeconds = loadSeconds; + self->installReport(report, path); + }, + Qt::QueuedConnection); + }).detach(); +} + + +void TTDBehaviorWidget::installReport(std::shared_ptr report, const QString& path) +{ + m_report = std::move(report); + m_reportPath = path; + // Every tab points at the same report -- that is the reason to have tabs at all -- + // but each keeps its own query, so each re-resolves and re-filters against it. + for (int i = 0; i < m_tabWidget->count(); ++i) + { + if (auto* query = qobject_cast(m_tabWidget->widget(i))) + { + query->setReport(m_report, m_reportPath); + query->setTimings(m_lastWriteSeconds, m_lastLoadSeconds); + } + } +} + + +void TTDBehaviorWidget::onLoadClicked() +{ + QString path = QFileDialog::getOpenFileName( + this, "Load TTD API Call Report", QString(), "TTD behavior reports (*.ttdb *.json);;All files (*)"); + if (!path.isEmpty()) + loadReport(path); +} + + +QString TTDBehaviorWidget::extractorPath(bool prompt) +{ + auto settings = Settings::Instance(); + + // A configured path wins outright, so someone running their own build always gets it. + // If it is set but wrong, say so rather than quietly falling back to the bundled one + // and reporting results from a different binary than they think. + QString configured = QString::fromStdString(settings->Get(kExtractorSetting)); + if (!configured.isEmpty()) + { + if (QFileInfo(configured).isExecutable()) + return configured; + if (prompt) + { + QMessageBox::warning(this, "TTD Behavior", + QString("%1 is set to\n\n%2\n\nwhich is not an executable. Clear the setting to use " + "the extractor shipped with the debugger.") + .arg(kExtractorSetting, configured)); + } + return QString(); + } + + // Otherwise the copy CMake stages beside debuggercore. See core/ttd-extract/README.md. + std::string pluginRoot = getenv("BN_STANDALONE_DEBUGGER") != nullptr + ? GetUserPluginDirectory() + : GetBundledPluginDirectory(); + QString bundled = + QDir(QString::fromStdString(pluginRoot)).filePath("ttd-extract/ttdcapa-extract.exe"); + if (QFileInfo(bundled).isExecutable()) + return bundled; + + if (!prompt) + return QString(); + + // Only reached if the install is incomplete, so let them point at one by hand. + QString path = QFileDialog::getOpenFileName(this, "Locate the TTD API call extractor", QString(), +#ifdef WIN32 + "Executables (*.exe);;All files (*)"); +#else + "All files (*)"); +#endif + if (!path.isEmpty()) + settings->Set(kExtractorSetting, path.toStdString()); + return path; +} + + +void TTDBehaviorWidget::onExtractClicked() +{ + if (m_extractProcess && m_extractProcess->state() != QProcess::NotRunning) + { + QMessageBox::information(this, "TTD Behavior", "An extraction is already running."); + return; + } + + QString extractor = extractorPath(true); + if (extractor.isEmpty()) + return; + + // Prefer the trace the current session is replaying; fall back to asking. + QString trace; + if (m_controller && m_controller->IsTTD()) + { + auto adapterSettings = m_controller->GetAdapterSettings(); + if (adapterSettings) + { + BNSettingsScope scope = SettingsResourceScope; + trace = QString::fromStdString(adapterSettings->Get("launch.trace_path", m_data, &scope)); + } + } + if (trace.isEmpty() || !QFileInfo::exists(trace)) + { + trace = QFileDialog::getOpenFileName( + this, "Select TTD Trace", QString(), "TTD traces (*.run);;All files (*)"); + } + if (trace.isEmpty()) + return; + + // Next to the trace, not in a temp directory: extraction takes minutes on a large + // trace, the report is the useful artifact, and it belongs with the trace it came + // from rather than somewhere the OS will eventually clean up. + QFileInfo traceInfo(trace); + QDir traceDir = traceInfo.absoluteDir(); + QString output = traceDir.filePath(traceInfo.completeBaseName() + ".ttdb"); + + // A trace can sit somewhere unwritable -- a read-only share, or a mounted image. + // Ask rather than silently falling back to a temp file, which is the thing we are + // deliberately not doing. + if (!QFileInfo(traceDir.absolutePath()).isWritable()) + { + output = QFileDialog::getSaveFileName(this, "Save Extracted Report As", output, + "TTD behavior reports (*.ttdb);;All files (*)"); + if (output.isEmpty()) + return; + } + else if (QFileInfo::exists(output)) + { + // Re-extracting is deterministic, so the existing report is as good as a fresh + // one -- offer to just load it instead of spending the minutes again. + QMessageBox box(this); + box.setWindowTitle("TTD Behavior"); + box.setText(QString("%1 already exists.").arg(QFileInfo(output).fileName())); + box.setInformativeText("Load the existing report, or extract again and replace it?"); + QPushButton* loadButton = box.addButton("Load Existing", QMessageBox::AcceptRole); + QPushButton* replaceButton = box.addButton("Extract Again", QMessageBox::DestructiveRole); + box.addButton(QMessageBox::Cancel); + box.setDefaultButton(loadButton); + box.exec(); + + if (box.clickedButton() == loadButton) + { + loadReport(output); + return; + } + if (box.clickedButton() != replaceButton) + return; + } + + m_extractProcess = new QProcess(this); + // The extractor loads its API metadata index and the TTD replay DLLs from its own + // directory, so it has to run from there. + m_extractProcess->setWorkingDirectory(QFileInfo(extractor).absolutePath()); + m_stderrTail.clear(); + + connect(m_extractProcess, &QProcess::readyReadStandardError, this, + &TTDBehaviorWidget::consumeExtractorStderr); + connect(m_extractProcess, &QProcess::finished, this, + [this, output](int exitCode, QProcess::ExitStatus status) { + consumeExtractorStderr(); + QString stderrText = QString::fromLocal8Bit(m_extractProcess->readAllStandardError()); + m_extractProcess->deleteLater(); + m_extractProcess = nullptr; + endOperation(); + + if (status != QProcess::NormalExit || exitCode != 0) + { + QMessageBox::warning(this, "TTD Behavior", + QString("Extraction failed (exit %1):\n\n%2").arg(exitCode).arg(stderrText)); + return; + } + // A cancelled sweep still writes what it collected, so this path is the same + // whether the run completed or was stopped early. + loadReport(output); + }); + + // --progress drives the bar below; --cancel-on-stdin lets Cancel stop the sweep and + // still keep everything recorded up to that point. + QStringList arguments {trace, "-b", output, "--progress", "--cancel-on-stdin"}; + int64_t maxBuffer = Settings::Instance()->Get("debugger.ttdBehaviorMaxBuffer"); + if (maxBuffer > 0) + arguments << "--max-buffer" << QString::number(maxBuffer); + // Without this a quarter of string parameters come back empty, because the sweep + // reads memory through an interface the SDK restricts to a fast, incomplete lookup. + // Point the extractor at WinDbg's replay engine when we know where it is; otherwise + // let it fall back to looking beside itself. + QString ttdDlls = ttdReplayDirectory(); + if (!ttdDlls.isEmpty()) + arguments << "--ttd-dlls" << QDir::toNativeSeparators(ttdDlls); + + beginOperation(QString("Extracting from %1").arg(QFileInfo(trace).fileName()), true); + setProgress(0.0, "Starting extractor"); + m_extractProcess->start(extractor, arguments); +} + + +void TTDBehaviorWidget::onCancelClicked() +{ + if (!m_extractProcess || m_extractProcess->state() == QProcess::NotRunning) + return; + + // The extractor watches stdin for this and interrupts the replay, then writes out + // everything it recorded. Killing the process instead would throw that away. + m_extractProcess->write("cancel\n"); + m_cancelButton->setEnabled(false); + setProgress(0.0, "Finishing up the calls recorded so far"); +} + + +void TTDBehaviorQueryWidget::onFilterTextEdited() +{ + m_filterTimer->start(); + // Only worth saying on a report big enough for the pass to be visible; below that + // the label would flicker for no reason. + if (m_report->GetCallCount() > 250000) + m_statusLabel->setText("Filtering..."); +} + + +void TTDBehaviorQueryWidget::applyFilter() +{ + m_filterTimer->stop(); + m_model->setFilter(m_filterEdit->text()); + updateStatus(); + emit filterApplied(m_filterEdit->text()); +} + + +void TTDBehaviorQueryWidget::showDetail(const TTDApiCall* call) +{ + if (!call) + { + m_detail->clear(); + return; + } + + QString text; + text += QString("%1!%2 @ %3 tid %4 (#%5)\n") + .arg(QString::fromStdString(call->module), QString::fromStdString(call->api), formatPosition(*call)) + .arg(call->tid) + .arg(call->seq); + text += QString("returned %1, to %2\n\n").arg(formatHex(call->ret), formatHex(call->returnAddress)); + + if (!call->decoded) + { + text += "No signature was available for this function; the values below are the\n"; + text += "raw argument registers captured heuristically.\n\n"; + text += QString(" %1\n").arg(call->paramSummary); + m_detail->setPlainText(text); + return; + } + + for (const TTDApiCallParam& param : call->params) + { + QString annotations; + if (param.out) + annotations += " [out]"; + if (param.atReturn) + annotations += " [read at return]"; + + text += QString("%1 : %2%3\n") + .arg(QString::fromStdString(param.name), QString::fromStdString(param.type), annotations); + text += QString(" value %1\n").arg(formatHex(param.value)); + if (!param.str.empty()) + text += QString(" string \"%1\"\n").arg(escapeString(QString::fromStdString(param.str))); + if (!param.flags.empty()) + { + QStringList names; + for (const std::string& flag : param.flags) + names.append(QString::fromStdString(flag)); + text += QString(" flags %1\n").arg(names.join(" | ")); + } + if (param.hasDeref) + text += QString(" deref %1 (%2)\n").arg(formatHex(param.deref)).arg(param.deref); + if (!param.bytes.empty()) + { + // Say so when this is only the head of a larger buffer: reporting the + // captured size alone reads as the buffer's real size. + if (param.bytesTotal > static_cast(param.bytes.size())) + { + text += QString(" buffer first %1 of %2 bytes (raise the extractor's --max-buffer for more)\n") + .arg(param.bytes.size()) + .arg(param.bytesTotal); + } + else + { + text += QString(" buffer %1 bytes\n").arg(param.bytes.size()); + } + + QByteArray shown = toByteArray(param.bytes).left(kMaxHexDumpBytes); + text += hexDump(shown); + if (shown.size() < param.bytes.size()) + { + text += QString(" ... %1 more captured bytes not shown\n") + .arg(param.bytes.size() - shown.size()); + } + } + text += "\n"; + } + + m_detail->setPlainText(text); +} + + +void TTDBehaviorQueryWidget::onSelectionChanged() +{ + QModelIndexList selected = m_table->selectionModel()->selectedRows(); + if (selected.isEmpty()) + { + showDetail(nullptr); + return; + } + showDetail(m_model->callAt(selected.first().row(), true)); +} + + +void TTDBehaviorQueryWidget::onDoubleClicked(const QModelIndex& index) +{ + const TTDApiCall* call = m_model->callAt(index.row()); + if (!call || !m_controller || !m_controller->IsTTD()) + return; + + QStringList parts = formatPosition(*call).split(':'); + if (parts.size() != 2) + return; + + bool sequenceOk = false, stepOk = false; + uint64_t sequence = parts[0].toULongLong(&sequenceOk, 16); + uint64_t step = parts[1].toULongLong(&stepOk, 16); + if (!sequenceOk || !stepOk) + return; + + if (!m_controller->SetTTDPosition(TTDPosition(sequence, step))) + { + m_statusLabel->setText(QString("Failed to travel to %1").arg(formatPosition(*call))); + return; + } + + // SetTTDPosition drives the backend's !tt directly and posts no stop event, so + // nothing else moves the view. Read the instruction pointer back from the adapter -- + // it is queried live, so it reflects the position we just landed on -- and navigate + // there ourselves. + uint64_t ip = m_controller->IP(); + BinaryViewRef liveView = m_controller->GetData(); + ViewFrame* frame = ViewFrame::viewFrameForWidget(this); + if (frame && liveView) + { + // The position of a call is the callee's entry, which for a system DLL is + // usually somewhere analysis has not defined a function yet. + if (liveView->GetAnalysisFunctionsContainingAddress(ip).empty() + && !m_controller->FunctionExistsInOldView(ip)) + { + auto id = liveView->BeginUndoActions(); + liveView->CreateUserFunction(liveView->GetDefaultPlatform(), ip); + liveView->ForgetUndoActions(id); + } + frame->navigate(liveView, ip, true, true); + } + + m_statusLabel->setText(QString("Traveled to %1, IP 0x%2").arg(formatPosition(*call)).arg(ip, 0, 16)); +} + + +void TTDBehaviorQueryWidget::copySelectedRows() +{ + QModelIndexList selected = m_table->selectionModel()->selectedRows(); + QStringList lines; + for (const QModelIndex& index : selected) + { + const TTDApiCall* call = m_model->callAt(index.row()); + if (!call) + continue; + lines.append(QString("%1 %2!%3(%4) -> %5") + .arg(formatPosition(*call), QString::fromStdString(call->module), QString::fromStdString(call->api), + QString::fromStdString(call->paramSummary), formatHex(call->ret))); + } + if (!lines.isEmpty()) + QApplication::clipboard()->setText(lines.join('\n')); +} + + +void TTDBehaviorQueryWidget::onContextMenu(const QPoint& pos) +{ + if (m_table->selectionModel()->selectedRows().isEmpty()) + return; + + QMenu menu(this); + QAction* copyAction = menu.addAction("Copy"); + QAction* navigateAction = nullptr; + if (m_controller && m_controller->IsTTD()) + navigateAction = menu.addAction("Time Travel Here"); + + QAction* chosen = menu.exec(m_table->viewport()->mapToGlobal(pos)); + if (!chosen) + return; + if (chosen == copyAction) + copySelectedRows(); + else if (chosen == navigateAction) + onDoubleClicked(m_table->selectionModel()->selectedRows().first()); +} + + +TTDBehaviorWidgetType::TTDBehaviorWidgetType() : + SidebarWidgetType(QImage(":/debugger/ttd-analysis"), "TTD Behavior") +{} + + +SidebarWidget* TTDBehaviorWidgetType::createWidget(ViewFrame* frame, BinaryViewRef data) +{ + return new TTDBehaviorWidget(data); +} diff --git a/ui/ttdbehaviorwidget.h b/ui/ttdbehaviorwidget.h new file mode 100644 index 00000000..35a57bff --- /dev/null +++ b/ui/ttdbehaviorwidget.h @@ -0,0 +1,245 @@ +/* +Copyright 2020-2026 Vector 35 Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "binaryninjaapi.h" +#include "debuggerapi.h" +#include "viewframe.h" +#include "debuggeruicommon.h" +#include "uitypes.h" + +using namespace BinaryNinja; +using namespace BinaryNinjaDebuggerAPI; + + +// The report reader and its query engine live in core (core/ttdbehavior.h), reachable +// here through the debugger API. That is what lets the same queries run from Python; +// this widget is now just a view over them. +// +// Rows are decoded from the mapped file on demand rather than held as objects, so the +// model asks the report for a row when it needs to paint one. +using TTDApiCall = BinaryNinjaDebuggerAPI::TTDApiCall; +using TTDApiCallParam = BinaryNinjaDebuggerAPI::TTDApiCallParam; +using TTDBehaviorReport = BinaryNinjaDebuggerAPI::TTDBehaviorReport; + + +class TTDBehaviorCallModel : public QAbstractTableModel +{ + Q_OBJECT + +public: + enum Column + { + SeqColumn = 0, + PositionColumn, + ThreadColumn, + ModuleColumn, + ApiColumn, + ParametersColumn, + ReturnColumn, + ReturnAddressColumn, + ColumnCount + }; + + TTDBehaviorCallModel(QObject* parent); + + void setReport(std::shared_ptr report); + + // Runs `text` as a query against the report and keeps the rows it matched. + // + // Deliberately not a QSortFilterProxyModel: that class maintains a bidirectional + // source/proxy row mapping, and any call to its rowCount() materialises the whole + // thing, which on a multi-million-call report made every keystroke look like a + // freeze. The core returns the matching indices in one call instead. + void setFilter(const QString& text); + + // Row of the table -> the call it shows, or nullptr when out of range. Decoded into a + // single-row cache, so asking for the same row repeatedly (as data() does, once per + // column) costs one decode. `withParams` is only needed by the detail pane. + const TTDApiCall* callAt(int row, bool withParams = false) const; + + int rowCount(const QModelIndex& parent = QModelIndex()) const override; + int columnCount(const QModelIndex& parent = QModelIndex()) const override; + QVariant data(const QModelIndex& index, int role) const override; + QVariant headerData(int section, Qt::Orientation orientation, int role) const override; + +private: + void rebuildVisible(); + + std::shared_ptr m_report; + QString m_filterText; // as typed, so a repeat of the same text is a no-op + std::vector m_visible; // source indices; empty when unfiltered + bool m_filtered = false; // whether m_visible is in use + + mutable TTDApiCall m_cached; + mutable size_t m_cachedIndex = static_cast(-1); + mutable bool m_cachedHasParams = false; +}; + + +// One query over the loaded report: a filter, the rows it matches, and the detail pane +// for whichever of them is selected. Tabs of these share a single report, since the +// point of having several is to ask different questions of the same trace without +// paying to load it again. +class TTDBehaviorQueryWidget : public QWidget +{ + Q_OBJECT + +private: + BinaryViewRef m_data; + DbgRef m_controller; + + QLineEdit* m_filterEdit; + QLabel* m_statusLabel; + QTableView* m_table; + QTextEdit* m_detail; + + TTDBehaviorCallModel* m_model; + std::shared_ptr m_report; + QString m_reportPath; // the core report holds no display name of its own + + // Set by the container, which is what knows them; shown in this tab's status line. + double m_writeSeconds = 0.0; + double m_loadSeconds = -1.0; + + // Filtering waits for a pause in typing rather than running on every keystroke: + // on a large trace each pass is real work, and eight of them while typing + // "kernel32" is eight times the work for seven results nobody looks at. + QTimer* m_filterTimer; + + void setupUI(); + void showDetail(const TTDApiCall* call); + +public: + TTDBehaviorQueryWidget(QWidget* parent, BinaryViewRef data); + + // Point this tab at a report. Cheap: the model only takes a reference to it. + void setReport(std::shared_ptr report, const QString& path); + + QString filterText() const; + void setFilterText(const QString& text); + + // Appended to the status line by the container, which is what knows them. + void setTimings(double writeSeconds, double loadSeconds); + void updateStatus(); + +Q_SIGNALS: + // So the container can label the tab after whatever the tab is asking. + void filterApplied(const QString& text); + +private Q_SLOTS: + void applyFilter(); + void onFilterTextEdited(); + void onSelectionChanged(); + void onDoubleClicked(const QModelIndex& index); + void onContextMenu(const QPoint& pos); + void copySelectedRows(); +}; + + +// Holds the report and the things that act on it as a whole -- loading, extracting, +// progress -- above a tab bar of queries. +class TTDBehaviorWidget : public SidebarWidget +{ + Q_OBJECT + +private: + BinaryViewRef m_data; + DbgRef m_controller; + + QPushButton* m_loadButton; + QPushButton* m_extractButton; + QTabWidget* m_tabWidget; + QToolButton* m_newTabButton; + + // Shown only while an extraction or a report load is in flight. + QWidget* m_progressRow; + QProgressBar* m_progressBar; + QLabel* m_progressLabel; + QPushButton* m_cancelButton; + QElapsedTimer m_operationTimer; + QByteArray m_stderrTail; // partial line left over between readyRead signals + + // Reported in each tab's status line: serialising and loading a report are the two + // costs most worth seeing, since they dominated the wall clock before the binary + // format. + double m_lastWriteSeconds = 0.0; + double m_lastLoadSeconds = -1.0; + + std::shared_ptr m_report; + QString m_reportPath; + QProcess* m_extractProcess = nullptr; + + void setupUI(); + QString extractorPath(bool prompt); + + void beginOperation(const QString& what, bool cancellable); + void endOperation(); + void setProgress(double percent, const QString& detail); + void consumeExtractorStderr(); + + TTDBehaviorQueryWidget* currentQuery() const; + +public: + TTDBehaviorWidget(BinaryViewRef data); + ~TTDBehaviorWidget(); + + // Parses `path` on a worker thread -- a multi-hundred-MB report takes long enough + // that doing it inline freezes the UI -- and installs it when finished. + void loadReport(const QString& path); + + // Swap in a parsed report and hand it to every tab. Must run on the UI thread. + void installReport(std::shared_ptr report, const QString& path); + +private Q_SLOTS: + void onLoadClicked(); + void onExtractClicked(); + void onCancelClicked(); + void createNewTab(); + void closeTab(int index); +}; + + +class TTDBehaviorWidgetType : public SidebarWidgetType +{ +public: + TTDBehaviorWidgetType(); + SidebarWidget* createWidget(ViewFrame* frame, BinaryViewRef data) override; + SidebarWidgetLocation defaultLocation() const override { return SidebarWidgetLocation::RightContent; } + SidebarContextSensitivity contextSensitivity() const override { return PerViewTypeSidebarContext; } + SidebarIconVisibility defaultIconVisibility() const override { return AlwaysShowSidebarIcon; } +}; diff --git a/ui/ui.cpp b/ui/ui.cpp index f85cc332..0e6e380f 100644 --- a/ui/ui.cpp +++ b/ui/ui.cpp @@ -50,6 +50,7 @@ limitations under the License. #include "ttdcallswidget.h" #include "ttdeventswidget.h" #include "ttdbookmarkwidget.h" +#include "ttdbehaviorwidget.h" #include "ttdanalysisdialog.h" #include "timestampnavigationdialog.h" #include "freeversion.h" @@ -2098,6 +2099,7 @@ void GlobalDebuggerUI::InitializeUI() Sidebar::addSidebarWidgetType(new TTDCallsWidgetType()); Sidebar::addSidebarWidgetType(new TTDEventsWidgetType()); Sidebar::addSidebarWidgetType(new TTDBookmarkWidgetType()); + Sidebar::addSidebarWidgetType(new TTDBehaviorWidgetType()); }