From f267209ddc3c89cbae8e7cf981ae84d2f9694acb Mon Sep 17 00:00:00 2001 From: Thomas Brady Date: Fri, 16 Oct 2026 16:52:42 -0400 Subject: [PATCH] fix(network-ops): surface structured findings in Agent Output; fix report_item models (ENG-8848, ENG-8832) network-ops shipped a local report_item that logged items under ad-hoc class names via log_output, so nothing ever reached Agent Output; the agent fell back to a freeform markdown report. The finding models also required fields an operator cannot know from enumeration and inferred host role from which optional fields happened to be populated. - capability.yaml: declare an outputs: contract (built-in finding/asset plus host/user/credential/hash/share/weakness) so the runtime builds the contract-backed report_item and records surface in Agent Output. - items.py: move the record models here; single Host with an explicit role discriminator (domain_controller/member_server); make Share/User fields an operator can't know optional; align Weakness severity to the built-in finding enum so the flattened report_item schema has no collision. - Remove the local report_item tool (tools/reporting.py). - Update agent/skill guidance to the host+role report_item contract. - Tests: assert the outputs: block resolves against items.py, and bind the coordinator's extracted field names to the models so a rename fails loudly. BREAKING CHANGE: report_item is now the runtime item_type-based tool; the local DomainController/MemberServer union and the old field names are removed. --- .../network-ops/agents/network-ops-agent.md | 2 +- .../agents/pipeline/discovery-operator.md | 4 +- capabilities/network-ops/capability.yaml | 16 +- capabilities/network-ops/items.py | 126 +++++++++++++ .../skills/network-ops-methodology/SKILL.md | 2 +- .../network-ops/tests/test_items_outputs.py | 169 ++++++++++++++++++ .../tests/test_worker_coordinator.py | 42 +++-- capabilities/network-ops/tools/reporting.py | 97 ---------- 8 files changed, 344 insertions(+), 114 deletions(-) create mode 100644 capabilities/network-ops/items.py create mode 100644 capabilities/network-ops/tests/test_items_outputs.py delete mode 100644 capabilities/network-ops/tools/reporting.py diff --git a/capabilities/network-ops/agents/network-ops-agent.md b/capabilities/network-ops/agents/network-ops-agent.md index 6bcc3796..cdab7129 100644 --- a/capabilities/network-ops/agents/network-ops-agent.md +++ b/capabilities/network-ops/agents/network-ops-agent.md @@ -71,4 +71,4 @@ Consult the **ad-enumeration-playbook** skill for enumeration priority order and ## Reporting -Use `report_item` frequently to log findings as structured data. Report domain controllers, member servers, users, credentials, hashes, shares, and weaknesses as you discover them. +Use `report_item` frequently to log findings as structured data. Report hosts (`item_type: host` with `role: domain_controller` or `member_server`), users, credentials, hashes, shares, and weaknesses as you discover them. diff --git a/capabilities/network-ops/agents/pipeline/discovery-operator.md b/capabilities/network-ops/agents/pipeline/discovery-operator.md index d73d048f..3553d6ff 100644 --- a/capabilities/network-ops/agents/pipeline/discovery-operator.md +++ b/capabilities/network-ops/agents/pipeline/discovery-operator.md @@ -19,7 +19,7 @@ Report every discovered host, service, DC, and member server immediately via `re 1. **Host Inventory**: all discovered hosts with IP addresses and open ports. 2. **Service Map**: identified services and versions per host. -3. **Domain Controllers**: hosts with AD service ports (88, 389, 636, 445). Report as DomainController. -4. **Member Servers**: non-DC Windows hosts. Report as MemberServer. +3. **Domain Controllers**: hosts with AD service ports (88, 389, 636, 445). Report via `report_item` as a `host` with `role: domain_controller`. +4. **Member Servers**: non-DC Windows hosts. Report via `report_item` as a `host` with `role: member_server`. 5. **Network Topology**: subnets observed, segmentation notes. 6. **Scan Coverage**: what was scanned, what was excluded, any failures. diff --git a/capabilities/network-ops/capability.yaml b/capabilities/network-ops/capability.yaml index 92325838..e0db205c 100644 --- a/capabilities/network-ops/capability.yaml +++ b/capabilities/network-ops/capability.yaml @@ -1,6 +1,6 @@ schema: 1 name: network-ops -version: "2.2.1" +version: "3.0.0" description: > Network operations and Active Directory exploitation. Multi-agent pipeline for autonomous red teaming with Nmap scanning, Netexec @@ -23,6 +23,20 @@ skills: tools: - tools/ +# Structured Agent Output. Enables the built-in `finding` and `asset` types +# plus the runtime report_item / update_item / link_items tools, and declares +# the AD-specific record types whose schemas live in items.py. Agents emit +# these through the runtime-provided report_item (item_type=…); they validate +# against the pinned contracts and surface in Agent Output. +outputs: + values: [finding, asset] + host: items:Host + user: items:User + credential: items:Credential + hash: items:Hash + share: items:Share + weakness: items:Weakness + workers: coordinator: path: workers/coordinator.py diff --git a/capabilities/network-ops/items.py b/capabilities/network-ops/items.py new file mode 100644 index 00000000..f1a338fe --- /dev/null +++ b/capabilities/network-ops/items.py @@ -0,0 +1,126 @@ +"""Structured item types for the network-ops capability. + +These Pydantic models define the schemas for items emitted via +``report_item`` during Active Directory and network penetration-testing +engagements. They are referenced by the ``outputs`` key in +``capability.yaml`` (as ``items:ClassName``) and validated at emit time, +so the agent gets immediate feedback on malformed payloads and the +records surface in Agent Output alongside the built-in ``finding`` and +``asset`` types. + +Design principles +----------------- +- Fields mirror what an operator can actually observe during + enumeration — not theoretical nice-to-haves. A field the producer + cannot reliably know is optional, so realistic findings validate + instead of failing or being hallucinated. +- Host role is explicit via a ``role`` discriminator rather than + inferred from which optional fields happen to be populated. +- Field names that the coordinator worker keys on for finding + extraction (``text``, ``hash_value``, ``severity``) are stable; see + ``workers/coordinator.py``. + +The ``report_item`` tool flattens every declared type's fields into a +single schema keyed by ``item_type``, so a field name shared across +types must carry the same type. ``title`` and ``description`` are +therefore kept string-typed everywhere, and the credential kind is +``credential_type`` to avoid colliding with other literal ``type`` +fields. +""" + +from typing import Literal + +from pydantic import BaseModel, ConfigDict, Field + + +class Host(BaseModel): + """A host discovered in the environment, with an explicit AD role. + + One model covers both domain controllers and member servers: ``role`` + records what the host is, rather than leaving it to be inferred from + which optional fields are populated. + """ + + model_config = ConfigDict(extra="ignore") + + title: str = Field(description="Short label for the host, typically the hostname or FQDN.") + role: Literal["domain_controller", "member_server"] = Field( + description="Host role in the Active Directory environment." + ) + hostname: str = Field(description="Short hostname (e.g., 'dc01').") + ip: str | None = Field(None, description="IP address of the host, if known.") + fqdn: str | None = Field(None, description="Fully qualified domain name, if known.") + domain_name: str | None = Field(None, description="AD domain the host serves or belongs to (e.g., 'corp.local').") + forest_root: str | None = Field(None, description="Forest root domain name (domain controllers only).") + + +class Hash(BaseModel): + """A password hash.""" + + model_config = ConfigDict(extra="ignore") + + title: str = Field(description="Short label, e.g. the account the hash belongs to.") + hash_value: str = Field(description="The actual hash value.") + hash_type: Literal["ntlm", "kerberos_tgs", "kerberos_asrep"] = Field(description="Type of hash algorithm used.") + + +class Credential(BaseModel): + """A credential, e.g. username/password combination or hash.""" + + model_config = ConfigDict(extra="ignore") + + title: str = Field(description="Short label, e.g. the account the credential is for.") + text: str = Field(description="Text representation of the credential.") + credential_type: Literal["username_password", "hash"] = Field(description="Kind of credential.") + + +class User(BaseModel): + """A user account.""" + + model_config = ConfigDict(extra="ignore") + + title: str = Field(description="Short label for the account, typically the username.") + username: str = Field(description="Username or account name.") + domain: str | None = Field(None, description="Domain the user belongs to, if known.") + description: str | None = Field(None, description="Account description or notes, if available.") + + +class Share(BaseModel): + """A network share. + + Only the share name is required; the remaining attributes are + optional because an operator often observes a share without knowing + its owner, group, size, or timestamps. + """ + + model_config = ConfigDict(extra="ignore") + + title: str = Field(description="Short label for the share, typically its name.") + share_name: str = Field(description="Share name (e.g., C$, ADMIN$, SYSVOL).") + path: str | None = Field(None, description="UNC path to the share, if known.") + permissions: str | None = Field( + None, description="Observed permissions for the share (e.g., 'READ', 'READ,WRITE')." + ) + description: str | None = Field(None, description="Share description or comment.") + owner: str | None = Field(None, description="Owner of the share, if known.") + group: str | None = Field(None, description="Group that owns the share, if known.") + size: int | None = Field(None, description="Size of the share in bytes, if known.") + last_modified: str | None = Field(None, description="Last modification timestamp, if known.") + + +class Weakness(BaseModel): + """A security vulnerability, weakness, or misconfiguration.""" + + model_config = ConfigDict(extra="ignore") + + title: str = Field(description="Weakness title/name.") + # Values match the platform's built-in finding severity (dreadnode + # ItemSeverity). report_item flattens every declared type into one schema, + # first definition wins, and the built-in finding owns `severity`; a narrower + # enum here would let the agent pick an offered value (such as "info") that + # this model then rejects. + severity: Literal["critical", "high", "medium", "low", "info"] = Field( + description="Impact level: critical, high, medium, low, or info." + ) + cve: str | None = Field(None, description="CVE identifier, if applicable.") + description: str | None = Field(None, description="Weakness description.") diff --git a/capabilities/network-ops/skills/network-ops-methodology/SKILL.md b/capabilities/network-ops/skills/network-ops-methodology/SKILL.md index 64ad0d19..d20f3e05 100644 --- a/capabilities/network-ops/skills/network-ops-methodology/SKILL.md +++ b/capabilities/network-ops/skills/network-ops-methodology/SKILL.md @@ -89,7 +89,7 @@ Every reported finding must include a confidence assessment. ## Reporting Standards Reports must: -- Use `report_item` with the correct model type (DomainController, Credential, Hash, Weakness, etc.) +- Use `report_item` with the correct `item_type`: `host` (with `role: domain_controller` or `member_server`), `user`, `credential`, `hash`, `share`, `weakness`, or the built-in `finding` / `asset` - Include credential provenance (source host, extraction method, verification status) - State access level explicitly (local admin, domain user, DA, EA) - Map weaknesses to specific misconfigurations (not "AD misconfiguration" but "unconstrained delegation on WEB01$") diff --git a/capabilities/network-ops/tests/test_items_outputs.py b/capabilities/network-ops/tests/test_items_outputs.py new file mode 100644 index 00000000..de4e4608 --- /dev/null +++ b/capabilities/network-ops/tests/test_items_outputs.py @@ -0,0 +1,169 @@ +"""Regression tests for the network-ops structured ``outputs`` item schemas. + +The manifest declares AD record types via ``outputs:`` (``host``, ``user``, +``credential``, ``hash``, ``share``, ``weakness``) whose models live in +``items.py``, plus the built-in ``finding`` / ``asset`` types. The Dreadnode +capability loader and OCI packager import ``items.py`` by path **without** +registering it in ``sys.modules`` and then call ``model_json_schema()``. + +These tests reproduce that import path and lock in two guarantees: fields an +operator cannot know are optional (so realistic findings validate instead of +failing or being hallucinated), and host role is an explicit discriminator +rather than inferred from which optional fields happen to be populated. +""" + +from __future__ import annotations + +import importlib.util +import sys +from pathlib import Path + +import pytest +from pydantic import BaseModel, ValidationError + +ITEMS_PATH = (Path(__file__).resolve().parent.parent / "items.py").resolve() + +# Types the manifest's `outputs:` block references by "module:Class". +DECLARED_MODELS = ("Host", "User", "Credential", "Hash", "Share", "Weakness") + + +def _load_items_like_the_loader(): + """Import items.py exactly the way the SDK loader/packager does.""" + mod_name = "_dn_test_network_ops_items" + sys.modules.pop(mod_name, None) + spec = importlib.util.spec_from_file_location(mod_name, ITEMS_PATH) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + # Intentionally NOT added to sys.modules — matches report_items.py and oci.py. + spec.loader.exec_module(module) + return module + + +def test_items_file_exists(): + assert ITEMS_PATH.is_file(), f"items.py not found at {ITEMS_PATH}" + + +@pytest.mark.parametrize("class_name", DECLARED_MODELS) +def test_declared_model_schema_builds_under_loader_import(class_name): + module = _load_items_like_the_loader() + model = getattr(module, class_name) + schema = model.model_json_schema() + assert schema["type"] == "object" + assert "properties" in schema + + +def test_all_module_models_are_fully_defined(): + """Every BaseModel in items.py must build a schema, so new types can't + silently regress.""" + module = _load_items_like_the_loader() + models = [ + obj + for obj in vars(module).values() + if isinstance(obj, type) + and issubclass(obj, BaseModel) + and obj is not BaseModel + and obj.__module__ == module.__name__ + ] + assert models, "expected at least one BaseModel defined in items.py" + for model in models: + schema = model.model_json_schema() # must not raise + assert schema["type"] == "object" + + +# --- Unknowable fields are optional --- + + +def test_share_validates_with_only_observable_fields(): + """A share reported the way an operator discovers it must validate.""" + module = _load_items_like_the_loader() + share = module.Share.model_validate( + {"title": "SYSVOL", "share_name": "SYSVOL", "path": r"\\dc01\SYSVOL", "permissions": "READ"} + ) + assert share.owner is None + assert share.size is None + assert share.last_modified is None + + +def test_user_description_is_optional(): + module = _load_items_like_the_loader() + user = module.User.model_validate({"title": "jsmith", "username": "jsmith"}) + assert user.description is None + assert user.domain is None + + +# --- Host role is explicit, not inferred from field population --- + + +def test_host_role_is_explicit_discriminator(): + module = _load_items_like_the_loader() + dc = module.Host.model_validate({"title": "dc01", "role": "domain_controller", "hostname": "dc01"}) + member = module.Host.model_validate( + { + "title": "ws01.corp.local", + "role": "member_server", + "hostname": "ws01", + "fqdn": "ws01.corp.local", + "domain_name": "corp.local", + } + ) + # Role is what the caller stated, regardless of which optional fields are set. + assert dc.role == "domain_controller" + assert member.role == "member_server" + + +def test_host_requires_a_role(): + module = _load_items_like_the_loader() + with pytest.raises(ValidationError): + module.Host.model_validate({"title": "dc01", "hostname": "dc01"}) + + +# --- Flattened report_item schema: no cross-type field-type collisions --- + + +def test_shared_field_types_are_consistent_across_declared_models(): + """report_item flattens every type's fields into one schema keyed by the + first definition, so a field name shared across types must carry the same + JSON *shape*. Guards against reintroducing a conflicting field such as a + list-typed description alongside a string-typed one. Human-readable + descriptions may legitimately differ per type, so only structural keys + are compared.""" + structural_keys = ("type", "enum", "anyOf", "items", "format", "$ref") + module = _load_items_like_the_loader() + seen: dict[str, str] = {} + for class_name in DECLARED_MODELS: + props = getattr(module, class_name).model_json_schema()["properties"] + for field, schema in props.items(): + shape = {k: schema[k] for k in structural_keys if k in schema} + fingerprint = str(sorted(shape.items())) + if field in seen: + assert seen[field] == fingerprint, f"field '{field}' has conflicting schemas across declared models" + else: + seen[field] = fingerprint + + +# --- The manifest's outputs: block actually resolves against items.py --- + + +def test_capability_outputs_block_resolves_to_real_models(): + """Nothing else parses capability.yaml, so a typo'd `items:ClassName` ref + would be silently dropped by the runtime resolver — reproducing the empty + Agent Output bug with a green suite. Parse the manifest and prove every + declared custom type maps to a model that builds a schema, and that the + built-in values are the ones we expect.""" + import yaml + + manifest_path = (Path(__file__).resolve().parent.parent / "capability.yaml").resolve() + outputs = yaml.safe_load(manifest_path.read_text())["outputs"] + + assert set(outputs["values"]) == {"finding", "asset"} + + module = _load_items_like_the_loader() + custom = {k: v for k, v in outputs.items() if k != "values"} + assert set(custom) == {"host", "user", "credential", "hash", "share", "weakness"} + + for item_type, ref in custom.items(): + module_name, _, class_name = ref.partition(":") + assert module_name == "items", f"{item_type} ref should point at items.py, got {ref!r}" + model = getattr(module, class_name, None) + assert model is not None, f"{ref} does not resolve to a class in items.py" + assert model.model_json_schema()["type"] == "object" diff --git a/capabilities/network-ops/tests/test_worker_coordinator.py b/capabilities/network-ops/tests/test_worker_coordinator.py index 54c0b98c..ecb0218b 100644 --- a/capabilities/network-ops/tests/test_worker_coordinator.py +++ b/capabilities/network-ops/tests/test_worker_coordinator.py @@ -27,9 +27,7 @@ def run(self): runtime.Worker = Worker sys.modules.setdefault("dreadnode", types.ModuleType("dreadnode")) - sys.modules.setdefault( - "dreadnode.capabilities", types.ModuleType("dreadnode.capabilities") - ) + sys.modules.setdefault("dreadnode.capabilities", types.ModuleType("dreadnode.capabilities")) sys.modules["dreadnode.capabilities.worker"] = runtime if "loguru" not in sys.modules: @@ -76,9 +74,7 @@ def test_normalize_scope_payload_passes_through_optional_fields(): def test_normalize_scope_payload_drops_empty_values(): coordinator = _load_coordinator() - scope = coordinator._normalize_scope_payload( - {"target": "10.10.10.0/24", "domain": "", "notes": None} - ) + scope = coordinator._normalize_scope_payload({"target": "10.10.10.0/24", "domain": "", "notes": None}) assert "domain" not in scope assert "notes" not in scope @@ -214,15 +210,11 @@ def test_prompts_thread_prior_stage_reports_forward(): discovery_prompt = coordinator._discovery_prompt('{"target":"x"}', "SCOPE_OUT", 10) assert "SCOPE_OUT" in discovery_prompt - enum_prompt = coordinator._enumeration_prompt( - '{"target":"x"}', "SCOPE_OUT", "DISCO_OUT", 12 - ) + enum_prompt = coordinator._enumeration_prompt('{"target":"x"}', "SCOPE_OUT", "DISCO_OUT", 12) assert "SCOPE_OUT" in enum_prompt assert "DISCO_OUT" in enum_prompt - exploit_prompt = coordinator._exploit_prompt( - '{"target":"x"}', "SCOPE_OUT", "DISCO_OUT", "ENUM_OUT", 20 - ) + exploit_prompt = coordinator._exploit_prompt('{"target":"x"}', "SCOPE_OUT", "DISCO_OUT", "ENUM_OUT", 20) assert "ENUM_OUT" in exploit_prompt harvest_prompt = coordinator._harvest_prompt( @@ -303,3 +295,29 @@ def test_event_constants_use_netops_prefix(): coordinator.FAILED_EVENT, ): assert event.startswith("netops."), event + + +# --- _HIGH_VALUE_FIELDS is coupled to items.py by bare string --- + + +def _load_items(): + path = Path(__file__).resolve().parents[1] / "items.py" + spec = importlib.util.spec_from_file_location("netops_items_for_coord_test", path) + assert spec and spec.loader + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def test_high_value_fields_exist_on_item_models(): + """`_extract_findings` keys attack results off bare field-name strings. If a + field is renamed in items.py without updating the coordinator, hashes and + weaknesses silently vanish from synthesis. This binds the two so a rename + breaks loudly instead.""" + coordinator = _load_coordinator() + items = _load_items() + model_fields: set[str] = set() + for model_name in ("Credential", "Hash", "Weakness"): + model_fields |= set(getattr(items, model_name).model_fields) + missing = coordinator._HIGH_VALUE_FIELDS - model_fields + assert not missing, f"_HIGH_VALUE_FIELDS keys not present on any item model: {missing}" diff --git a/capabilities/network-ops/tools/reporting.py b/capabilities/network-ops/tools/reporting.py deleted file mode 100644 index 2fa33f8b..00000000 --- a/capabilities/network-ops/tools/reporting.py +++ /dev/null @@ -1,97 +0,0 @@ -from typing import Literal - -import dreadnode as dn -from dreadnode.agents.tools import tool -from pydantic import BaseModel, Field - - -class Target(BaseModel): - ip: str = Field(description="IP address of the target") - - -class Service(BaseModel): - """A service running on a host.""" - - service_name: str = Field(description="Name of the service (e.g., SMB, SSH, HTTP)") - version: str = Field(description="Version of the service") - - -class DomainController(BaseModel): - """Represents a Domain Controller in an Active Directory environment.""" - - hostname: str = Field(description="Short hostname (e.g., 'servername')") - fqdn: str = Field(description="Fully qualified domain name") - ip: str = Field(description="IP address of the domain controller") - domain_name: str = Field( - description="AD domain this DC serves (e.g., 'subname.servername.local')" - ) - forest_root: str | None = Field(None, description="Forest root domain name") - - -class MemberServer(BaseModel): - """Represents a Member Server in an Active Directory environment.""" - - hostname: str = Field(description="Short hostname (e.g., 'servername')") - fqdn: str | None = Field(None, description="Fully qualified domain name") - ip: str = Field(description="IP address of the member server") - domain_name: str | None = Field(None, description="AD domain the server belongs to") - - -class Hash(BaseModel): - """A password hash.""" - - hash_value: str = Field(description="The actual hash value") - hash_type: Literal["ntlm", "kerberos_tgs", "kerberos_asrep"] = Field( - description="Type of hash algorithm used" - ) - - -class Credential(BaseModel): - """A credential, e.g. username/password combination or hash.""" - - text: str = Field(description="Text representation of the credential") - type: Literal["username_password", "hash"] = Field(description="Type of credential") - - -class User(BaseModel): - """A user account.""" - - username: str = Field(description="Username or account name") - domain: str = Field(description="Domain the user belongs to") - description: list[str] = Field(description="User account description or notes") - - -class Share(BaseModel): - """A network share.""" - - share_name: str = Field(description="Share name (e.g., C$, ADMIN$, SYSVOL)") - path: str = Field(description="UNC path to the share") - description: str = Field(description="Share description or comment") - permissions: str = Field(description="Permission string for the share") - type: Literal["read", "write", "execute"] = Field(description="Access type available") - owner: str = Field(description="Owner of the share") - group: str = Field(description="Group that owns the share") - size: int = Field(description="Size of the share in bytes") - last_modified: str = Field(description="Last modification timestamp") - - -class Weakness(BaseModel): - """A security vulnerability, weakness, or misconfiguration.""" - - cve: str | None = Field(None, description="CVE identifier") - title: str = Field(description="Weakness title/name") - description: str | None = Field(None, description="Weakness description") - severity: Literal["low", "medium", "high", "critical"] = Field( - description="Severity of the weakness" - ) - - -@tool -def report_item( - item: DomainController | MemberServer | User | Credential | Share | Hash | Weakness, -) -> None: - """ - Report a relevant item found during enumeration. \ - Use this tool immediately and often. - """ - dn.log_output(item.__class__.__name__, item)