adaptive_export: capture ppid/pcomm in dc_snoop (process-forest seed)

k8s/vizier/adaptive_export/kustomization.yaml

@@ -7,4 +7,4 @@ resources:
 images:
   - name: vizier-adaptive_export_image
     newName: ghcr.io/k8sstormcenter/vizier-adaptive_export_image
-    newTag: 0.14.19-aeprod57
+    newTag: 0.14.19-aeprod58

src/vizier/services/adaptive_export/internal/clickhouse/schema.sql

@@ -649,11 +649,23 @@ CREATE TABLE IF NOT EXISTS forensic_db.dx_ptrace (
 
 -- dc_snoop (dentry cache, V1/V2 process+file) — exported via the OTel/ClickHouse
 -- retention plugin (px.export). pid-keyed; t = R (reference) / M (miss).
+-- ppid/pcomm captured inline in the tracepoint (curtask->real_parent) so every
+-- dcache event carries its parent with no join. pid_start/ppid_start are
+-- group_leader->start_time (ns since boot) — a pid-reuse-stable identity for the
+-- process and its parent, and the (pid,pid_start)/(ppid,ppid_start) keys the
+-- per-node process forest walks to correlate evidence to ancestry. Deeper
+-- ancestry (pod attribution of a blank-pod child) is resolved by dx against the
+-- forest; the AE only cuts 1-level own-stack noise here via the ppid->namespace
+-- join.
 -- One column per line (schema-verify parser is line-oriented).
 CREATE TABLE IF NOT EXISTS forensic_db.dc_snoop (
   time_ DateTime64(9, 'UTC'),
   pid Int64,
+  pid_start Int64,
+  ppid Int64,
+  ppid_start Int64,
   comm String,
+  pcomm String,
   t String,
   file String,
   namespace String,

src/vizier/services/adaptive_export/internal/pxl/queryfor.go

@@ -90,11 +90,19 @@ func QueryFor(table string, t anomaly.Target, sliceStart, sliceEnd, now time.Tim
 			}
 		}
 	} else if IsDarkVector(table) {
-		// Node-scoped (transient attack pids resolve blank ns, so no pod filter). Drop
-		// own-stack comms before the pid-merge to keep it cheap, then drop infra namespaces.
-		b.WriteString(darkCommExclusion(table))
+		// Node-scoped (transient attack pids resolve blank ns, so no pod filter).
+		// Noise is cut structurally, NOT by a hardcoded comm blocklist (deleted):
+		//   1. child-namespace exclusion drops resolved own-stack processes (vizier,
+		//      pem, node-agent, … resolve to pl/honey/clickhouse via their pod);
+		//   2. parent-ancestry exclusion (ppid -> parent namespace) drops the
+		//      transient children of own-stack pods that resolve blank themselves.
+		// Host-level runtime + kernel threads (containerd-shim/systemd/k3s/kworker)
+		// have a blank namespace AND a blank/kernel-thread parent, so 1-level ancestry
+		// cannot resolve them — they enter the always-on retention look-back and are
+		// relevance-filtered by dx's multi-level process forest, not here.
 		b.WriteString(PodEnrichPxL(table))
 		b.WriteString(darkNamespaceExclusion())
+		b.WriteString(darkParentAncestryExclusion(table))
 	} else {
 		b.WriteString(PodEnrichPxL(table))
 		if t.Namespace != "" {
@@ -132,83 +140,67 @@ func pixieSourceFor(table string) string {
 	return table
 }
 
-// Dark-vector tables carrying a comm column (so the comm exclusion applies).
-var darkVectorHasComm = map[string]bool{
-	"dc_snoop": true, "creds_change": true, "dx_vfs_events": true,
-	"dx_unlink": true, "dx_dlookup": true, "dx_mprotect": true,
-	"dx_bpf": true, "dx_ptrace": true,
+// Dark-vector tables that carry a ppid column (so the parent-ancestry exclusion
+// applies). Only dc_snoop captures ppid today; creds_change and the dx_* bpftrace
+// tables do not, so they get the child-namespace exclusion only.
+var darkVectorHasPpid = map[string]bool{
+	"dc_snoop": true,
 }
 
-// Own-stack + node/system comms dropped from the node-scoped dark capture; workload
-// comms (redis-*, etc.) are never listed. Override via DC_SNOOP_EXCLUDE_COMMS (csv).
-var darkExcludeCommsDefault = []string{
-	"pem", "kelvin", "containerd", "containerd-shim", "runc", "node-agent",
-	"runc:[2:INIT]", "runc:[1:CHILD]",
-	"vizier-query-broker", "vizier-metadata", "nats-server", "k3s-server",
-	"k3s-agent", "systemd", "systemd-journal", "SystemLogFlush", "kubelet",
-	"AsyncInsertQ", "BgSchPool", "Collector", "AsyncMetrics", "MergeMutate",
-	"MergeTreeIndex", "CgrpMemUsgObsr", "coredns", "metadata", "storage",
-	"operator", "iptables", "iptables-save", "iptables-restor", "ip6tables",
-	"ConfigReloader", "clickhouse-oper", "Formatter", "(setup.sh)", "cmd",
-	"vector-worker", "metrics-server", "local-path-prov", "portmap",
-	"(udev-worker)", "systemd-resolve", "systemd-timesyn",
-	"systemd-udevd", "systemd-sysctl", "host-local", "bridge", "flannel",
-	"loopback", "bandwidth", "dbus-daemon", "mount", "umount", "tailscaled",
-	"grpc_health_pro", "kubevuln", "opm", "(spawn)", "kube-proxy",
-	"pause", "systemd-logind",
-}
-
-// Kernel-thread families whose names carry a variable suffix (kworker/u8:3) that
-// exact match misses; dropped via px.contains.
-var darkExcludeCommSubstrings = []string{
-	"kworker", "ksoftirqd", "migration", "rcu_", "kthreadd", "kdevtmpfs",
-	"kcompactd", "khugepaged", "kswapd", "watchdog", "cpuhp", "ksmd", "irq/",
-}
-
-// Infra namespaces dropped from the node-scoped dark capture. Blank-namespace rows
-// (transient attack children) survive. Override via DC_SNOOP_EXCLUDE_NAMESPACES.
+// Infra namespaces dropped from the node-scoped dark capture — applied BOTH to the
+// row's own namespace (resolved own-stack processes) and, for tables with ppid, to
+// the parent's namespace (ancestry). Blank-namespace rows survive (transient attack
+// children; host/kernel processes). Override via DC_SNOOP_EXCLUDE_NAMESPACES.
 var darkExcludeNamespacesDefault = []string{
-	"pl", "honey", "px-operator", "olm", "clickhouse", "socdemo", "socdemo-ch",
+	"pl", "honey", "px-operator", "olm", "clickhouse",
 	"kube-system", "kube-public", "kube-node-lease", "local-path-storage",
 }
 
-func darkCommExclusion(table string) string {
-	if !darkVectorHasComm[table] {
-		return ""
-	}
-	comms := darkExcludeCommsDefault
-	if v := strings.TrimSpace(os.Getenv("DC_SNOOP_EXCLUDE_COMMS")); v != "" {
-		comms = nil
+// darkExcludeNamespaces resolves the effective infra-namespace list (env override
+// or default), shared by the child-namespace and parent-ancestry exclusions so the
+// two can never drift.
+func darkExcludeNamespaces() []string {
+	if v := strings.TrimSpace(os.Getenv("DC_SNOOP_EXCLUDE_NAMESPACES")); v != "" {
+		var nss []string
 		for _, s := range strings.Split(v, ",") {
 			if s = strings.TrimSpace(s); s != "" {
-				comms = append(comms, s)
+				nss = append(nss, s)
 			}
 		}
+		return nss
 	}
+	return darkExcludeNamespacesDefault
+}
+
+func darkNamespaceExclusion() string {
 	var b strings.Builder
-	for _, c := range comms {
-		b.WriteString("df = df[df.comm != '" + escapePxL(c) + "']\n")
-	}
-	for _, s := range darkExcludeCommSubstrings {
-		b.WriteString("df = df[px.logicalNot(px.contains(df.comm, '" + escapePxL(s) + "'))]\n")
+	for _, ns := range darkExcludeNamespaces() {
+		b.WriteString("df = df[df.namespace != '" + escapePxL(ns) + "']\n")
 	}
 	return b.String()
 }
 
-func darkNamespaceExclusion() string {
-	nss := darkExcludeNamespacesDefault
-	if v := strings.TrimSpace(os.Getenv("DC_SNOOP_EXCLUDE_NAMESPACES")); v != "" {
-		nss = nil
-		for _, s := range strings.Split(v, ",") {
-			if s = strings.TrimSpace(s); s != "" {
-				nss = append(nss, s)
-			}
-		}
+// darkParentAncestryExclusion resolves each row's PARENT namespace via a
+// process_stats join on ppid and drops rows whose parent lives in an own-stack
+// namespace — even when the row's own pod is blank (a transient process exec'd by
+// an infra pod). This is the ppid-based ancestry cut that REPLACES the deleted
+// hardcoded comm blocklist: pod/namespace-rooted, not comm-matched. parent_namespace
+// is dropped before display so the sink projection is unchanged. Emits nothing for
+// tables without a ppid column. Mirrors the retention path (dc_snoop.pxl).
+func darkParentAncestryExclusion(table string) string {
+	if !darkVectorHasPpid[table] {
+		return ""
 	}
 	var b strings.Builder
-	for _, ns := range nss {
-		b.WriteString("df = df[df.namespace != '" + escapePxL(ns) + "']\n")
+	b.WriteString("par = px.DataFrame(table='process_stats', start_time='" + darkProcStatsWindow + "')\n")
+	b.WriteString("par.parent_namespace = par.ctx['namespace']\n")
+	b.WriteString("par.ppid = px.upid_to_pid(par.upid)\n")
+	b.WriteString("par = par.groupby(['parent_namespace', 'ppid']).agg()\n")
+	b.WriteString("df = df.merge(par, how='left', left_on=['ppid'], right_on=['ppid'], suffixes=['', '_par'])\n")
+	for _, ns := range darkExcludeNamespaces() {
+		b.WriteString("df = df[df.parent_namespace != '" + escapePxL(ns) + "']\n")
 	}
+	b.WriteString("df = df.drop(['parent_namespace'])\n")
 	return b.String()
 }
 

src/vizier/services/adaptive_export/internal/pxl/queryfor_test.go

@@ -37,6 +37,30 @@ var (
 	}
 )
 
+// TestQueryFor_CleanupNoCollateral — the pure-ancestry cleanup touched ONLY the
+// dark-vector branch. Every native/protocol table (redis_events, dns_events, …) and
+// stack_trace must be UNAFFECTED: no ancestry join, no parent_namespace, no comm
+// drops, and they must still pin the alert pod (dx-steered pod scope). This is the
+// "other CH tables not affected" guard.
+func TestQueryFor_CleanupNoCollateral(t *testing.T) {
+	native := []string{"redis_events", "dns_events", "http_events", "conn_stats", "pgsql_events", "mysql_events", "stack_trace"}
+	for _, tbl := range native {
+		q, err := QueryFor(tbl, target, fixedStart, fixedEnd, fixedNow)
+		if err != nil {
+			t.Fatalf("QueryFor(%s): %v", tbl, err)
+		}
+		for _, banned := range []string{"parent_namespace", "left_on=['ppid']", "df.comm !=", "px.contains(df.comm", "df = df[df.namespace != '"} {
+			if strings.Contains(q, banned) {
+				t.Errorf("%s (native) must not contain dark-path machinery %q; got:\n%s", tbl, banned, q)
+			}
+		}
+		// still pod-scoped (dx-steered): the alert pod is pinned
+		if !strings.Contains(q, "redis-6fbcfb97c-82qxv") {
+			t.Errorf("%s must still pin the alert pod; got:\n%s", tbl, q)
+		}
+	}
+}
+
 // TestQueryFor_UnknownTable — non-builtin tables wrap ErrUnknownTable.
 func TestQueryFor_UnknownTable(t *testing.T) {
 	_, err := QueryFor("nope_table", target, fixedStart, fixedEnd, fixedNow)
@@ -378,14 +402,14 @@ func TestQueryFor_NoEndTimeAtLiveEdge(t *testing.T) {
 }
 
 // TestQueryFor_DarkNamespaceExclusion — the node-scoped dark capture (dc_snoop)
-// must drop infra namespaces (pl, kube-system, …) while KEEPING blank-namespace
-// transient rows (the attack's short-lived children). Mirrors the shipped preset.
+// must drop infra namespaces by the row's OWN namespace (resolved own-stack
+// processes) while KEEPING blank-namespace transient rows. Mirrors the shipped
+// preset.
 func TestQueryFor_DarkNamespaceExclusion(t *testing.T) {
 	q, err := QueryFor("dc_snoop", target, fixedStart, fixedEnd, fixedNow)
 	if err != nil {
 		t.Fatalf("QueryFor: %v", err)
 	}
-	// namespace drops present for infra
 	for _, ns := range []string{"pl", "kube-system", "clickhouse"} {
 		if !strings.Contains(q, "df = df[df.namespace != '"+ns+"']") {
 			t.Errorf("dark capture must drop infra namespace %q; got:\n%s", ns, q)
@@ -395,47 +419,93 @@ func TestQueryFor_DarkNamespaceExclusion(t *testing.T) {
 	if strings.Contains(q, "df = df[df.namespace == '") {
 		t.Errorf("dark capture must not pin df.namespace ==; got:\n%s", q)
 	}
-	// host/CNI comm drops present
-	for _, c := range []string{"host-local", "systemd-udevd", "tailscaled", "kubevuln"} {
-		if !strings.Contains(q, "df = df[df.comm != '"+c+"']") {
-			t.Errorf("dark capture must drop host/CNI comm %q; got:\n%s", c, q)
-		}
-	}
 }
 
 // TestQueryFor_DarkNamespaceExclusion_EnvOverride — DC_SNOOP_EXCLUDE_NAMESPACES
-// replaces the default list.
+// replaces the default list for BOTH the child-namespace and the parent-ancestry
+// drops (they share one resolved list, so they can never drift).
 func TestQueryFor_DarkNamespaceExclusion_EnvOverride(t *testing.T) {
 	t.Setenv("DC_SNOOP_EXCLUDE_NAMESPACES", "foo,bar")
 	q, err := QueryFor("dc_snoop", target, fixedStart, fixedEnd, fixedNow)
 	if err != nil {
 		t.Fatalf("QueryFor: %v", err)
 	}
-	if !strings.Contains(q, "df = df[df.namespace != 'foo']") || !strings.Contains(q, "df = df[df.namespace != 'bar']") {
-		t.Errorf("env override must emit foo/bar drops; got:\n%s", q)
+	for _, want := range []string{
+		"df = df[df.namespace != 'foo']", "df = df[df.namespace != 'bar']",
+		"df = df[df.parent_namespace != 'foo']", "df = df[df.parent_namespace != 'bar']",
+	} {
+		if !strings.Contains(q, want) {
+			t.Errorf("env override must emit %q; got:\n%s", want, q)
+		}
+	}
+	if strings.Contains(q, "!= 'pl']") {
+		t.Errorf("env override must REPLACE the default (no 'pl' in either child or parent drop); got:\n%s", q)
+	}
+}
+
+// TestQueryFor_DarkNoCommBlocklist — the hardcoded comm blocklist and kernel-thread
+// substring filter were DELETED (pure-ancestry cleanup). The dark capture must emit
+// NO comm-based drops at all: no `df.comm != ...`, no `px.contains(df.comm, ...)`.
+// Noise is cut structurally (own + parent namespace), so host/kernel comms now enter
+// the retention look-back for dx's forest to relevance-filter.
+func TestQueryFor_DarkNoCommBlocklist(t *testing.T) {
+	q, err := QueryFor("dc_snoop", target, fixedStart, fixedEnd, fixedNow)
+	if err != nil {
+		t.Fatalf("QueryFor: %v", err)
+	}
+	if strings.Contains(q, "df.comm !=") {
+		t.Errorf("comm blocklist must be gone (no df.comm != drops); got:\n%s", q)
 	}
-	if strings.Contains(q, "df = df[df.namespace != 'pl']") {
-		t.Errorf("env override must REPLACE the default (no 'pl'); got:\n%s", q)
+	if strings.Contains(q, "px.contains(df.comm") {
+		t.Errorf("kernel-thread substring filter must be gone; got:\n%s", q)
 	}
 }
 
-// dc_snoop drops kernel-thread families (variable suffix) via px.logicalNot(px.contains),
-// keeps workload comms (redis-server), and doesn't pin the alert pod's namespace.
-func TestQueryFor_DarkCommSubstringExclusion(t *testing.T) {
+// TestQueryFor_DarkParentAncestry — for a ppid-bearing dark table (dc_snoop) the
+// query must resolve the PARENT namespace via a process_stats join on ppid, drop
+// own-stack parents (the ancestry cut that replaced the comm blocklist), and DROP
+// parent_namespace before display so the sink projection is unchanged.
+func TestQueryFor_DarkParentAncestry(t *testing.T) {
 	q, err := QueryFor("dc_snoop", target, fixedStart, fixedEnd, fixedNow)
 	if err != nil {
 		t.Fatalf("QueryFor: %v", err)
 	}
-	for _, sub := range []string{"kworker", "ksoftirqd", "rcu_"} {
-		want := "df = df[px.logicalNot(px.contains(df.comm, '" + sub + "'))]"
+	for _, want := range []string{
+		"par = px.DataFrame(table='process_stats'",
+		"par.parent_namespace = par.ctx['namespace']",
+		"par.ppid = px.upid_to_pid(par.upid)",
+		"left_on=['ppid'], right_on=['ppid']",
+		"df = df[df.parent_namespace != 'pl']",
+		"df = df[df.parent_namespace != 'clickhouse']",
+		"df = df.drop(['parent_namespace'])",
+	} {
 		if !strings.Contains(q, want) {
-			t.Errorf("want kernel-thread drop %q; got:\n%s", want, q)
+			t.Errorf("dark ancestry filter missing %q; got:\n%s", want, q)
 		}
 	}
-	if strings.Contains(q, "df.comm != 'redis-server'") || strings.Contains(q, "df.comm, 'redis") {
-		t.Errorf("workload comm redis-* must NOT be excluded; got:\n%s", q)
+	// parent_namespace must be dropped BEFORE px.display (not leak to the sink).
+	di, pi := strings.Index(q, "df.drop(['parent_namespace'])"), strings.Index(q, "px.display")
+	if di < 0 || pi < 0 || di > pi {
+		t.Errorf("parent_namespace must be dropped before px.display; drop@%d display@%d", di, pi)
 	}
-	if !strings.Contains(q, "df = df[df.comm != 'pause']") {
-		t.Errorf("want exact drop of 'pause'; got:\n%s", q)
+}
+
+// TestQueryFor_AncestryOnlyForPpidTables — the ancestry join applies ONLY to dark
+// tables that capture ppid (dc_snoop). creds_change and the dx_* tables have no
+// ppid, so they must NOT get a process_stats-on-ppid parent merge (it would fail to
+// compile), only the child-namespace exclusion.
+func TestQueryFor_AncestryOnlyForPpidTables(t *testing.T) {
+	for _, tbl := range []string{"creds_change", "dx_vfs_events"} {
+		q, err := QueryFor(tbl, target, fixedStart, fixedEnd, fixedNow)
+		if err != nil {
+			t.Fatalf("QueryFor(%s): %v", tbl, err)
+		}
+		if strings.Contains(q, "parent_namespace") || strings.Contains(q, "left_on=['ppid']") {
+			t.Errorf("%s has no ppid — must NOT get the ancestry join; got:\n%s", tbl, q)
+		}
+		// but it still gets the child-namespace exclusion
+		if !strings.Contains(q, "df = df[df.namespace != 'pl']") {
+			t.Errorf("%s must still get child-namespace exclusion; got:\n%s", tbl, q)
+		}
 	}
 }