From 4dee1799885d42622c140d9ea26c0f8bc0d864b7 Mon Sep 17 00:00:00 2001 From: entlein Date: Sat, 1 Aug 2026 16:05:24 +0200 Subject: [PATCH 01/66] adaptive_export: chunk + bound the ordered capture path (flaky-capture fix) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause of the flaky dx-steered capture (dc_snoop/http erratically 0 while light tables always land): OrderExportAll fans out ~20 tables concurrently, each OrderQuery issued ONE unbounded PxL query over the full ~600s control window against the single node-local PEM (pem-direct). QueryFor only set start_time, so every query re-scanned [sliceStart, now] and post-filtered — the heavy tables materialize huge result sets on a saturated PEM and lose the fixed 180s deadline race, dropping out; the cheap tables (redis/conn/stack) return instantly and survive. Reconcile fingerprint: the same dc_snoop query returns 2459 rows in isolation but 0 + 1 err under the fan-out. Fix (durable — removes the data-volume↔deadline coupling, not just tunes it): - pxl.QueryFor: bound the PEM source scan on BOTH sides. Emit a relative end_time (floored toward now so nothing real is clipped; the exact upper bound stays enforced by the df.time_ < sliceEnd nanos post-filter) whenever sliceEnd is in the past. Live-edge slices keep scanning to now (no end_time), preserving prior behavior for the most-recent window. - controller.OrderQuery: walk the capture window in OrderChunk-sized sub-windows (default 60s, env ADAPTIVE_ORDER_CHUNK_SEC), each a both-sides bounded query, so no single query re-materializes the whole window. captureSpan adaptively halves any chunk that still fails with a transient (deadline/overload) error down to orderMinChunk (1s); non-transient errors (missing dark table) surface immediately without wasteful splitting. Overlapping/retried spans dedupe in the ReplacingMergeTree evidence tables, so re-pulls are idempotent. One aggregated reconcile row per table (not per chunk). Chunks run sequentially per table, so OrderExportAll's per-table concurrency is unchanged while each table now issues cheap bounded queries instead of one firehose — reliable capture without needing the global inflight throttle set. Tests: queryfor end_time present for past windows / absent at the live edge; OrderQuery chunking, single aggregated reconcile row, adaptive subdivision on transient error, no-split on non-transient error, termination at min-chunk. --- .../services/adaptive_export/cmd/main.go | 9 +- .../internal/controller/controller.go | 154 ++++++++++++--- .../internal/controller/order_chunk_test.go | 186 ++++++++++++++++++ .../adaptive_export/internal/pxl/queryfor.go | 28 ++- .../internal/pxl/queryfor_test.go | 36 ++++ 5 files changed, 386 insertions(+), 27 deletions(-) create mode 100644 src/vizier/services/adaptive_export/internal/controller/order_chunk_test.go diff --git a/src/vizier/services/adaptive_export/cmd/main.go b/src/vizier/services/adaptive_export/cmd/main.go index 359a3fe0408..c5def0ab63b 100644 --- a/src/vizier/services/adaptive_export/cmd/main.go +++ b/src/vizier/services/adaptive_export/cmd/main.go @@ -94,8 +94,12 @@ const ( // envExportAllFloorSec bounds how often the dx-steered full capture // (OrderExportAll) re-runs for the same target. Default 30s. envExportAllFloorSec = "ADAPTIVE_EXPORT_ALL_FLOOR_SEC" - envTriggerPollMS = "ADAPTIVE_TRIGGER_POLL_MS" - envPruneIntervalSec = "ADAPTIVE_PRUNE_INTERVAL_SEC" + // envOrderChunkSec is the sub-window the ordered path walks the capture window + // in, so each pixie query is both-sides bounded instead of re-scanning the whole + // window on the node-local PEM. Default 60s (see controller.Config.OrderChunk). + envOrderChunkSec = "ADAPTIVE_ORDER_CHUNK_SEC" + envTriggerPollMS = "ADAPTIVE_TRIGGER_POLL_MS" + envPruneIntervalSec = "ADAPTIVE_PRUNE_INTERVAL_SEC" // envPushRefreshSec overrides controller.PushRefreshInterval. Unset → // 30s default. A NEGATIVE value selects single-shot mode (one pull per @@ -417,6 +421,7 @@ func main() { After: durEnv(envWindowAfterSec, 5*time.Minute, time.Second), QueryLag: durEnv(envQueryLagSec, 30*time.Second, time.Second), ExportAllFloor: durEnv(envExportAllFloorSec, 30*time.Second, time.Second), + OrderChunk: durEnv(envOrderChunkSec, 60*time.Second, time.Second), // EXPORT_MODE=never → the kubescape trigger stops self-steering; only a // control client (dx) drives exports via /export/start + /query. DisableSelfSteer: strings.EqualFold(strings.TrimSpace(os.Getenv("EXPORT_MODE")), "never"), diff --git a/src/vizier/services/adaptive_export/internal/controller/controller.go b/src/vizier/services/adaptive_export/internal/controller/controller.go index 5a4b44ec1c4..ad6f0f8f91a 100644 --- a/src/vizier/services/adaptive_export/internal/controller/controller.go +++ b/src/vizier/services/adaptive_export/internal/controller/controller.go @@ -33,6 +33,7 @@ import ( "context" "errors" "fmt" + "strings" "sync" "time" @@ -135,6 +136,15 @@ type Config struct { // captures over overlapping windows. Defaulted to 30s in defaulted(). ExportAllFloor time.Duration + // OrderChunk is the sub-window span the ordered path (OrderQuery) walks the + // capture window in. Each chunk is a both-sides bounded pixie query, so no single + // query re-scans the whole (up to 600s) window on the node-local PEM — the fix for + // heavy tables (dc_snoop) losing the per-query deadline race under the + // OrderExportAll fan-out. A chunk that still times out under contention is + // adaptively halved down to orderMinChunk. Defaulted to defaultOrderChunk in + // defaulted(); env ADAPTIVE_ORDER_CHUNK_SEC overrides. + OrderChunk time.Duration + // === Throughput-protection knobs === // // At high anomaly rates (many concurrent active hashes), the default @@ -207,9 +217,24 @@ func (c *Config) defaulted() Config { if out.ExportAllFloor == 0 { out.ExportAllFloor = 30 * time.Second } + if out.OrderChunk == 0 { + out.OrderChunk = defaultOrderChunk + } return out } +const ( + // defaultOrderChunk is the sub-window the ordered path walks the capture window + // in. 60s over the 600s control lookback = 10 bounded queries per table, each + // cheap enough to complete well inside the 180s deadline even when 20 tables + // fan out concurrently against one node-local PEM. + defaultOrderChunk = 60 * time.Second + // orderMinChunk is the floor for adaptive subdivision: a span this small that + // still fails is surfaced rather than split further (a 1s window that can't be + // captured is a real error, not contention). + orderMinChunk = 1 * time.Second +) + // Controller is the live orchestrator. One instance per operator process. type Controller struct { trig Trigger @@ -295,23 +320,84 @@ func (c *Controller) OrderQuery(target anomaly.Target, table string, start, end return errors.New("controller: no pixie querier (operator-side push disabled)") } now := c.clock.Now() - q, err := pxl.QueryFor(table, target, start, end, now) - if err != nil { - return err + chunk := c.cfg.OrderChunk + if chunk <= 0 { + chunk = defaultOrderChunk + } + // Walk the window oldest→newest in fixed chunks. Each chunk is a both-sides + // bounded pixie query (QueryFor stamps end_time), so no single query re-scans the + // whole window on the node-local PEM — the flaky-capture fix. captureSpan halves + // any chunk that still times out under fan-out contention. Chunks run + // sequentially per table, so OrderExportAll's per-table concurrency (20 tables) + // is unchanged while each table now issues cheap bounded queries instead of one + // firehose. ReplacingMergeTree makes the overlapping/retried spans idempotent. + var readTotal, wroteTotal int + var firstErr error + for s := start; s.Before(end); s = s.Add(chunk) { + e := s.Add(chunk) + if e.After(end) { + e = end + } + qid := fmt.Sprintf("%s:%d-%d", queryID, s.Unix(), e.Unix()) + r, w, err := c.captureSpan(target, table, s, e, qid) + readTotal += r + wroteTotal += w + if err != nil && firstErr == nil { + firstErr = err + } + } + recErr := "" + if firstErr != nil { + recErr = firstErr.Error() + } + // One reconcile row per table, aggregating every chunk — the read/wrote counts a + // forensic dump reads stay per-table, not per-chunk. + c.cfg.Rec.Record(context.Background(), reconcile.Row{ + TS: now, Mode: "ordered", Table: table, + Namespace: target.Namespace, Pod: target.Pod, + WinStart: start, WinEnd: end, + ReadCount: int64(readTotal), WroteCount: int64(wroteTotal), + WriteErr: recErr, Hostname: c.cfg.Hostname, + }) + return firstErr +} + +// captureSpan captures [start,end) for one table, subdividing on a transient +// (deadline/overload) failure down to orderMinChunk. A span that times out under +// PEM contention is retried as two half-spans — each scans less data at the source +// (QueryFor bounds end_time), so a dense window that blows the 180s deadline as one +// query completes as several small ones. Idempotent: overlapping/retried spans +// dedupe in the ReplacingMergeTree evidence tables. Non-transient errors (e.g. a +// missing dark-vector table) surface immediately without wasteful splitting. +func (c *Controller) captureSpan(target anomaly.Target, table string, start, end time.Time, queryID string) (readCount, wroteCount int, err error) { + r, w, e := c.orderQuerySlice(target, table, start, end, queryID) + if e == nil || !isRetriableSpanErr(e) || end.Sub(start) <= orderMinChunk { + return r, w, e + } + log.WithError(e).WithFields(log.Fields{ + "table": table, "pod": target.Pod, "span": end.Sub(start).String(), + }).Warn("ordered capture: transient failure, subdividing span") + mid := start.Add(end.Sub(start) / 2) + r1, w1, e1 := c.captureSpan(target, table, start, mid, queryID+".l") + r2, w2, e2 := c.captureSpan(target, table, mid, end, queryID+".r") + if e1 != nil { + return r1 + r2, w1 + w2, e1 + } + return r1 + r2, w1 + w2, e2 +} + +// orderQuerySlice runs ONE bounded (target, table, [start,end)) capture: query +// pixie, write the rows, return the read/wrote counts. It records NO reconcile row — +// the OrderQuery driver aggregates across chunks and records once. globalSem still +// bounds broker load per slice. Background ctx with per-op timeouts mirrors +// pushPixieRows: a control-ordered capture completes independently of any anomaly +// window's lifecycle. +func (c *Controller) orderQuerySlice(target anomaly.Target, table string, start, end time.Time, queryID string) (readCount, wroteCount int, err error) { + now := c.clock.Now() + q, qerr := pxl.QueryFor(table, target, start, end, now) + if qerr != nil { + return 0, 0, qerr } - // Background ctx with per-op timeouts mirroring pushPixieRows: a control-ordered - // capture must complete independently of any anomaly window's lifecycle. - var readCount, wroteCount int - var recErr string - defer func() { - c.cfg.Rec.Record(context.Background(), reconcile.Row{ - TS: now, Mode: "ordered", Table: table, - Namespace: target.Namespace, Pod: target.Pod, - WinStart: start, WinEnd: end, - ReadCount: int64(readCount), WroteCount: int64(wroteCount), - WriteErr: recErr, Hostname: c.cfg.Hostname, - }) - }() if c.globalSem != nil { c.globalSem <- struct{}{} defer func() { <-c.globalSem }() @@ -320,25 +406,45 @@ func (c *Controller) OrderQuery(target anomaly.Target, table string, start, end rows, qerr := c.querier.Query(qctx, q) cancel() if qerr != nil { - recErr = qerr.Error() - return qerr + return 0, 0, qerr } - readCount = len(rows) if len(rows) == 0 { - return nil // nothing to persist; the read/0-wrote reconcile row still records it + return 0, 0, nil // nothing to persist; the driver's reconcile row still records the read } wctx, wcancel := context.WithTimeout(context.Background(), 60*time.Second) werr := c.sink.WritePixieRows(wctx, table, rows) wcancel() if werr != nil { - recErr = werr.Error() - return werr + return len(rows), 0, werr } - wroteCount = len(rows) log.WithFields(log.Fields{ "table": table, "rows": len(rows), "pod": target.Pod, "query_id": queryID, }).Info("ordered pixie rows written to forensic_db (dx→AE /query)") - return nil + return len(rows), len(rows), nil +} + +// isRetriableSpanErr reports whether a slice error is a transient overload/timeout +// worth retrying as a narrower span (vs. a structural error like a missing table, +// which no amount of subdivision fixes). Covers ctx deadlines and the gRPC status +// strings the pixie querier surfaces (DeadlineExceeded / ResourceExhausted / +// Unavailable), which do NOT satisfy errors.Is(context.DeadlineExceeded). +func isRetriableSpanErr(err error) bool { + if err == nil { + return false + } + if errors.Is(err, context.DeadlineExceeded) { + return true + } + s := strings.ToLower(err.Error()) + for _, m := range []string{ + "deadline", "timeout", "exceeded", "resourceexhausted", + "resource exhausted", "unavailable", "context canceled", "context cancelled", + } { + if strings.Contains(s, m) { + return true + } + } + return false } // OrderExportAll runs a one-shot OrderQuery for EVERY configured pixie table for diff --git a/src/vizier/services/adaptive_export/internal/controller/order_chunk_test.go b/src/vizier/services/adaptive_export/internal/controller/order_chunk_test.go new file mode 100644 index 00000000000..7557bf28433 --- /dev/null +++ b/src/vizier/services/adaptive_export/internal/controller/order_chunk_test.go @@ -0,0 +1,186 @@ +/* + * Copyright 2018- The Pixie Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package controller + +// Tests for the CHUNKED + adaptively-subdividing ordered capture path — the +// durable fix for heavy tables (dc_snoop) losing the per-query deadline race under +// the OrderExportAll fan-out. Each chunk is a both-sides bounded pixie query; a +// chunk that still times out under contention is halved down to orderMinChunk. + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "px.dev/pixie/src/vizier/services/adaptive_export/internal/reconcile" +) + +// countingQuerier counts Query calls and can fail the first failN calls with a +// configurable error (to exercise adaptive subdivision) or fail every call. +type countingQuerier struct { + mu sync.Mutex + calls int + rows []map[string]any + failN int // fail the first failN calls, then succeed + failAll bool // fail every call + failErr error // error to return on a failed call +} + +func (q *countingQuerier) Query(context.Context, string) ([]map[string]any, error) { + q.mu.Lock() + defer q.mu.Unlock() + q.calls++ + if q.failAll || q.calls <= q.failN { + return nil, q.failErr + } + return q.rows, nil +} + +func (q *countingQuerier) callCount() int { + q.mu.Lock() + defer q.mu.Unlock() + return q.calls +} + +// recordingRec captures every reconcile Row so a test can assert the ordered path +// records exactly ONE aggregated row per table (not one per chunk). +type recordingRec struct { + mu sync.Mutex + rows []reconcile.Row +} + +func (r *recordingRec) Record(_ context.Context, row reconcile.Row) { + r.mu.Lock() + defer r.mu.Unlock() + r.rows = append(r.rows, row) +} + +func chunkCtl(snk Sink, q PixieQuerier, rec reconcile.Recorder, chunk time.Duration) *Controller { + cfg := defaultCfg() + cfg.OrderChunk = chunk + cfg.Rec = rec + c := New(newFakeTrigger(), snk, cfg, &fakeClock{t: canonicalEventTime}) + if q != nil { + c = c.WithPixieQuerier(q) + } + return c +} + +var deadlineErr = errors.New("rpc error: code = DeadlineExceeded desc = context deadline exceeded") + +// A wide window is walked in OrderChunk-sized slices: one pixie query per chunk, +// each writing its rows. 180s window / 60s chunk = 3 bounded queries. +func TestOrderQueryChunksWideWindow(t *testing.T) { + snk := newRecordingSink() + q := &countingQuerier{rows: []map[string]any{{"comm": "whoami"}}} + end := canonicalEventTime + start := end.Add(-180 * time.Second) + if err := chunkCtl(snk, q, reconcile.Nop{}, 60*time.Second). + OrderQuery(oqTarget, "dc_snoop", start, end, "qid-w"); err != nil { + t.Fatalf("OrderQuery: %v", err) + } + if got := q.callCount(); got != 3 { + t.Errorf("want 3 chunk queries for a 180s/60s window, got %d", got) + } + if got := snk.count("dc_snoop"); got != 3 { + t.Errorf("want 3 rows written (one per chunk), got %d", got) + } +} + +// The ordered path records exactly ONE reconcile row per table, aggregating the +// per-chunk read/wrote counts — a forensic dump reads per-table, not per-chunk. +func TestOrderQuerySingleReconcileRowPerTable(t *testing.T) { + snk := newRecordingSink() + rec := &recordingRec{} + q := &countingQuerier{rows: []map[string]any{{"comm": "cat"}}} + end := canonicalEventTime + start := end.Add(-120 * time.Second) // 2 chunks + if err := chunkCtl(snk, q, rec, 60*time.Second). + OrderQuery(oqTarget, "dc_snoop", start, end, "qid-r"); err != nil { + t.Fatalf("OrderQuery: %v", err) + } + if len(rec.rows) != 1 { + t.Fatalf("want 1 aggregated reconcile row, got %d", len(rec.rows)) + } + if rec.rows[0].ReadCount != 2 || rec.rows[0].WroteCount != 2 { + t.Errorf("want aggregated read=2 wrote=2 across chunks, got read=%d wrote=%d", + rec.rows[0].ReadCount, rec.rows[0].WroteCount) + } + if rec.rows[0].WriteErr != "" { + t.Errorf("clean capture must record no error, got %q", rec.rows[0].WriteErr) + } +} + +// A chunk that fails with a TRANSIENT (deadline) error is retried as narrower +// half-spans and recovers — the flaky-capture fix. The querier fails only its first +// call, so the initial full-chunk query subdivides and the halves succeed. +func TestCaptureSpanSubdividesOnTransientError(t *testing.T) { + snk := newRecordingSink() + q := &countingQuerier{rows: []map[string]any{{"comm": "getent"}}, failN: 1, failErr: deadlineErr} + end := canonicalEventTime + start := end.Add(-8 * time.Second) // single 60s chunk covers it → one initial query + if err := chunkCtl(snk, q, reconcile.Nop{}, 60*time.Second). + OrderQuery(oqTarget, "dc_snoop", start, end, "qid-t"); err != nil { + t.Fatalf("transient failure must recover via subdivision, got %v", err) + } + // call 1 (8s span) fails → split into two 4s halves (calls 2 & 3), both succeed. + if got := q.callCount(); got != 3 { + t.Errorf("want 3 calls (1 failed + 2 half-span retries), got %d", got) + } + if got := snk.count("dc_snoop"); got != 2 { + t.Errorf("want 2 half-span writes after subdivision, got %d", got) + } +} + +// A NON-transient error (e.g. a missing dark-vector table) surfaces immediately — +// no wasteful subdivision. Exactly one query per chunk, error returned. +func TestCaptureSpanDoesNotSplitNonTransient(t *testing.T) { + snk := newRecordingSink() + q := &countingQuerier{failAll: true, failErr: errors.New("table 'dx_bpf' not found")} + end := canonicalEventTime + start := end.Add(-30 * time.Second) // < one chunk → single chunk + err := chunkCtl(snk, q, reconcile.Nop{}, 60*time.Second). + OrderQuery(oqTarget, "dx_bpf", start, end, "qid-n") + if err == nil { + t.Fatal("non-transient error must surface") + } + if got := q.callCount(); got != 1 { + t.Errorf("non-transient error must NOT subdivide; want 1 call, got %d", got) + } +} + +// A persistently-timing-out span subdivides down to orderMinChunk and then surfaces +// the error instead of looping forever — the recursion terminates at the floor. +func TestCaptureSpanTerminatesAtMinChunk(t *testing.T) { + snk := newRecordingSink() + q := &countingQuerier{failAll: true, failErr: deadlineErr} + end := canonicalEventTime + start := end.Add(-4 * time.Second) // 4s → 2s → 1s (floor), bounded call count + err := chunkCtl(snk, q, reconcile.Nop{}, 60*time.Second). + OrderQuery(oqTarget, "dc_snoop", start, end, "qid-f") + if err == nil { + t.Fatal("a span that never succeeds must ultimately surface the error") + } + // 4s→(2s,2s)→each (1s,1s): calls = 1 + 2 + 4 = 7, finite. Assert it stayed bounded. + if got := q.callCount(); got == 0 || got > 15 { + t.Errorf("subdivision must terminate at orderMinChunk with a bounded call count, got %d", got) + } +} diff --git a/src/vizier/services/adaptive_export/internal/pxl/queryfor.go b/src/vizier/services/adaptive_export/internal/pxl/queryfor.go index 4f9d8d6d37c..cb4f1afca47 100644 --- a/src/vizier/services/adaptive_export/internal/pxl/queryfor.go +++ b/src/vizier/services/adaptive_export/internal/pxl/queryfor.go @@ -63,7 +63,18 @@ func QueryFor(table string, t anomaly.Target, sliceStart, sliceEnd, now time.Tim var b strings.Builder b.WriteString(pxSetMaxRows) b.WriteString("import px\n") - b.WriteString("df = px.DataFrame(table='" + pixieSourceFor(table) + "', start_time='" + relStart + "')\n") + // Bound the PEM source scan on BOTH sides. Without an end_time the planner + // scans [sliceStart, now] for EVERY query, so an old ordered window (e.g. the + // 600s control lookback) re-materializes the whole span on the node-local PEM; + // under the OrderExportAll fan-out the heavy tables (dc_snoop) then blow the + // per-query deadline and drop out (the flaky-capture RCA). relEndBound caps the + // scan at ~sliceEnd; the exact upper bound is still trimmed by the df.time_ < + // sliceEnd nanos filter below, so nothing real is clipped. + dfArgs := "table='" + pixieSourceFor(table) + "', start_time='" + relStart + "'" + if relEnd := relEndBound(now, sliceEnd); relEnd != "" { + dfArgs += ", end_time='" + relEnd + "'" + } + b.WriteString("df = px.DataFrame(" + dfArgs + ")\n") b.WriteString("df = df[df.time_ >= px.int64_to_time(" + strconv.FormatInt(sliceStart.UnixNano(), 10) + ")]\n") b.WriteString("df = df[df.time_ < px.int64_to_time(" + strconv.FormatInt(sliceEnd.UnixNano(), 10) + ")]\n") // Native tables: px.upid_to_pod_name returns "/" (carnot: @@ -130,6 +141,21 @@ func QueryFor(table string, t anomaly.Target, sliceStart, sliceEnd, now time.Tim return b.String(), nil } +// relEndBound returns a RELATIVE end_time ("-s") that caps the PEM's source +// scan at ~sliceEnd, or "" when sliceEnd is at/after now (scan to the live edge). +// The gap is floored to whole seconds so the source window ends slightly LATER +// than sliceEnd and never clips real rows — the precise upper bound is enforced by +// the df.time_ < sliceEnd nanos post-filter. This is the load lever behind the +// chunked ordered path: each chunk materializes only its own span instead of +// [chunkStart, now]. +func relEndBound(now, sliceEnd time.Time) string { + gap := now.Sub(sliceEnd) + if gap < time.Second { + return "" // at/after now → default end_time (scan to now) + } + return "-" + strconv.FormatInt(int64(gap/time.Second), 10) + "s" +} + // pixieSourceFor returns the Pixie table a builtin is sourced FROM when it // differs from the ClickHouse table it is written TO. stack_trace is written to // CH as 'stack_trace' but sourced from the CANONICAL native continuous profiler diff --git a/src/vizier/services/adaptive_export/internal/pxl/queryfor_test.go b/src/vizier/services/adaptive_export/internal/pxl/queryfor_test.go index 562ea794cc0..9c9594525bb 100644 --- a/src/vizier/services/adaptive_export/internal/pxl/queryfor_test.go +++ b/src/vizier/services/adaptive_export/internal/pxl/queryfor_test.go @@ -18,6 +18,7 @@ package pxl import ( "errors" + "strconv" "strings" "testing" "time" @@ -340,3 +341,38 @@ func TestQueryFor_PodOnlyRegexEscapesQuoteMetaInjection(t *testing.T) { t.Fatalf("pod-only path injection succeeded:\n%s", q) } } + +// TestQueryFor_EndTimeBoundsPastWindow — a window whose upper bound is in the past +// must emit a relative end_time so the PEM scan is bounded on BOTH sides (not +// [sliceStart, now]). The precise upper bound is still enforced by the df.time_ < +// nanos post-filter. +func TestQueryFor_EndTimeBoundsPastWindow(t *testing.T) { + // sliceEnd 2 minutes before now → end_time must appear. + end := fixedNow.Add(-2 * time.Minute) + start := fixedNow.Add(-7 * time.Minute) + q, err := QueryFor("dc_snoop", target, start, end, fixedNow) + if err != nil { + t.Fatalf("QueryFor: %v", err) + } + if !strings.Contains(q, "end_time='-120s'") { + t.Fatalf("past-window query must bound the source scan with end_time='-120s'; got:\n%s", q) + } + // exact upper bound still trimmed precisely in nanos. + if !strings.Contains(q, "df = df[df.time_ < px.int64_to_time("+ + strconv.FormatInt(end.UnixNano(), 10)+")]") { + t.Fatalf("precise nanos upper-bound filter must remain; got:\n%s", q) + } +} + +// TestQueryFor_NoEndTimeAtLiveEdge — a window that reaches now must NOT emit +// end_time (scan to the live edge), preserving the pre-chunking behavior for the +// most-recent slice. +func TestQueryFor_NoEndTimeAtLiveEdge(t *testing.T) { + q, err := QueryFor("dc_snoop", target, fixedNow.Add(-1*time.Minute), fixedNow, fixedNow) + if err != nil { + t.Fatalf("QueryFor: %v", err) + } + if strings.Contains(q, "end_time=") { + t.Fatalf("live-edge window must not bound end_time; got:\n%s", q) + } +} From 743fbd90c6cc67e0cac83f61e6463185a4b7c983 Mon Sep 17 00:00:00 2001 From: entlein Date: Sun, 2 Aug 2026 14:52:12 +0200 Subject: [PATCH 02/66] adaptive_export/pxl: filter infra noise from node-scoped dark capture (dc_snoop) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The dx-steered OrderExportAll path applied only a partial comm denylist and NO namespace filter to the node-scoped dark-vector tables — unlike the shipped cron preset (script/presets dc_snoop.pxl __DC_SNOOP_EXCLUSION__, built from presets.go defaultExcludeNamespaces + defaultExcludeComms). So every dc_snoop capture drowned in infra dcache churn: on a real k3s node a single window returned ~54k rows dominated by ConfigReloader/iptables/CNI(host-local,bridge,flannel,loopback)/host daemons(systemd-udevd,dbus-daemon,tailscaled)/kubevuln — burying the salient attack specimens (whoami/cat/getent reading /etc/shadow + the SA token). - Extend darkExcludeCommsDefault with the host/CNI/node daemons that were leaking (systemd-udevd, host-local, bridge, flannel, loopback, bandwidth, dbus-daemon, mount, umount, tailscaled, grpc_health_pro, kubevuln, opm, kube-proxy, …). - Add darkExcludeNamespacesDefault + darkNamespaceExclusion(), applied in the IsDarkVector branch AFTER PodEnrichPxL resolves df.namespace, dropping infra namespaces (pl, kube-system, clickhouse, …). Blank-namespace transient rows survive (each `!=` is true for ''), so the attack's short-lived children — which resolve blank — are never dropped. Overridable via DC_SNOOP_EXCLUDE_NAMESPACES. Kept in sync with script/presets.go. Tests: infra namespaces + host/CNI comms dropped; df.namespace never pinned to the alert pod (node-scoped); env override replaces the default list. --- .../adaptive_export/internal/pxl/queryfor.go | 45 +++++++++++++++++++ .../internal/pxl/queryfor_test.go | 42 +++++++++++++++++ 2 files changed, 87 insertions(+) diff --git a/src/vizier/services/adaptive_export/internal/pxl/queryfor.go b/src/vizier/services/adaptive_export/internal/pxl/queryfor.go index cb4f1afca47..cfe0bc17707 100644 --- a/src/vizier/services/adaptive_export/internal/pxl/queryfor.go +++ b/src/vizier/services/adaptive_export/internal/pxl/queryfor.go @@ -121,6 +121,13 @@ func QueryFor(table string, t anomaly.Target, sliceStart, sliceEnd, now time.Tim // of workload rows (bash/redis/whoami/cat), so the dark capture completes. b.WriteString(darkCommExclusion(table)) b.WriteString(PodEnrichPxL(table)) + // AFTER the pid-merge resolves df.namespace: drop infra/system namespaces + // (blank-namespace transient workload rows are KEPT — see darkCommExclusion + // note; the attack's short-lived children resolve blank). This mirrors the + // shipped cron preset (script/presets dc_snoop.pxl __DC_SNOOP_EXCLUSION__); + // the OrderExportAll path was missing it, so infra pods' dcache churn + // (ConfigReloader/iptables/CNI/host daemons) flooded every dc_snoop capture. + b.WriteString(darkNamespaceExclusion()) } else { b.WriteString(PodEnrichPxL(table)) if t.Namespace != "" { @@ -191,6 +198,23 @@ var darkExcludeCommsDefault = []string{ "ConfigReloader", "clickhouse-oper", "Formatter", "(setup.sh)", "cmd", "vector-worker", "metrics-server", "local-path-prov", "portmap", "(udev-worker)", "systemd-resolve", "systemd-timesyn", + // host/CNI/node daemons that flood dc_snoop with dcache churn but carry no + // workload forensic value (observed leaking on a real k3s node, aeprod54). + "systemd-udevd", "systemd-sysctl", "host-local", "bridge", "flannel", + "loopback", "bandwidth", "dbus-daemon", "mount", "umount", "tailscaled", + "grpc_health_pro", "kubevuln", "opm", "(spawn)", "kube-proxy", +} + +// darkExcludeNamespacesDefault drops infra/system namespaces from the node-scoped +// dark capture (blank-namespace transient workload rows are KEPT — the attack's +// short-lived children resolve blank, so a namespace filter must never drop them). +// Overridable via DC_SNOOP_EXCLUDE_NAMESPACES (csv). Kept in sync with +// script/presets.go defaultExcludeNamespaces — the shipped cron path already +// filtered these; the dx-steered OrderExportAll path did not, so infra pods' +// process churn flooded every capture. +var darkExcludeNamespacesDefault = []string{ + "pl", "honey", "px-operator", "olm", "clickhouse", "socdemo", "socdemo-ch", + "kube-system", "kube-public", "kube-node-lease", "local-path-storage", } // darkCommExclusion builds the infra-comm drop filter for a dark-vector table @@ -215,6 +239,27 @@ func darkCommExclusion(table string) string { return b.String() } +// darkNamespaceExclusion builds the infra-namespace drop filter for the node-scoped +// dark capture. Emitted AFTER PodEnrichPxL resolves df.namespace. Blank-namespace +// rows survive (each `!=` predicate is true for ”), so transient attack children +// are never dropped. Overridable via DC_SNOOP_EXCLUDE_NAMESPACES (csv). +func darkNamespaceExclusion() string { + nss := darkExcludeNamespacesDefault + if v := strings.TrimSpace(os.Getenv("DC_SNOOP_EXCLUDE_NAMESPACES")); v != "" { + nss = nil + for _, s := range strings.Split(v, ",") { + if s = strings.TrimSpace(s); s != "" { + nss = append(nss, s) + } + } + } + var b strings.Builder + for _, ns := range nss { + b.WriteString("df = df[df.namespace != '" + escapePxL(ns) + "']\n") + } + return b.String() +} + // pxlEscaper turns raw bytes that could break out of a PxL single-quoted // string into their Python-style escape sequences. The backslash MUST be // mapped FIRST so its own substitution doesn't get double-escaped when diff --git a/src/vizier/services/adaptive_export/internal/pxl/queryfor_test.go b/src/vizier/services/adaptive_export/internal/pxl/queryfor_test.go index 9c9594525bb..fa2e121ebf3 100644 --- a/src/vizier/services/adaptive_export/internal/pxl/queryfor_test.go +++ b/src/vizier/services/adaptive_export/internal/pxl/queryfor_test.go @@ -376,3 +376,45 @@ func TestQueryFor_NoEndTimeAtLiveEdge(t *testing.T) { t.Fatalf("live-edge window must not bound end_time; got:\n%s", q) } } + +// TestQueryFor_DarkNamespaceExclusion — the node-scoped dark capture (dc_snoop) +// must drop infra namespaces (pl, kube-system, …) while KEEPING blank-namespace +// transient rows (the attack's short-lived children). Mirrors the shipped preset. +func TestQueryFor_DarkNamespaceExclusion(t *testing.T) { + q, err := QueryFor("dc_snoop", target, fixedStart, fixedEnd, fixedNow) + if err != nil { + t.Fatalf("QueryFor: %v", err) + } + // namespace drops present for infra + for _, ns := range []string{"pl", "kube-system", "clickhouse"} { + if !strings.Contains(q, "df = df[df.namespace != '"+ns+"']") { + t.Errorf("dark capture must drop infra namespace %q; got:\n%s", ns, q) + } + } + // must NOT pin to the alert pod's namespace (node-scoped keeps blank + other workloads) + if strings.Contains(q, "df = df[df.namespace == '") { + t.Errorf("dark capture must not pin df.namespace ==; got:\n%s", q) + } + // host/CNI comm drops present + for _, c := range []string{"host-local", "systemd-udevd", "tailscaled", "kubevuln"} { + if !strings.Contains(q, "df = df[df.comm != '"+c+"']") { + t.Errorf("dark capture must drop host/CNI comm %q; got:\n%s", c, q) + } + } +} + +// TestQueryFor_DarkNamespaceExclusion_EnvOverride — DC_SNOOP_EXCLUDE_NAMESPACES +// replaces the default list. +func TestQueryFor_DarkNamespaceExclusion_EnvOverride(t *testing.T) { + t.Setenv("DC_SNOOP_EXCLUDE_NAMESPACES", "foo,bar") + q, err := QueryFor("dc_snoop", target, fixedStart, fixedEnd, fixedNow) + if err != nil { + t.Fatalf("QueryFor: %v", err) + } + if !strings.Contains(q, "df = df[df.namespace != 'foo']") || !strings.Contains(q, "df = df[df.namespace != 'bar']") { + t.Errorf("env override must emit foo/bar drops; got:\n%s", q) + } + if strings.Contains(q, "df = df[df.namespace != 'pl']") { + t.Errorf("env override must REPLACE the default (no 'pl'); got:\n%s", q) + } +} From 831162a5d09fba06eccf4e321adcb9a03277d54e Mon Sep 17 00:00:00 2001 From: entlein Date: Sun, 2 Aug 2026 14:58:24 +0200 Subject: [PATCH 03/66] adaptive_export: bound ordered-capture subdivision (circuit-breaker + depth cap) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Live RCA on aeprod54: the chunk fix is correct in isolation (pem unit suite — dc_snoop 54k, redis/conn/stack written per-chunk) but UNSAFE under the dx steering firehose. dx does generic collect-per-alert, so OrderExportAll (20 tables) fires on every noisy pl system pod continuously; all land on the ONE node-local PEM (pem-direct) → it saturates → 100% DeadlineExceeded. captureSpan then split every timeout into two narrower retries, amplifying a busy PEM into a query storm where nothing completes (observed: "0 ordered pixie rows written" across the whole run; draining dx + restarting AE → pem-direct instantly serves again). Make subdivision safe: - Circuit-breaker: orderTimeoutStreak (atomic) counts CONSECUTIVE transient failures; any success resets it. Above orderBreakerTrip (8) captureSpan stops subdividing — a saturated PEM must not be flooded with retries. It still splits a genuinely-oversized window on a healthy PEM (the reset keeps that path live). - Depth cap: maxOrderSplitDepth (3) bounds one chunk to ≤2^3 leaf queries even if it keeps timing out (was ~64 splitting 60s→1s). Tests: a 10-chunk all-timeout window stays <60 queries (ungated ≈640); a single transient failure still recovers (breaker resets on success, no latch). NOTE (deployment, not code): the firehose root also needs dx steering scoped so it doesn't fire 20-table captures on every noisy pl/system-pod alert — tracked separately for dx-agent. --- .../internal/controller/controller.go | 54 +++++++++++++++++-- .../internal/controller/order_chunk_test.go | 37 +++++++++++++ 2 files changed, 86 insertions(+), 5 deletions(-) diff --git a/src/vizier/services/adaptive_export/internal/controller/controller.go b/src/vizier/services/adaptive_export/internal/controller/controller.go index ad6f0f8f91a..c65de83f678 100644 --- a/src/vizier/services/adaptive_export/internal/controller/controller.go +++ b/src/vizier/services/adaptive_export/internal/controller/controller.go @@ -35,6 +35,7 @@ import ( "fmt" "strings" "sync" + "sync/atomic" "time" log "github.com/sirupsen/logrus" @@ -268,8 +269,29 @@ type Controller struct { exportAllMu sync.Mutex exportAllAt map[string]time.Time // per-target floor for OrderExportAll (steer-all) + + // orderTimeoutStreak counts CONSECUTIVE transient (deadline/overload) failures on + // the ordered path across all in-flight captures. Any successful ordered query + // resets it to 0. When it exceeds orderBreakerTrip the node-local PEM is treated + // as saturated and captureSpan STOPS subdividing (fails fast) — otherwise each + // timeout would spawn two narrower retries, and under the dx steering firehose + // (OrderExportAll × 20 tables × every noisy pod) that amplification turns a busy + // PEM into a query storm where nothing completes. The breaker makes subdivision + // safe: it splits a genuinely-too-large window on a healthy PEM, but never floods + // a saturated one. + orderTimeoutStreak atomic.Int32 } +const ( + // maxOrderSplitDepth caps captureSpan recursion so one chunk can spawn at most + // 2^depth leaf queries even if it keeps timing out (3 → ≤8, vs. ~64 splitting a + // 60s chunk to the 1s floor). Bounds worst-case amplification per chunk. + maxOrderSplitDepth = 3 + // orderBreakerTrip is the consecutive-timeout count above which captureSpan stops + // subdividing (PEM saturated → splitting only makes it worse). Reset by any success. + orderBreakerTrip = 8 +) + // New wires a Controller. nil clock falls through to RealClock. // nil querier disables the rev-1 push path (controller will only // write attribution rows; expects cloud's retention plugin to write @@ -339,7 +361,7 @@ func (c *Controller) OrderQuery(target anomaly.Target, table string, start, end e = end } qid := fmt.Sprintf("%s:%d-%d", queryID, s.Unix(), e.Unix()) - r, w, err := c.captureSpan(target, table, s, e, qid) + r, w, err := c.captureSpan(target, table, s, e, qid, 0) readTotal += r wroteTotal += w if err != nil && firstErr == nil { @@ -369,17 +391,32 @@ func (c *Controller) OrderQuery(target anomaly.Target, table string, start, end // query completes as several small ones. Idempotent: overlapping/retried spans // dedupe in the ReplacingMergeTree evidence tables. Non-transient errors (e.g. a // missing dark-vector table) surface immediately without wasteful splitting. -func (c *Controller) captureSpan(target anomaly.Target, table string, start, end time.Time, queryID string) (readCount, wroteCount int, err error) { +// +// Subdivision is bounded to stay SAFE under load: it stops at (a) the orderMinChunk +// floor, (b) maxOrderSplitDepth (worst case ≤2^depth leaves per chunk), and (c) the +// saturation circuit-breaker (orderTimeoutStreak > orderBreakerTrip). Without these, +// a saturated node-local PEM under the dx steering firehose turns every timeout into +// two retries → a query storm where nothing completes. +func (c *Controller) captureSpan(target anomaly.Target, table string, start, end time.Time, queryID string, depth int) (readCount, wroteCount int, err error) { r, w, e := c.orderQuerySlice(target, table, start, end, queryID) if e == nil || !isRetriableSpanErr(e) || end.Sub(start) <= orderMinChunk { return r, w, e } + if depth >= maxOrderSplitDepth { + return r, w, e // depth-capped: don't amplify a persistently-failing span + } + if c.orderTimeoutStreak.Load() > orderBreakerTrip { + // PEM saturated (sustained timeouts) — splitting would only add load. + log.WithFields(log.Fields{"table": table, "pod": target.Pod}). + Warn("ordered capture: circuit-breaker open (PEM saturated), not subdividing") + return r, w, e + } log.WithError(e).WithFields(log.Fields{ - "table": table, "pod": target.Pod, "span": end.Sub(start).String(), + "table": table, "pod": target.Pod, "span": end.Sub(start).String(), "depth": depth, }).Warn("ordered capture: transient failure, subdividing span") mid := start.Add(end.Sub(start) / 2) - r1, w1, e1 := c.captureSpan(target, table, start, mid, queryID+".l") - r2, w2, e2 := c.captureSpan(target, table, mid, end, queryID+".r") + r1, w1, e1 := c.captureSpan(target, table, start, mid, queryID+".l", depth+1) + r2, w2, e2 := c.captureSpan(target, table, mid, end, queryID+".r", depth+1) if e1 != nil { return r1 + r2, w1 + w2, e1 } @@ -406,8 +443,15 @@ func (c *Controller) orderQuerySlice(target anomaly.Target, table string, start, rows, qerr := c.querier.Query(qctx, q) cancel() if qerr != nil { + // Feed the saturation circuit-breaker: sustained transient failures mean the + // node-local PEM is overloaded, so captureSpan should stop subdividing. + if isRetriableSpanErr(qerr) { + c.orderTimeoutStreak.Add(1) + } return 0, 0, qerr } + // A completed query means the PEM is serving — clear the breaker. + c.orderTimeoutStreak.Store(0) if len(rows) == 0 { return 0, 0, nil // nothing to persist; the driver's reconcile row still records the read } diff --git a/src/vizier/services/adaptive_export/internal/controller/order_chunk_test.go b/src/vizier/services/adaptive_export/internal/controller/order_chunk_test.go index 7557bf28433..a949767cf90 100644 --- a/src/vizier/services/adaptive_export/internal/controller/order_chunk_test.go +++ b/src/vizier/services/adaptive_export/internal/controller/order_chunk_test.go @@ -184,3 +184,40 @@ func TestCaptureSpanTerminatesAtMinChunk(t *testing.T) { t.Errorf("subdivision must terminate at orderMinChunk with a bounded call count, got %d", got) } } + +// A persistently-timing-out multi-chunk window must NOT explode into a query storm. +// Without guards, 10 chunks each subdividing 60s→1s ≈ 10×64 = 640 queries against a +// saturated PEM. The depth cap (≤2^3 leaves/chunk) + circuit-breaker (stop +// subdividing after orderBreakerTrip consecutive timeouts) bound it hard. +func TestOrderQueryCircuitBreakerBoundsStorm(t *testing.T) { + snk := newRecordingSink() + q := &countingQuerier{failAll: true, failErr: deadlineErr} + end := canonicalEventTime + start := end.Add(-600 * time.Second) // 10 chunks @ 60s, all time out + _ = chunkCtl(snk, q, reconcile.Nop{}, 60*time.Second). + OrderQuery(oqTarget, "dc_snoop", start, end, "qid-storm") + got := q.callCount() + if got > 60 { + t.Errorf("depth-cap + circuit-breaker must bound the storm; got %d calls (want <=60, ungrafted would be ~640)", got) + } + if got < 10 { + t.Errorf("must still attempt each of the 10 chunks at least once; got %d", got) + } +} + +// A healthy PEM (queries succeed) must NOT trip the breaker — subdivision stays +// available for genuinely-oversized windows. A querier that fails ONCE then succeeds +// still subdivides and recovers (breaker reset by the success). +func TestCircuitBreakerResetsOnSuccess(t *testing.T) { + snk := newRecordingSink() + q := &countingQuerier{rows: []map[string]any{{"comm": "cat"}}, failN: 1, failErr: deadlineErr} + end := canonicalEventTime + start := end.Add(-8 * time.Second) + if err := chunkCtl(snk, q, reconcile.Nop{}, 60*time.Second). + OrderQuery(oqTarget, "dc_snoop", start, end, "qid-reset"); err != nil { + t.Fatalf("single transient failure must recover (breaker must not latch); got %v", err) + } + if snk.count("dc_snoop") < 1 { + t.Errorf("recovered subdivision must write rows; got %d", snk.count("dc_snoop")) + } +} From 7645a58bae99eec950b233ee510606a373051722 Mon Sep 17 00:00:00 2001 From: entlein Date: Sun, 2 Aug 2026 16:17:28 +0200 Subject: [PATCH 04/66] adaptive_export/control: widen near-zero /query windows to the lookback MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Live RCA (aeprod55): every dx-steered capture in the e2e returned 0 rows, and the reconcile showed why — all 36 ordered captures had ~512ns-wide windows (width_s=0), so they matched no pixie rows. /export/start already reaches back controlExportLookback, but a control client that keys the /query window on a single finding's event_time sends lo≈hi (a sub-microsecond span). That passes the lo=5s (hi preserved); a 120s window is untouched. NOTE (dx-agent): dx should send a real window (or use /export/start) rather than a point window per finding — tracked separately. This is the AE-side safety net. --- .../internal/control/server.go | 21 +++++++- .../internal/control/server_test.go | 52 ++++++++++++++++++- 2 files changed, 69 insertions(+), 4 deletions(-) diff --git a/src/vizier/services/adaptive_export/internal/control/server.go b/src/vizier/services/adaptive_export/internal/control/server.go index 96292715fcb..bfd53fe6673 100644 --- a/src/vizier/services/adaptive_export/internal/control/server.go +++ b/src/vizier/services/adaptive_export/internal/control/server.go @@ -67,6 +67,13 @@ type exportAller interface { // anomaly is comfortably inside the pulled slice. const controlExportLookback = 600 * time.Second +// minControlQueryWindow is the floor for a /query window. A control client that +// keys the window on a single finding's timestamp can send a sub-microsecond span +// (lo≈hi) that passes the lo= %v; got %v", minControlQueryWindow, got) + } + // hi must be preserved (we widen the lower bound only). + if rn.lastEnd.UnixNano() != hi { + t.Errorf("hi must be preserved; want %d got %d", hi, rn.lastEnd.UnixNano()) + } +} + +// A comfortably-wide window is passed through unchanged (no over-widening). +func TestQueryWideWindowUnchanged(t *testing.T) { + rn := &fakeRunner{} + srv := New(&fakeExporter{}, rn) + hi := int64(1_000_000_000_000) // 1000s in ns, so lo stays positive + lo := hi - int64(120*time.Second) + resp := do(t, srv, http.MethodPost, "/query", + `{"pod":"p","namespace":"redis-demo","table":"dc_snoop","query_id":"q2","window":[`+ + itoa(lo)+`,`+itoa(hi)+`]}`) + if resp.StatusCode != http.StatusAccepted { + t.Fatalf("want 202, got %d", resp.StatusCode) + } + if got := rn.lastEnd.Sub(rn.lastStart); got != 120*time.Second { + t.Errorf("wide window must pass through unchanged; want 120s got %v", got) + } +} + +func itoa(n int64) string { return strconv.FormatInt(n, 10) } From 438bc3cceba123aac5ad061929ff25bae7befc9c Mon Sep 17 00:00:00 2001 From: entlein Date: Mon, 3 Aug 2026 21:10:11 +0200 Subject: [PATCH 05/66] k8s/vizier: adaptive-export bootstrap = functional pem-direct DaemonSet MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bootstrap manifest was a replicas:0 Deployment with minimal env (EXPORT_MODE= auto, no pem-direct, no throttle) — it never ran and could not do node-local pem-direct. Replace it with the working config that the e2e RCA validated: - DaemonSet (one-per-node) so each pod queries its OWN node's vizier-pem at HOST_IP:50305 (pem-direct: node-local, desync-immune). - dx-steered: EXPORT_MODE=never + CONTROL_ADDR=:9100 + the control Service (internalTrafficPolicy:Local so dx reaches its co-located AE). - PEM-protection: ADAPTIVE_MAX_INFLIGHT_QUERIES_GLOBAL=4 and ADAPTIVE_ORDER_CHUNK_SEC =600 (one query per table, no window pre-chunking) so the AE never saturates the single node-local PEM it shares with dx. See RCA_ae_capture_20260803. Secret still seeded per-cluster (unchanged). --- .../bootstrap/adaptive_export_deployment.yaml | 149 ++++++++---------- 1 file changed, 69 insertions(+), 80 deletions(-) diff --git a/k8s/vizier/bootstrap/adaptive_export_deployment.yaml b/k8s/vizier/bootstrap/adaptive_export_deployment.yaml index 2db195ff408..f7a3a4b6ba3 100644 --- a/k8s/vizier/bootstrap/adaptive_export_deployment.yaml +++ b/k8s/vizier/bootstrap/adaptive_export_deployment.yaml @@ -1,115 +1,104 @@ --- +# adaptive-export: node-local forensic capture operator. DaemonSet (not a +# Deployment) because it defaults to PEM-DIRECT — each pod queries its OWN node's +# vizier-pem at HOST_IP:50305, so it must run one-per-node and be co-located with +# the PEM whose data it reads. (The old replicas:0 Deployment never ran and could +# not do node-local pem-direct.) +# +# Secret pl-adaptive-export-secrets (pixie-api-key + clickhouse-dsn) is NOT bundled +# in the kustomization — seed it per-cluster (see adaptive_export_secrets.yaml). apiVersion: apps/v1 -kind: Deployment +kind: DaemonSet metadata: name: adaptive-export + labels: { name: adaptive-export, plane: control } spec: - replicas: 0 selector: - matchLabels: - name: adaptive-export + matchLabels: { name: adaptive-export } template: metadata: - labels: - name: adaptive-export - plane: control + labels: { name: adaptive-export, plane: control } spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - # The beta.kubernetes.io/os label has been deprecated since - # k8s v1.14; every modern kubelet sets kubernetes.io/os. The - # single term below is enough — kept both ORed terms in the - # past for pre-1.14 compatibility. - matchExpressions: - - key: kubernetes.io/os - operator: In - values: - - linux + - { key: kubernetes.io/os, operator: In, values: [linux] } serviceAccountName: pl-adaptive-export-service-account containers: - name: adaptive-export image: vizier-adaptive_export_image:latest - # Bounded so AE can never memory-pressure a node (measured: AE uses - # only ~16-38Mi steady; passthrough with the raised 1M-row cap can - # spike, so 1Gi caps the worst case). CPU was pinned at the old 300m - # limit under concurrent passthrough → raised to 1 core. + ports: + - { name: control, containerPort: 9100, hostPort: 9100 } resources: - requests: - cpu: 200m - memory: 128Mi - limits: - cpu: "1" - memory: 1Gi + requests: { cpu: 100m, memory: 128Mi } + limits: { cpu: "1", memory: 1Gi } env: + # --- pem-direct: query THIS node's own vizier-pem (node-local, desync-immune) + - name: HOST_IP + valueFrom: { fieldRef: { fieldPath: status.hostIP } } + - name: ADAPTIVE_VIZIER_DIRECT_ADDR + value: "$(HOST_IP):50305" + - name: PL_JWT_SIGNING_KEY + valueFrom: { secretKeyRef: { name: pl-cluster-secrets, key: jwt-signing-key } } + - name: PX_DISABLE_TLS + value: "1" - name: PL_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace + valueFrom: { fieldRef: { fieldPath: metadata.namespace } } + - name: NODE_NAME + valueFrom: { fieldRef: { fieldPath: spec.nodeName } } - name: PIXIE_API_KEY - valueFrom: - secretKeyRef: - name: pl-adaptive-export-secrets - key: pixie-api-key + valueFrom: { secretKeyRef: { name: pl-adaptive-export-secrets, key: pixie-api-key } } - name: CLICKHOUSE_DSN - valueFrom: - secretKeyRef: - name: pl-adaptive-export-secrets - key: clickhouse-dsn - - name: VERBOSE - value: "true" - - name: DETECTION_INTERVAL_SEC - value: "10" - - name: DETECTION_LOOKBACK_SEC - value: "30" - # EXPORT_MODE controls the reconcile behaviour: - # auto - detection drives on/off (default) - # always - plugin always enabled (bypass detection) - # never - plugin always disabled and ch-* scripts purged - - name: EXPORT_MODE - value: "auto" - # Number of consecutive empty detection ticks before auto-disable fires. - - name: EXPORT_QUIET_TICKS - value: "6" - # Optional overrides for the ClickHouse PxL scripts. When unset they are - # parsed from CLICKHOUSE_DSN. Individual fields win over the parsed DSN. - # Defaults below match soc/tree/clickhouse-lab (forensic-soc-db CHI, - # ingest_writer user, forensic_db database). + valueFrom: { secretKeyRef: { name: pl-adaptive-export-secrets, key: clickhouse-dsn } } - name: KUBESCAPE_TABLE value: "kubescape_logs" - # - name: CLICKHOUSE_HOST - # value: "clickhouse-forensic-soc-db.clickhouse.svc.cluster.local" - # - name: CLICKHOUSE_PORT - # value: "9000" - # - name: CLICKHOUSE_USER - # value: "ingest_writer" - # - name: CLICKHOUSE_PASSWORD - # value: "changeme-ingest" - # - name: CLICKHOUSE_DATABASE - # value: "forensic_db" - # TLS for the control surface (CONTROL_TLS=true). server.crt/key from the - # same service-tls-certs secret the broker/PEM use; without this the dx - # bearer JWT crosses the CNI in cleartext. Harmless when control is off. + # --- dx-steered capture: dx drives OrderExportAll/OrderQuery via the control + # surface; self-steer off so only dx opens capture windows. + - name: EXPORT_MODE + value: "never" + - name: CONTROL_ADDR + value: ":9100" + - name: ADAPTIVE_PUSH_PIXIE_ROWS + value: "true" + - name: ADAPTIVE_RECONCILE + value: "true" + - name: DEPLOY_TRACEPOINTS + value: "true" + - name: INSTALL_PRESET_SCRIPTS + value: "false" + # --- PEM-protection: one node-local PEM serves both dx and AE, so cap the + # AE's concurrent queries and issue ONE query per table (no window + # pre-chunking) to avoid saturating it. See RCA_ae_capture_20260803. + - name: ADAPTIVE_MAX_INFLIGHT_QUERIES_GLOBAL + value: "4" + - name: ADAPTIVE_ORDER_CHUNK_SEC + value: "600" + - name: VERBOSE + value: "true" volumeMounts: - - name: certs - mountPath: /certs - readOnly: true + - { name: certs, mountPath: /certs, readOnly: true } securityContext: allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - seccompProfile: - type: RuntimeDefault + capabilities: { drop: [ALL] } + seccompProfile: { type: RuntimeDefault } volumes: - name: certs - secret: - secretName: service-tls-certs + secret: { secretName: service-tls-certs } securityContext: runAsUser: 10100 runAsGroup: 10100 fsGroup: 10100 runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + seccompProfile: { type: RuntimeDefault } +--- +apiVersion: v1 +kind: Service +metadata: + name: adaptive-export-control +spec: + selector: { name: adaptive-export } + internalTrafficPolicy: Local # dx reaches its co-located (same-node) AE + ports: + - { name: control, port: 9100, targetPort: 9100 } From 279b436a57f7beb1dac7dbc95e209a2739206349 Mon Sep 17 00:00:00 2001 From: entlein Date: Tue, 4 Aug 2026 08:55:18 +0200 Subject: [PATCH 06/66] adaptive_export: dc_snoop kernel-thread comm filter + adaptive-only default; trim comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - queryfor.go: add darkExcludeCommSubstrings (kworker/ksoftirqd/rcu_/… — kernel threads with variable suffixes exact-match misses) applied via px.logicalNot( px.contains); add pause + systemd-logind exact. Workload comms (redis-*) untouched. - controller.go: defaultOrderChunk 60s -> 600s (one query per table; pre-chunking 10x-amplified queries on the single node-local PEM). - Strip verbose comments across queryfor.go/controller.go/server.go + the AE manifest. Test: kernel-thread substrings dropped, workload comms kept, pause dropped. --- .../bootstrap/adaptive_export_deployment.yaml | 16 +--- .../internal/control/server.go | 15 +-- .../internal/controller/controller.go | 92 +++++------------- .../adaptive_export/internal/pxl/queryfor.go | 94 +++++-------------- .../internal/pxl/queryfor_test.go | 21 +++++ 5 files changed, 71 insertions(+), 167 deletions(-) diff --git a/k8s/vizier/bootstrap/adaptive_export_deployment.yaml b/k8s/vizier/bootstrap/adaptive_export_deployment.yaml index f7a3a4b6ba3..3e6609594b5 100644 --- a/k8s/vizier/bootstrap/adaptive_export_deployment.yaml +++ b/k8s/vizier/bootstrap/adaptive_export_deployment.yaml @@ -1,12 +1,6 @@ --- -# adaptive-export: node-local forensic capture operator. DaemonSet (not a -# Deployment) because it defaults to PEM-DIRECT — each pod queries its OWN node's -# vizier-pem at HOST_IP:50305, so it must run one-per-node and be co-located with -# the PEM whose data it reads. (The old replicas:0 Deployment never ran and could -# not do node-local pem-direct.) -# -# Secret pl-adaptive-export-secrets (pixie-api-key + clickhouse-dsn) is NOT bundled -# in the kustomization — seed it per-cluster (see adaptive_export_secrets.yaml). +# adaptive-export: node-local forensic capture operator. DaemonSet so each pod +# queries its own node's vizier-pem (pem-direct). Secret seeded per-cluster. apiVersion: apps/v1 kind: DaemonSet metadata: @@ -35,7 +29,6 @@ spec: requests: { cpu: 100m, memory: 128Mi } limits: { cpu: "1", memory: 1Gi } env: - # --- pem-direct: query THIS node's own vizier-pem (node-local, desync-immune) - name: HOST_IP valueFrom: { fieldRef: { fieldPath: status.hostIP } } - name: ADAPTIVE_VIZIER_DIRECT_ADDR @@ -54,8 +47,6 @@ spec: valueFrom: { secretKeyRef: { name: pl-adaptive-export-secrets, key: clickhouse-dsn } } - name: KUBESCAPE_TABLE value: "kubescape_logs" - # --- dx-steered capture: dx drives OrderExportAll/OrderQuery via the control - # surface; self-steer off so only dx opens capture windows. - name: EXPORT_MODE value: "never" - name: CONTROL_ADDR @@ -68,9 +59,6 @@ spec: value: "true" - name: INSTALL_PRESET_SCRIPTS value: "false" - # --- PEM-protection: one node-local PEM serves both dx and AE, so cap the - # AE's concurrent queries and issue ONE query per table (no window - # pre-chunking) to avoid saturating it. See RCA_ae_capture_20260803. - name: ADAPTIVE_MAX_INFLIGHT_QUERIES_GLOBAL value: "4" - name: ADAPTIVE_ORDER_CHUNK_SEC diff --git a/src/vizier/services/adaptive_export/internal/control/server.go b/src/vizier/services/adaptive_export/internal/control/server.go index bfd53fe6673..1f049af5f28 100644 --- a/src/vizier/services/adaptive_export/internal/control/server.go +++ b/src/vizier/services/adaptive_export/internal/control/server.go @@ -67,11 +67,8 @@ type exportAller interface { // anomaly is comfortably inside the pulled slice. const controlExportLookback = 600 * time.Second -// minControlQueryWindow is the floor for a /query window. A control client that -// keys the window on a single finding's timestamp can send a sub-microsecond span -// (lo≈hi) that passes the lo orderBreakerTrip). Without these, -// a saturated node-local PEM under the dx steering firehose turns every timeout into -// two retries → a query storm where nothing completes. +// captureSpan captures [start,end) for one table, subdividing a transient failure into +// half-spans down to orderMinChunk. Bounded by maxOrderSplitDepth + the breaker so a +// saturated PEM isn't stormed; overlapping retries dedupe in the ReplacingMergeTree tables. func (c *Controller) captureSpan(target anomaly.Target, table string, start, end time.Time, queryID string, depth int) (readCount, wroteCount int, err error) { r, w, e := c.orderQuerySlice(target, table, start, end, queryID) if e == nil || !isRetriableSpanErr(e) || end.Sub(start) <= orderMinChunk { @@ -423,12 +385,8 @@ func (c *Controller) captureSpan(target anomaly.Target, table string, start, end return r1 + r2, w1 + w2, e2 } -// orderQuerySlice runs ONE bounded (target, table, [start,end)) capture: query -// pixie, write the rows, return the read/wrote counts. It records NO reconcile row — -// the OrderQuery driver aggregates across chunks and records once. globalSem still -// bounds broker load per slice. Background ctx with per-op timeouts mirrors -// pushPixieRows: a control-ordered capture completes independently of any anomaly -// window's lifecycle. +// orderQuerySlice runs one (target, table, [start,end)) capture and writes the rows. +// It records no reconcile row — the OrderQuery driver aggregates and records once. func (c *Controller) orderQuerySlice(target anomaly.Target, table string, start, end time.Time, queryID string) (readCount, wroteCount int, err error) { now := c.clock.Now() q, qerr := pxl.QueryFor(table, target, start, end, now) @@ -443,17 +401,14 @@ func (c *Controller) orderQuerySlice(target anomaly.Target, table string, start, rows, qerr := c.querier.Query(qctx, q) cancel() if qerr != nil { - // Feed the saturation circuit-breaker: sustained transient failures mean the - // node-local PEM is overloaded, so captureSpan should stop subdividing. if isRetriableSpanErr(qerr) { - c.orderTimeoutStreak.Add(1) + c.orderTimeoutStreak.Add(1) // feed the saturation breaker } return 0, 0, qerr } - // A completed query means the PEM is serving — clear the breaker. - c.orderTimeoutStreak.Store(0) + c.orderTimeoutStreak.Store(0) // a completed query clears the breaker if len(rows) == 0 { - return 0, 0, nil // nothing to persist; the driver's reconcile row still records the read + return 0, 0, nil } wctx, wcancel := context.WithTimeout(context.Background(), 60*time.Second) werr := c.sink.WritePixieRows(wctx, table, rows) @@ -467,11 +422,8 @@ func (c *Controller) orderQuerySlice(target anomaly.Target, table string, start, return len(rows), len(rows), nil } -// isRetriableSpanErr reports whether a slice error is a transient overload/timeout -// worth retrying as a narrower span (vs. a structural error like a missing table, -// which no amount of subdivision fixes). Covers ctx deadlines and the gRPC status -// strings the pixie querier surfaces (DeadlineExceeded / ResourceExhausted / -// Unavailable), which do NOT satisfy errors.Is(context.DeadlineExceeded). +// isRetriableSpanErr reports whether an error is a transient timeout/overload (vs. a +// structural error like a missing table) — covers ctx deadlines and gRPC status strings. func isRetriableSpanErr(err error) bool { if err == nil { return false diff --git a/src/vizier/services/adaptive_export/internal/pxl/queryfor.go b/src/vizier/services/adaptive_export/internal/pxl/queryfor.go index cfe0bc17707..18c877c9db7 100644 --- a/src/vizier/services/adaptive_export/internal/pxl/queryfor.go +++ b/src/vizier/services/adaptive_export/internal/pxl/queryfor.go @@ -63,13 +63,7 @@ func QueryFor(table string, t anomaly.Target, sliceStart, sliceEnd, now time.Tim var b strings.Builder b.WriteString(pxSetMaxRows) b.WriteString("import px\n") - // Bound the PEM source scan on BOTH sides. Without an end_time the planner - // scans [sliceStart, now] for EVERY query, so an old ordered window (e.g. the - // 600s control lookback) re-materializes the whole span on the node-local PEM; - // under the OrderExportAll fan-out the heavy tables (dc_snoop) then blow the - // per-query deadline and drop out (the flaky-capture RCA). relEndBound caps the - // scan at ~sliceEnd; the exact upper bound is still trimmed by the df.time_ < - // sliceEnd nanos filter below, so nothing real is clipped. + // Bound the source scan on both sides; the df.time_ < sliceEnd filter trims the exact upper bound. dfArgs := "table='" + pixieSourceFor(table) + "', start_time='" + relStart + "'" if relEnd := relEndBound(now, sliceEnd); relEnd != "" { dfArgs += ", end_time='" + relEnd + "'" @@ -77,18 +71,9 @@ func QueryFor(table string, t anomaly.Target, sliceStart, sliceEnd, now time.Tim b.WriteString("df = px.DataFrame(" + dfArgs + ")\n") b.WriteString("df = df[df.time_ >= px.int64_to_time(" + strconv.FormatInt(sliceStart.UnixNano(), 10) + ")]\n") b.WriteString("df = df[df.time_ < px.int64_to_time(" + strconv.FormatInt(sliceEnd.UnixNano(), 10) + ")]\n") - // Native tables: px.upid_to_pod_name returns "/" (carnot: - // metadata_ops.h UPIDToPodNameUDF::Exec → absl::Substitute("$0/$1", ns, name)), - // not the bare pod name. Dark-vector tracepoint tables (pid-keyed) resolve pod - // via a process_stats pid-merge instead and yield a BARE pod name (dx#126). + // px.upid_to_pod_name yields "/"; dark-vector tables resolve pod via a pid-merge (bare name). if table == "stack_trace" { - // stack_trace is the CANONICAL native continuous profiler (stack_traces.beta, - // upid-keyed — NOT a pid tracepoint, so NOT a dark-vector pid-merge). Resolve - // pod/namespace/container/hostname exactly like the export preset - // (script/presets/stack_trace.pxl) and stamp event_time = time_ so the CH - // stack_trace row is complete. df.ctx['pod'] is the NAMESPACED "/" - // key (verified live), so the pod filter is namespaced — same as the native - // upid_to_pod_name path below. + // Native profiler (stack_traces.beta): resolve pod/ns/container from ctx, stamp event_time. b.WriteString("df.namespace = df.ctx['namespace']\n") b.WriteString("df.pod = df.ctx['pod']\n") b.WriteString("df.container = df.ctx['container']\n") @@ -105,28 +90,10 @@ func QueryFor(table string, t anomaly.Target, sliceStart, sliceEnd, now time.Tim } } } else if IsDarkVector(table) { - // Dark-vector tracepoints emit a RAW kernel pid. The malignant transient - // pids an incident actually produces — an attack's whoami/cat/getent - // children — are too short-lived to land in process_stats, so their - // pod/namespace resolves BLANK; a pod (or even namespace) filter drops - // exactly the evidence, which is why the dark tables came back empty. - // The AE is node-local (pem-direct → the node's own PEM), so the query is - // already scoped to the alert's node. - // - // ORDER MATTERS: drop the infra/self comms FIRST (env-driven, no recompile), - // THEN do the process_stats pid-merge. The node's dark stream is huge - // (Formatter/vector/runc/... thousands of rows per window); merging every - // one against process_stats is the query that timed out and silently - // dropped dc_snoop. Filtering comm first shrinks the merge to the handful - // of workload rows (bash/redis/whoami/cat), so the dark capture completes. + // Node-scoped (transient attack pids resolve blank ns, so no pod filter). Drop + // own-stack comms before the pid-merge to keep it cheap, then drop infra namespaces. b.WriteString(darkCommExclusion(table)) b.WriteString(PodEnrichPxL(table)) - // AFTER the pid-merge resolves df.namespace: drop infra/system namespaces - // (blank-namespace transient workload rows are KEPT — see darkCommExclusion - // note; the attack's short-lived children resolve blank). This mirrors the - // shipped cron preset (script/presets dc_snoop.pxl __DC_SNOOP_EXCLUSION__); - // the OrderExportAll path was missing it, so infra pods' dcache churn - // (ConfigReloader/iptables/CNI/host daemons) flooded every dc_snoop capture. b.WriteString(darkNamespaceExclusion()) } else { b.WriteString(PodEnrichPxL(table)) @@ -148,13 +115,7 @@ func QueryFor(table string, t anomaly.Target, sliceStart, sliceEnd, now time.Tim return b.String(), nil } -// relEndBound returns a RELATIVE end_time ("-s") that caps the PEM's source -// scan at ~sliceEnd, or "" when sliceEnd is at/after now (scan to the live edge). -// The gap is floored to whole seconds so the source window ends slightly LATER -// than sliceEnd and never clips real rows — the precise upper bound is enforced by -// the df.time_ < sliceEnd nanos post-filter. This is the load lever behind the -// chunked ordered path: each chunk materializes only its own span instead of -// [chunkStart, now]. +// relEndBound returns a relative end_time ("-s"), or "" when sliceEnd is at/after now. func relEndBound(now, sliceEnd time.Time) string { gap := now.Sub(sliceEnd) if gap < time.Second { @@ -163,11 +124,7 @@ func relEndBound(now, sliceEnd time.Time) string { return "-" + strconv.FormatInt(int64(gap/time.Second), 10) + "s" } -// pixieSourceFor returns the Pixie table a builtin is sourced FROM when it -// differs from the ClickHouse table it is written TO. stack_trace is written to -// CH as 'stack_trace' but sourced from the CANONICAL native continuous profiler -// 'stack_traces.beta' — the always-on Pixie profiler, NOT an AE-invented table. -// (Dotted-name DataFrames compile fine in a direct query; verified live.) +// pixieSourceFor maps a CH table to the pixie table it's read from (stack_trace ← stack_traces.beta). func pixieSourceFor(table string) string { if table == "stack_trace" { return "stack_traces.beta" @@ -175,18 +132,15 @@ func pixieSourceFor(table string) string { return table } -// darkVectorHasComm lists the dark-vector tables that carry a `comm` column, so -// the infra-comm exclusion only emits for those (stack_trace is upid-only). +// Dark-vector tables carrying a comm column (so the comm exclusion applies). var darkVectorHasComm = map[string]bool{ "dc_snoop": true, "creds_change": true, "dx_vfs_events": true, "dx_unlink": true, "dx_dlookup": true, "dx_mprotect": true, "dx_bpf": true, "dx_ptrace": true, } -// darkExcludeCommsDefault is the node's own infra/self comms dropped from the -// node-scoped dark capture so the workload's activity stands out. Overridable at -// runtime via DC_SNOOP_EXCLUDE_COMMS (csv) — a process can be added without a -// recompile. Kept in sync with script.presets defaultExcludeComms. +// Own-stack + node/system comms dropped from the node-scoped dark capture; workload +// comms (redis-*, etc.) are never listed. Override via DC_SNOOP_EXCLUDE_COMMS (csv). var darkExcludeCommsDefault = []string{ "pem", "kelvin", "containerd", "containerd-shim", "runc", "node-agent", "runc:[2:INIT]", "runc:[1:CHILD]", @@ -198,27 +152,26 @@ var darkExcludeCommsDefault = []string{ "ConfigReloader", "clickhouse-oper", "Formatter", "(setup.sh)", "cmd", "vector-worker", "metrics-server", "local-path-prov", "portmap", "(udev-worker)", "systemd-resolve", "systemd-timesyn", - // host/CNI/node daemons that flood dc_snoop with dcache churn but carry no - // workload forensic value (observed leaking on a real k3s node, aeprod54). "systemd-udevd", "systemd-sysctl", "host-local", "bridge", "flannel", "loopback", "bandwidth", "dbus-daemon", "mount", "umount", "tailscaled", "grpc_health_pro", "kubevuln", "opm", "(spawn)", "kube-proxy", + "pause", "systemd-logind", } -// darkExcludeNamespacesDefault drops infra/system namespaces from the node-scoped -// dark capture (blank-namespace transient workload rows are KEPT — the attack's -// short-lived children resolve blank, so a namespace filter must never drop them). -// Overridable via DC_SNOOP_EXCLUDE_NAMESPACES (csv). Kept in sync with -// script/presets.go defaultExcludeNamespaces — the shipped cron path already -// filtered these; the dx-steered OrderExportAll path did not, so infra pods' -// process churn flooded every capture. +// Kernel-thread families whose names carry a variable suffix (kworker/u8:3) that +// exact match misses; dropped via px.contains. +var darkExcludeCommSubstrings = []string{ + "kworker", "ksoftirqd", "migration", "rcu_", "kthreadd", "kdevtmpfs", + "kcompactd", "khugepaged", "kswapd", "watchdog", "cpuhp", "ksmd", "irq/", +} + +// Infra namespaces dropped from the node-scoped dark capture. Blank-namespace rows +// (transient attack children) survive. Override via DC_SNOOP_EXCLUDE_NAMESPACES. var darkExcludeNamespacesDefault = []string{ "pl", "honey", "px-operator", "olm", "clickhouse", "socdemo", "socdemo-ch", "kube-system", "kube-public", "kube-node-lease", "local-path-storage", } -// darkCommExclusion builds the infra-comm drop filter for a dark-vector table -// that has a comm column. Returns "" for comm-less tables (stack_trace). func darkCommExclusion(table string) string { if !darkVectorHasComm[table] { return "" @@ -236,13 +189,12 @@ func darkCommExclusion(table string) string { for _, c := range comms { b.WriteString("df = df[df.comm != '" + escapePxL(c) + "']\n") } + for _, s := range darkExcludeCommSubstrings { + b.WriteString("df = df[px.logicalNot(px.contains(df.comm, '" + escapePxL(s) + "'))]\n") + } return b.String() } -// darkNamespaceExclusion builds the infra-namespace drop filter for the node-scoped -// dark capture. Emitted AFTER PodEnrichPxL resolves df.namespace. Blank-namespace -// rows survive (each `!=` predicate is true for ”), so transient attack children -// are never dropped. Overridable via DC_SNOOP_EXCLUDE_NAMESPACES (csv). func darkNamespaceExclusion() string { nss := darkExcludeNamespacesDefault if v := strings.TrimSpace(os.Getenv("DC_SNOOP_EXCLUDE_NAMESPACES")); v != "" { diff --git a/src/vizier/services/adaptive_export/internal/pxl/queryfor_test.go b/src/vizier/services/adaptive_export/internal/pxl/queryfor_test.go index fa2e121ebf3..f42c779bd5b 100644 --- a/src/vizier/services/adaptive_export/internal/pxl/queryfor_test.go +++ b/src/vizier/services/adaptive_export/internal/pxl/queryfor_test.go @@ -418,3 +418,24 @@ func TestQueryFor_DarkNamespaceExclusion_EnvOverride(t *testing.T) { t.Errorf("env override must REPLACE the default (no 'pl'); got:\n%s", q) } } + +// dc_snoop drops kernel-thread families (variable suffix) via px.logicalNot(px.contains), +// keeps workload comms (redis-server), and doesn't pin the alert pod's namespace. +func TestQueryFor_DarkCommSubstringExclusion(t *testing.T) { + q, err := QueryFor("dc_snoop", target, fixedStart, fixedEnd, fixedNow) + if err != nil { + t.Fatalf("QueryFor: %v", err) + } + for _, sub := range []string{"kworker", "ksoftirqd", "rcu_"} { + want := "df = df[px.logicalNot(px.contains(df.comm, '" + sub + "'))]" + if !strings.Contains(q, want) { + t.Errorf("want kernel-thread drop %q; got:\n%s", want, q) + } + } + if strings.Contains(q, "df.comm != 'redis-server'") || strings.Contains(q, "df.comm, 'redis") { + t.Errorf("workload comm redis-* must NOT be excluded; got:\n%s", q) + } + if !strings.Contains(q, "df = df[df.comm != 'pause']") { + t.Errorf("want exact drop of 'pause'; got:\n%s", q) + } +} From f33b4eed1a29e7b2484ef35800a88c6560a1ca6d Mon Sep 17 00:00:00 2001 From: entlein Date: Thu, 6 Aug 2026 14:22:40 +0200 Subject: [PATCH 07/66] k8s/vizier/dx: skaffold module for dx-daemon deploy Deploys the dx-daemon DaemonSet + Service into honey and mirrors the pl->honey secrets (jwt-signing-key, cluster-id, cloud-addr, api-key, clickhouse http-url) via a before-hook, replacing the hand-applied manifest used in the e2e. Deploy with: skaffold deploy -f k8s/vizier/dx/skaffold.yaml CH http-url defaults to the soc clickhouse Service; override with DX_CH_HTTP_URL. --- k8s/vizier/dx/dx-daemon.yaml | 58 ++++++++++++++++++++++++++++++++++++ k8s/vizier/dx/skaffold.yaml | 36 ++++++++++++++++++++++ 2 files changed, 94 insertions(+) create mode 100644 k8s/vizier/dx/dx-daemon.yaml create mode 100644 k8s/vizier/dx/skaffold.yaml diff --git a/k8s/vizier/dx/dx-daemon.yaml b/k8s/vizier/dx/dx-daemon.yaml new file mode 100644 index 00000000000..98797e7937c --- /dev/null +++ b/k8s/vizier/dx/dx-daemon.yaml @@ -0,0 +1,58 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: { name: dx-daemon, namespace: honey } +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: dx-daemon + namespace: honey + labels: { app: dx-daemon } +spec: + selector: { matchLabels: { app: dx-daemon } } + template: + metadata: + labels: { app: dx-daemon } + spec: + serviceAccountName: dx-daemon + tolerations: [{ operator: Exists }] + terminationGracePeriodSeconds: 35 + containers: + - name: dx-daemon + image: docker.io/entlein/dx-daemon:0.3.0-public3 + ports: + - { name: findings, containerPort: 9099, hostPort: 9099 } + env: + - { name: NODE_NAME, valueFrom: { fieldRef: { fieldPath: spec.nodeName } } } + - { name: HOST_IP, valueFrom: { fieldRef: { fieldPath: status.hostIP } } } + - { name: DX_RECEIVER_TLS, value: "1" } + - { name: AE_CONTROL_ADDR, value: "http://adaptive-export-control.pl.svc.cluster.local:9100" } + - { name: PX_API_KEY, valueFrom: { secretKeyRef: { name: dx-pixie-auth, key: api-key, optional: true } } } + - { name: PX_CLUSTER_ID, valueFrom: { secretKeyRef: { name: dx-pixie-auth, key: cluster-id, optional: true } } } + - { name: PX_CLOUD_ADDR, valueFrom: { secretKeyRef: { name: dx-pixie-auth, key: cloud-addr, optional: true } } } + - { name: DX_BENCH, value: "pemdirect" } + - { name: PL_JWT_SIGNING_KEY, valueFrom: { secretKeyRef: { name: dx-vizier-direct, key: jwt-signing-key, optional: true } } } + - { name: DX_VIZIER_DIRECT_ADDR, value: "vizier-query-broker-svc.pl.svc.cluster.local:50300" } + - { name: PX_DISABLE_TLS, value: "1" } + - { name: DX_CLUSTER_MALIGNANT_HTTP, valueFrom: { secretKeyRef: { name: dx-metastasis-ch, key: http-url, optional: true } } } + - { name: DX_PX_TIMEOUT_S, value: "90" } + - { name: DX_TELEMETRY_CACHE, value: "1" } + - { name: DX_WORKERS, value: "4" } + readinessProbe: + httpGet: { path: /healthz, port: 9099, scheme: HTTPS } + initialDelaySeconds: 3 + periodSeconds: 10 + resources: + requests: { cpu: 50m, memory: 256Mi } + limits: { cpu: "2", memory: 1Gi } +--- +apiVersion: v1 +kind: Service +metadata: + name: dx-daemon + namespace: honey +spec: + selector: { app: dx-daemon } + internalTrafficPolicy: Local + ports: + - { name: findings, port: 9099, targetPort: 9099 } diff --git a/k8s/vizier/dx/skaffold.yaml b/k8s/vizier/dx/skaffold.yaml new file mode 100644 index 00000000000..5144302f561 --- /dev/null +++ b/k8s/vizier/dx/skaffold.yaml @@ -0,0 +1,36 @@ +apiVersion: skaffold/v4beta11 +kind: Config +metadata: + name: dx-daemon +manifests: + rawYaml: + - dx-daemon.yaml +deploy: + kubectl: + defaultNamespace: "" + hooks: + before: + - host: + command: + - bash + - -c + - | + set -e + kubectl create namespace honey --dry-run=client -o yaml | kubectl apply -f - + JWT=$(kubectl -n pl get secret pl-cluster-secrets -o jsonpath='{.data.jwt-signing-key}' | base64 -d) + CID=$(kubectl -n pl get secret pl-cluster-secrets -o jsonpath='{.data.cluster-id}' | base64 -d) + CA=$(kubectl -n pl get cm pl-cloud-config -o jsonpath='{.data.PL_CLOUD_ADDR}') + API=$(kubectl -n pl get secret pl-adaptive-export-secrets -o jsonpath='{.data.pixie-api-key}' 2>/dev/null | base64 -d) + CH_URL="${DX_CH_HTTP_URL:-http://ingest_writer:changeme-ingest@clickhouse-forensic-soc-db.clickhouse.svc.cluster.local:8123/?database=forensic_db}" + kubectl -n honey create secret generic dx-vizier-direct \ + --from-literal=jwt-signing-key="$JWT" \ + --dry-run=client -o yaml | kubectl apply -f - + kubectl -n honey create secret generic dx-pixie-auth \ + --from-literal=api-key="$API" \ + --from-literal=cluster-id="$CID" \ + --from-literal=cloud-addr="$CA" \ + --dry-run=client -o yaml | kubectl apply -f - + kubectl -n honey create secret generic dx-metastasis-ch \ + --from-literal=http-url="$CH_URL" \ + --dry-run=client -o yaml | kubectl apply -f - + os: [linux, darwin] From c4282c140b3ec211a1080d7e6ef219f5b41bf99f Mon Sep 17 00:00:00 2001 From: entlein Date: Thu, 6 Aug 2026 17:17:12 +0200 Subject: [PATCH 08/66] k8s/vizier/adaptive_export: skaffold overlay for AE deploy Replaces the imperative seed-secret + patch-cloud-addr + sed-image + kubectl-apply sequence with a single skaffold module: skaffold deploy -f k8s/vizier/adaptive_export/skaffold.yaml - kustomize overlay reuses bootstrap/adaptive_export_{role,deployment} and pins the image via images: (ghcr aeprod tag) instead of sed. - before-hook patches PL_CLOUD_ADDR :443 and seeds pl-adaptive-export-secrets ONLY when PIXIE_API_KEY/PX_API_KEY is set, never clobbering an existing secret with an empty key. - LoadRestrictionsNone so the overlay can reuse the bootstrap manifests in place (no duplication/drift). Pairs with the dx-daemon skaffold (k8s/vizier/dx). Bump the AE image by editing newTag in kustomization.yaml. --- k8s/vizier/adaptive_export/kustomization.yaml | 10 +++++ k8s/vizier/adaptive_export/skaffold.yaml | 41 +++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 k8s/vizier/adaptive_export/kustomization.yaml create mode 100644 k8s/vizier/adaptive_export/skaffold.yaml diff --git a/k8s/vizier/adaptive_export/kustomization.yaml b/k8s/vizier/adaptive_export/kustomization.yaml new file mode 100644 index 00000000000..94bcd745b64 --- /dev/null +++ b/k8s/vizier/adaptive_export/kustomization.yaml @@ -0,0 +1,10 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: pl +resources: + - ../bootstrap/adaptive_export_role.yaml + - ../bootstrap/adaptive_export_deployment.yaml +images: + - name: vizier-adaptive_export_image + newName: ghcr.io/k8sstormcenter/vizier-adaptive_export_image + newTag: 0.14.19-aeprod57 diff --git a/k8s/vizier/adaptive_export/skaffold.yaml b/k8s/vizier/adaptive_export/skaffold.yaml new file mode 100644 index 00000000000..d457dd0fa2e --- /dev/null +++ b/k8s/vizier/adaptive_export/skaffold.yaml @@ -0,0 +1,41 @@ +apiVersion: skaffold/v4beta11 +kind: Config +metadata: + name: adaptive-export +manifests: + kustomize: + paths: + - . + buildArgs: + - --load-restrictor=LoadRestrictionsNone +deploy: + kubectl: + defaultNamespace: pl + hooks: + before: + - host: + command: + - bash + - -c + - | + set -e + # PL_CLOUD_ADDR must carry an explicit :443 or the AE cloud client + # crashloops (see AE per-PG notes). + CA=$(kubectl -n pl get cm pl-cloud-config -o jsonpath='{.data.PL_CLOUD_ADDR}' 2>/dev/null || true) + case "$CA" in ""|*:*) ;; *) kubectl -n pl patch cm pl-cloud-config --type merge -p "{\"data\":{\"PL_CLOUD_ADDR\":\"$CA:443\"}}";; esac + # Seed pl-adaptive-export-secrets ONLY when a key is supplied, and + # never clobber an existing good secret with an empty one. + API="${PIXIE_API_KEY:-${PX_API_KEY:-}}" + CH_DSN="${AE_CH_DSN:-ingest_writer:changeme-ingest@clickhouse-forensic-soc-db.clickhouse.svc.cluster.local:9000/forensic_db}" + if [ -n "$API" ]; then + kubectl -n pl create secret generic pl-adaptive-export-secrets \ + --from-literal=pixie-api-key="$API" \ + --from-literal=clickhouse-dsn="$CH_DSN" \ + --dry-run=client -o yaml | kubectl apply -f - + elif ! kubectl -n pl get secret pl-adaptive-export-secrets >/dev/null 2>&1; then + echo "ERROR: set PIXIE_API_KEY (or source keys.env) to seed pl-adaptive-export-secrets" >&2 + exit 1 + else + echo "pl-adaptive-export-secrets exists; PIXIE_API_KEY unset -> leaving it untouched" + fi + os: [linux, darwin] From 2e05ed29aeb9a5a13cfd7edc46fbce32dca5dc57 Mon Sep 17 00:00:00 2001 From: entlein Date: Thu, 6 Aug 2026 18:40:47 +0200 Subject: [PATCH 09/66] skaffold: move AE/dx deploy configs to skaffold/ with root-relative paths The AE/dx skaffold configs lived inside their overlay dirs with kustomize paths: [.], which skaffold resolves against the shell CWD (repo root), not the config-file dir -> 'unable to find kustomization.yaml in /.../pixie'. Match the repo convention instead (skaffold/skaffold_vizier.yaml et al.): skaffold configs live in skaffold/ and reference overlays by repo-root- relative kustomize paths. Overlays stay in k8s/vizier/{adaptive_export,dx}. skaffold deploy -f skaffold/skaffold_adaptive_export.yaml skaffold deploy -f skaffold/skaffold_dx.yaml # run from repo root - dx overlay gains a kustomization.yaml (was rawYaml). - both validated with 'skaffold render' from repo root (image overrides + RBAC/DaemonSet/Service resolve). --- k8s/vizier/dx/kustomization.yaml | 5 +++++ .../skaffold_adaptive_export.yaml | 13 ++++++++----- .../dx/skaffold.yaml => skaffold/skaffold_dx.yaml | 12 +++++++++--- 3 files changed, 22 insertions(+), 8 deletions(-) create mode 100644 k8s/vizier/dx/kustomization.yaml rename k8s/vizier/adaptive_export/skaffold.yaml => skaffold/skaffold_adaptive_export.yaml (79%) rename k8s/vizier/dx/skaffold.yaml => skaffold/skaffold_dx.yaml (82%) diff --git a/k8s/vizier/dx/kustomization.yaml b/k8s/vizier/dx/kustomization.yaml new file mode 100644 index 00000000000..7e7edebde60 --- /dev/null +++ b/k8s/vizier/dx/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: honey +resources: + - dx-daemon.yaml diff --git a/k8s/vizier/adaptive_export/skaffold.yaml b/skaffold/skaffold_adaptive_export.yaml similarity index 79% rename from k8s/vizier/adaptive_export/skaffold.yaml rename to skaffold/skaffold_adaptive_export.yaml index d457dd0fa2e..b04d8550990 100644 --- a/k8s/vizier/adaptive_export/skaffold.yaml +++ b/skaffold/skaffold_adaptive_export.yaml @@ -1,3 +1,8 @@ +--- +# Deploy-only Skaffold for the adaptive_export DaemonSet using a prebuilt image +# (lab / review), overlaying an already-running vizier. Run from the repo root: +# skaffold deploy -f skaffold/skaffold_adaptive_export.yaml +# Bump the image via newTag in k8s/vizier/adaptive_export/kustomization.yaml. apiVersion: skaffold/v4beta11 kind: Config metadata: @@ -5,7 +10,7 @@ metadata: manifests: kustomize: paths: - - . + - k8s/vizier/adaptive_export buildArgs: - --load-restrictor=LoadRestrictionsNone deploy: @@ -19,12 +24,10 @@ deploy: - -c - | set -e - # PL_CLOUD_ADDR must carry an explicit :443 or the AE cloud client - # crashloops (see AE per-PG notes). + # PL_CLOUD_ADDR must carry an explicit :443 or the AE cloud client crashloops. CA=$(kubectl -n pl get cm pl-cloud-config -o jsonpath='{.data.PL_CLOUD_ADDR}' 2>/dev/null || true) case "$CA" in ""|*:*) ;; *) kubectl -n pl patch cm pl-cloud-config --type merge -p "{\"data\":{\"PL_CLOUD_ADDR\":\"$CA:443\"}}";; esac - # Seed pl-adaptive-export-secrets ONLY when a key is supplied, and - # never clobber an existing good secret with an empty one. + # Seed pl-adaptive-export-secrets ONLY when a key is supplied; never clobber a good secret with an empty one. API="${PIXIE_API_KEY:-${PX_API_KEY:-}}" CH_DSN="${AE_CH_DSN:-ingest_writer:changeme-ingest@clickhouse-forensic-soc-db.clickhouse.svc.cluster.local:9000/forensic_db}" if [ -n "$API" ]; then diff --git a/k8s/vizier/dx/skaffold.yaml b/skaffold/skaffold_dx.yaml similarity index 82% rename from k8s/vizier/dx/skaffold.yaml rename to skaffold/skaffold_dx.yaml index 5144302f561..ea3ee0c7e14 100644 --- a/k8s/vizier/dx/skaffold.yaml +++ b/skaffold/skaffold_dx.yaml @@ -1,13 +1,19 @@ +--- +# Deploy-only Skaffold for the dx-daemon DaemonSet (prebuilt image), overlaying an +# already-running vizier + soc stack. Run from the repo root, AFTER adaptive_export +# (the hook mirrors pl-adaptive-export-secrets into honey): +# skaffold deploy -f skaffold/skaffold_dx.yaml apiVersion: skaffold/v4beta11 kind: Config metadata: name: dx-daemon manifests: - rawYaml: - - dx-daemon.yaml + kustomize: + paths: + - k8s/vizier/dx deploy: kubectl: - defaultNamespace: "" + defaultNamespace: honey hooks: before: - host: From 0ac57f53a90f7fefeb71483c49b8a3898d94ba11 Mon Sep 17 00:00:00 2001 From: entlein Date: Fri, 14 Aug 2026 17:14:27 +0200 Subject: [PATCH 10/66] lab: bump dx->rc6 + AE->aeprod63, add evidence_graph env dx image 0.3.0-public3 -> 0.4.0-ssotforest-rc6 (forest scope + evidence_graph + isTableAbsent; broker no longer blinds the verdict). AE aeprod57 -> aeprod63 (upid + OOM firehose-collapse + px.any dark-export fixes). Add DX_FOREST_SCOPE, DX_PRECORRELATE_GRAPH, DX_EVIDENCE_GRAPH_CH so dx populates forensic_db.dx_evidence_graph. Verified live on a pemdq1 rig: dc_snoop + evidence_graph populate; DX_BENCH=pemdirect (already set) keeps dx off the shared broker so AE's export doesn't DeadlineExceed. --- k8s/vizier/adaptive_export/kustomization.yaml | 2 +- k8s/vizier/dx/dx-daemon.yaml | 7 ++++++- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/k8s/vizier/adaptive_export/kustomization.yaml b/k8s/vizier/adaptive_export/kustomization.yaml index 94bcd745b64..186a589c5b0 100644 --- a/k8s/vizier/adaptive_export/kustomization.yaml +++ b/k8s/vizier/adaptive_export/kustomization.yaml @@ -7,4 +7,4 @@ resources: images: - name: vizier-adaptive_export_image newName: ghcr.io/k8sstormcenter/vizier-adaptive_export_image - newTag: 0.14.19-aeprod57 + newTag: 0.14.19-aeprod63 diff --git a/k8s/vizier/dx/dx-daemon.yaml b/k8s/vizier/dx/dx-daemon.yaml index 98797e7937c..5889c098114 100644 --- a/k8s/vizier/dx/dx-daemon.yaml +++ b/k8s/vizier/dx/dx-daemon.yaml @@ -19,7 +19,7 @@ spec: terminationGracePeriodSeconds: 35 containers: - name: dx-daemon - image: docker.io/entlein/dx-daemon:0.3.0-public3 + image: docker.io/entlein/dx-daemon:0.4.0-ssotforest-rc6 ports: - { name: findings, containerPort: 9099, hostPort: 9099 } env: @@ -38,6 +38,11 @@ spec: - { name: DX_PX_TIMEOUT_S, value: "90" } - { name: DX_TELEMETRY_CACHE, value: "1" } - { name: DX_WORKERS, value: "4" } + # evidence-graph: forest-scope the evidence, write the per-anomaly edge set, + # sink it straight to forensic_db.dx_evidence_graph (soc ingest_writer). + - { name: DX_FOREST_SCOPE, value: "1" } + - { name: DX_PRECORRELATE_GRAPH, value: "1" } + - { name: DX_EVIDENCE_GRAPH_CH, value: "http://ingest_writer:changeme-ingest@clickhouse-forensic-soc-db.clickhouse.svc.cluster.local:8123/forensic_db" } readinessProbe: httpGet: { path: /healthz, port: 9099, scheme: HTTPS } initialDelaySeconds: 3 From 3a7672efcc222a8958fd1b28c9749d37646bb215 Mon Sep 17 00:00:00 2001 From: entlein Date: Fri, 14 Aug 2026 22:12:17 +0200 Subject: [PATCH 11/66] =?UTF-8?q?lab(dx):=20raise=20memory=20limit=201Gi->?= =?UTF-8?q?2Gi=20=E2=80=94=20fix=20OOM=20that=20empties=20the=20evidence?= =?UTF-8?q?=5Fgraph?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause of 'attack fires but dx_evidence_graph stays empty': with DX_PRECORRELATE_GRAPH=1 the workup pulls the per-anomaly full-evidence set into memory and at the 1Gi limit dx is OOM-killed (exit 137) mid-workup, BEFORE writing the graph — then crash-loops, so no edges ever land. Reproduced on a pemdq1 rig: dx received the referral (comm=ls/sh rule=R0001) then died OOMKilled x4. 2Gi clears it (verified: graph 23->34, 32 malignant). Request 256Mi->512Mi. Operational note (not a manifest change): the vector->dx sink can wedge when the dx pod bounces (findings stop arriving, no referral) — bounce the node-01 vector pod after any dx redeploy. Also seen: transient node-01 PEM restart -> pemdirect 'connection refused' -> BLIND verdicts (edges still write via generic-malignant). --- k8s/vizier/dx/dx-daemon.yaml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/k8s/vizier/dx/dx-daemon.yaml b/k8s/vizier/dx/dx-daemon.yaml index 5889c098114..1d9779572b6 100644 --- a/k8s/vizier/dx/dx-daemon.yaml +++ b/k8s/vizier/dx/dx-daemon.yaml @@ -48,8 +48,13 @@ spec: initialDelaySeconds: 3 periodSeconds: 10 resources: - requests: { cpu: 50m, memory: 256Mi } - limits: { cpu: "2", memory: 1Gi } + # memory: the precorrelate/full-evidence workup (DX_PRECORRELATE_GRAPH) pulls + # the per-anomaly evidence set into memory; at 1Gi dx is OOM-killed mid-workup + # (exit 137) BEFORE it writes forensic_db.dx_evidence_graph → crash-loop, empty + # graph. 2Gi clears it on an 8GiB node (verified: dc_snoop ~4k rows). Bump to + # 3Gi if a heavier attack/firehose still OOMs. + requests: { cpu: 50m, memory: 512Mi } + limits: { cpu: "2", memory: 2Gi } --- apiVersion: v1 kind: Service From 2160f6d2bf1d5fda26a71c8a75bc321acf7a4e70 Mon Sep 17 00:00:00 2001 From: entlein Date: Sat, 15 Aug 2026 00:23:41 +0200 Subject: [PATCH 12/66] =?UTF-8?q?lab(dx):=20DX=5FWORKERS=3D4->1=20?= =?UTF-8?q?=E2=80=94=20fix=20SIGSEGV=20under=20the=20kill-chain=20referral?= =?UTF-8?q?=20flood?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fresh-PG validation of the 2Gi memory fix surfaced a SECOND bug: the bobctl attacks/redis-oss.yaml kill-chain fires ~31 distinct comm/rule anomalies on redis-master-0, and dx with 4 concurrent workers SIGSEGVs (exit 139, no Go panic = hard crash in the concurrent workup/pemdirect path) — crash-loops, graph stays empty. Serializing workups (DX_WORKERS=1) eliminates it: restarts=0, graph 0->32 (30 malignant) on a fresh pemdq1 rig with the full kill-chain. The 2Gi fix (previous commit) handles OOM; this handles the concurrency crash. Root fix for the race (so >1 worker is safe) tracked separately. --- k8s/vizier/dx/dx-daemon.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/vizier/dx/dx-daemon.yaml b/k8s/vizier/dx/dx-daemon.yaml index 1d9779572b6..023f4f0c0d2 100644 --- a/k8s/vizier/dx/dx-daemon.yaml +++ b/k8s/vizier/dx/dx-daemon.yaml @@ -37,7 +37,7 @@ spec: - { name: DX_CLUSTER_MALIGNANT_HTTP, valueFrom: { secretKeyRef: { name: dx-metastasis-ch, key: http-url, optional: true } } } - { name: DX_PX_TIMEOUT_S, value: "90" } - { name: DX_TELEMETRY_CACHE, value: "1" } - - { name: DX_WORKERS, value: "4" } + - { name: DX_WORKERS, value: "1" } # evidence-graph: forest-scope the evidence, write the per-anomaly edge set, # sink it straight to forensic_db.dx_evidence_graph (soc ingest_writer). - { name: DX_FOREST_SCOPE, value: "1" } From 0b49e112b78a4519e354ba4f190339d8f6af7fbb Mon Sep 17 00:00:00 2001 From: entlein Date: Sat, 15 Aug 2026 10:22:47 +0200 Subject: [PATCH 13/66] lab(dx): rc8 (pxapi nil-Timing SIGSEGV fix) + revert DX_WORKERS band-aid to 4 The DX_WORKERS=1 workaround is no longer needed: the crash was a nil qes.Timing deref in pxapi handleStats (fixed in rc8 via pixie@6422c0508782 + dx nil-rs guard + TriagePull recover), not a dx concurrency defect (race-detector floods clean). Restore the default 4 workers and the fixed image. Memory stays 2Gi (real precorrelate need). --- k8s/vizier/dx/dx-daemon.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/k8s/vizier/dx/dx-daemon.yaml b/k8s/vizier/dx/dx-daemon.yaml index 023f4f0c0d2..2c7fe92f7d2 100644 --- a/k8s/vizier/dx/dx-daemon.yaml +++ b/k8s/vizier/dx/dx-daemon.yaml @@ -19,7 +19,7 @@ spec: terminationGracePeriodSeconds: 35 containers: - name: dx-daemon - image: docker.io/entlein/dx-daemon:0.4.0-ssotforest-rc6 + image: docker.io/entlein/dx-daemon:0.4.0-ssotforest-rc8 ports: - { name: findings, containerPort: 9099, hostPort: 9099 } env: @@ -37,7 +37,7 @@ spec: - { name: DX_CLUSTER_MALIGNANT_HTTP, valueFrom: { secretKeyRef: { name: dx-metastasis-ch, key: http-url, optional: true } } } - { name: DX_PX_TIMEOUT_S, value: "90" } - { name: DX_TELEMETRY_CACHE, value: "1" } - - { name: DX_WORKERS, value: "1" } + - { name: DX_WORKERS, value: "4" } # evidence-graph: forest-scope the evidence, write the per-anomaly edge set, # sink it straight to forensic_db.dx_evidence_graph (soc ingest_writer). - { name: DX_FOREST_SCOPE, value: "1" } From c9b3742c05069c6a40d58bafb6e2b214800336e4 Mon Sep 17 00:00:00 2001 From: entlein Date: Sat, 15 Aug 2026 18:47:19 +0200 Subject: [PATCH 14/66] dx lab manifest: non-garble optdbg2 image + 3Gi + DX_FOREST_PUSHDOWN MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Sync the entlein/dx#138 fixes into the skaffold-deployed lab manifest: - image rc8 -> optdbg2 (non-garble; rc8 predates the manifest+pushdown code and the garble release crashes exit 139 under load — fault 2, unresolved) - memory 2Gi -> 3Gi (fault 1: 1.3GB/round peak, OOM below) - DX_FOREST_PUSHDOWN=1 + depth 4 (fault 3: PxL lineage pushdown frees the PEM so AE exports dc_snoop under load; validated restarts=0 over 6+ rounds, dc_snoop 0->1777). --- k8s/vizier/dx/dx-daemon.yaml | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/k8s/vizier/dx/dx-daemon.yaml b/k8s/vizier/dx/dx-daemon.yaml index 2c7fe92f7d2..195b55d7220 100644 --- a/k8s/vizier/dx/dx-daemon.yaml +++ b/k8s/vizier/dx/dx-daemon.yaml @@ -19,7 +19,12 @@ spec: terminationGracePeriodSeconds: 35 containers: - name: dx-daemon - image: docker.io/entlein/dx-daemon:0.4.0-ssotforest-rc8 + # NON-GARBLE build: the garble -literals -tiny release images (rc8..rc11) SIGSEGV + # (exit 139) under the kill-chain (entlein/dx#138 fault 2, unresolved); the plain + # go build is reliable (restarts=0 over 6+ rounds). optdbg2 also carries the + # evidence-manifest + DX_FOREST_PUSHDOWN code (rc8 predates both). Swap back to a + # garble release once the obfuscation miscompile is fixed. + image: docker.io/entlein/dx-daemon:0.4.0-ssotforest-optdbg2 ports: - { name: findings, containerPort: 9099, hostPort: 9099 } env: @@ -41,6 +46,11 @@ spec: # evidence-graph: forest-scope the evidence, write the per-anomaly edge set, # sink it straight to forensic_db.dx_evidence_graph (soc ingest_writer). - { name: DX_FOREST_SCOPE, value: "1" } + # FOREST_PUSHDOWN (entlein/dx#138 fault 3): push the dc_snoop ppid-lineage filter + # INTO the PxL so dx pulls only the alert pod's subtree, not the whole node — + # frees the node-local PEM so AE can export dc_snoop under load (validated: 0→1777). + - { name: DX_FOREST_PUSHDOWN, value: "1" } + - { name: DX_FOREST_PUSHDOWN_DEPTH, value: "4" } - { name: DX_PRECORRELATE_GRAPH, value: "1" } - { name: DX_EVIDENCE_GRAPH_CH, value: "http://ingest_writer:changeme-ingest@clickhouse-forensic-soc-db.clickhouse.svc.cluster.local:8123/forensic_db" } readinessProbe: @@ -48,13 +58,13 @@ spec: initialDelaySeconds: 3 periodSeconds: 10 resources: - # memory: the precorrelate/full-evidence workup (DX_PRECORRELATE_GRAPH) pulls - # the per-anomaly evidence set into memory; at 1Gi dx is OOM-killed mid-workup - # (exit 137) BEFORE it writes forensic_db.dx_evidence_graph → crash-loop, empty - # graph. 2Gi clears it on an 8GiB node (verified: dc_snoop ~4k rows). Bump to - # 3Gi if a heavier attack/firehose still OOMs. - requests: { cpu: 50m, memory: 512Mi } - limits: { cpu: "2", memory: 2Gi } + # memory: the precorrelate/full-evidence workup (DX_PRECORRELATE_GRAPH) + pemdirect + # gRPC result streams pull the per-anomaly evidence set into memory; at 1Gi dx is + # OOM-killed mid-workup (exit 137) BEFORE it writes the graph → crash-loop, empty + # graph. Measured peak ~1.3GB/round under the redis kill-chain (entlein/dx#138 + # fault 1); 3Gi clears it reliably on an 8GiB node (validated: restarts=0 over 6+ rounds). + requests: { cpu: 50m, memory: 1Gi } + limits: { cpu: "2", memory: 3Gi } --- apiVersion: v1 kind: Service From cb1b68d05397b553ac6dbf09561725cbd4d15ac0 Mon Sep 17 00:00:00 2001 From: entlein Date: Sat, 15 Aug 2026 23:18:55 +0200 Subject: [PATCH 15/66] dx lab manifest: optdbg2 -> rc13 (obfuscated release, #138 fault 2 fixed) rc13 = garble -literals (dropped -tiny, the SIGSEGV cause). Obfuscated + survives the kill-chain (restarts=0). Replaces the non-garble optdbg2 debug tag. --- k8s/vizier/dx/dx-daemon.yaml | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/k8s/vizier/dx/dx-daemon.yaml b/k8s/vizier/dx/dx-daemon.yaml index 195b55d7220..1cb34480455 100644 --- a/k8s/vizier/dx/dx-daemon.yaml +++ b/k8s/vizier/dx/dx-daemon.yaml @@ -19,12 +19,11 @@ spec: terminationGracePeriodSeconds: 35 containers: - name: dx-daemon - # NON-GARBLE build: the garble -literals -tiny release images (rc8..rc11) SIGSEGV - # (exit 139) under the kill-chain (entlein/dx#138 fault 2, unresolved); the plain - # go build is reliable (restarts=0 over 6+ rounds). optdbg2 also carries the - # evidence-manifest + DX_FOREST_PUSHDOWN code (rc8 predates both). Swap back to a - # garble release once the obfuscation miscompile is fixed. - image: docker.io/entlein/dx-daemon:0.4.0-ssotforest-optdbg2 + # OBFUSCATED release rc13 (entlein/dx#138 fault 2 RESOLVED): garble -literals + # (WITHOUT -tiny — -tiny's pclntab stripping SIGSEGV'd under load). Passes the + # obfuscation gate AND survives the kill-chain (restarts=0, 4 rounds). Carries the + # evidence-manifest + DX_FOREST_PUSHDOWN code. + image: docker.io/entlein/dx-daemon:0.4.0-ssotforest-rc13 ports: - { name: findings, containerPort: 9099, hostPort: 9099 } env: From 5a52eb65a235388e3e24730308ea5274d15bae91 Mon Sep 17 00:00:00 2001 From: entlein Date: Sun, 16 Aug 2026 07:04:13 +0200 Subject: [PATCH 16/66] dx lab manifest: rc13 -> rc14 (clean trivy-free build, same config) --- k8s/vizier/dx/dx-daemon.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/vizier/dx/dx-daemon.yaml b/k8s/vizier/dx/dx-daemon.yaml index 1cb34480455..8b4eadf1bbc 100644 --- a/k8s/vizier/dx/dx-daemon.yaml +++ b/k8s/vizier/dx/dx-daemon.yaml @@ -23,7 +23,7 @@ spec: # (WITHOUT -tiny — -tiny's pclntab stripping SIGSEGV'd under load). Passes the # obfuscation gate AND survives the kill-chain (restarts=0, 4 rounds). Carries the # evidence-manifest + DX_FOREST_PUSHDOWN code. - image: docker.io/entlein/dx-daemon:0.4.0-ssotforest-rc13 + image: docker.io/entlein/dx-daemon:0.4.0-ssotforest-rc14 ports: - { name: findings, containerPort: 9099, hostPort: 9099 } env: From c5c2736dd48040b25e89102da6264c69d19ff8d1 Mon Sep 17 00:00:00 2001 From: entlein Date: Sun, 16 Aug 2026 09:01:05 +0200 Subject: [PATCH 17/66] adaptive_export: bounded-lookback watermark + wall-clock poison clamp + metrics (#97) The trigger's strict forward-only high-water-mark on the content event_time could silently halt AE forever (F8/AE-9, loadtest E8): one far-future row jumped the cursor past all real data, and out-of-order / clock-skewed / restart-buried rows were dropped with no signal. Fix: - Bounded lookback (ADAPTIVE_TRIGGER_LOOKBACK_SEC, default 300; 0 = legacy strict HWM): each poll scans [watermark-lookback, inf) and a bounded insertion-ordered LRU of row fingerprints (dedup.go) makes re-seen rows exactly-once. Includes in-window paging (catchup floor) so backlogs wider than PollLimit still drain. - Wall-clock poison clamp (ADAPTIVE_TRIGGER_MAX_SKEW_SEC, default 3600): a normalized event_time past now+skew is emitted once but never advances the cursor; an already-poisoned persisted watermark is clamped at load, so E8 recovers with no manual ALTER TABLE + restart. - Metrics on the default prometheus registry (metrics.go), served via the shared services/metrics /metrics handler in cmd/main.go (AE_PPROF_ADDR mux + optional AE_METRICS_ADDR listener): ae_trigger_watermark_ns{table,hostname}, ae_trigger_below_watermark_total, ae_trigger_event_time_rejected_total. normalizeEventTimeNanos stays as the first line of defense; the monotonic happy path with LOOKBACK=0 is byte-identical to before (existing suite runs unchanged). New tests: late-arrival exactly-once, below-lookback bound, E8 poison non-halt + self-recovery, strict-mode regression, dedup LRU unit tests. Fixes k8sstormcenter#97 --- .../services/adaptive_export/cmd/BUILD.bazel | 1 + .../services/adaptive_export/cmd/main.go | 58 +++- .../internal/trigger/BUILD.bazel | 11 +- .../internal/trigger/clickhouse.go | 234 +++++++++++-- .../adaptive_export/internal/trigger/dedup.go | 98 ++++++ .../internal/trigger/dedup_test.go | 92 +++++ .../internal/trigger/lookback_test.go | 321 ++++++++++++++++++ .../internal/trigger/metrics.go | 57 ++++ 8 files changed, 846 insertions(+), 26 deletions(-) create mode 100644 src/vizier/services/adaptive_export/internal/trigger/dedup.go create mode 100644 src/vizier/services/adaptive_export/internal/trigger/dedup_test.go create mode 100644 src/vizier/services/adaptive_export/internal/trigger/lookback_test.go create mode 100644 src/vizier/services/adaptive_export/internal/trigger/metrics.go diff --git a/src/vizier/services/adaptive_export/cmd/BUILD.bazel b/src/vizier/services/adaptive_export/cmd/BUILD.bazel index a1bbeb52e1c..8f299633e72 100644 --- a/src/vizier/services/adaptive_export/cmd/BUILD.bazel +++ b/src/vizier/services/adaptive_export/cmd/BUILD.bazel @@ -25,6 +25,7 @@ go_library( deps = [ "//src/api/go/pxapi", "//src/shared/services", + "//src/shared/services/metrics", "//src/vizier/services/adaptive_export/internal/activeset", "//src/vizier/services/adaptive_export/internal/clickhouse", "//src/vizier/services/adaptive_export/internal/config", diff --git a/src/vizier/services/adaptive_export/cmd/main.go b/src/vizier/services/adaptive_export/cmd/main.go index c5def0ab63b..1a33a0cae90 100644 --- a/src/vizier/services/adaptive_export/cmd/main.go +++ b/src/vizier/services/adaptive_export/cmd/main.go @@ -57,6 +57,7 @@ import ( "px.dev/pixie/src/api/go/pxapi" "px.dev/pixie/src/shared/services" + "px.dev/pixie/src/shared/services/metrics" "px.dev/pixie/src/vizier/services/adaptive_export/internal/activeset" "px.dev/pixie/src/vizier/services/adaptive_export/internal/clickhouse" "px.dev/pixie/src/vizier/services/adaptive_export/internal/config" @@ -116,6 +117,21 @@ const ( // drains in ceil(N/PollLimit) polls instead of one giant scan. envTriggerPollLimit = "ADAPTIVE_TRIGGER_POLL_LIMIT" + // envTriggerLookbackSec — bounded lookback below the trigger + // watermark (#97 / F8 / AE-9). Each poll re-scans + // [watermark-lookback, ∞) with content-fingerprint dedup so an + // out-of-order / clock-skewed / restart-buried kubescape row is + // still processed exactly once instead of being dropped forever + // (the "writes stop, data still on Pixie" halt). Default 300; + // explicit "0" restores the legacy strict high-water-mark. + envTriggerLookbackSec = "ADAPTIVE_TRIGGER_LOOKBACK_SEC" + + // envTriggerMaxSkewSec — wall-clock poison clamp (#97): a + // normalized event_time more than this many seconds past now never + // advances the watermark (counted in + // ae_trigger_event_time_rejected_total). Default 3600 (1h). + envTriggerMaxSkewSec = "ADAPTIVE_TRIGGER_MAX_SKEW_SEC" + // envWatermarkSaveSec — minimum interval between persistent // watermark INSERTs (default 5s). The in-memory watermark // advances every successful poll; flush is throttled. @@ -212,7 +228,7 @@ func main() { // DefaultServeMux. Bind loopback in containers unless you port-forward. if addr := os.Getenv("AE_PPROF_ADDR"); addr != "" { go func() { - log.WithField("addr", addr).Info("pprof listening (/debug/pprof/*)") + log.WithField("addr", addr).Info("pprof listening (/debug/pprof/* + /metrics)") if err := http.ListenAndServe(addr, nil); err != nil && err != http.ErrServerClosed { log.WithError(err).Error("pprof listener stopped") @@ -220,6 +236,23 @@ func main() { }() } + // Prometheus /metrics on the DefaultServeMux via the shared pixie + // metrics scaffold (same DefaultGatherer pattern every other pixie + // service uses). The trigger's #97 watermark metrics (ae_trigger_*) + // promauto-register on the default registry, so they are served here. + // Reachable on the AE_PPROF_ADDR listener above (same mux) and, for a + // scrape-only port, on AE_METRICS_ADDR (e.g. ":50901"; off when unset). + metrics.MustRegisterMetricsHandler(http.DefaultServeMux) + if addr := os.Getenv("AE_METRICS_ADDR"); addr != "" { + go func() { + log.WithField("addr", addr).Info("metrics listening (/metrics)") + if err := http.ListenAndServe(addr, nil); err != nil && + err != http.ErrServerClosed { + log.WithError(err).Error("metrics listener stopped") + } + }() + } + log.Info("starting adaptive-export operator (push flow, rev 2)") cfg, err := config.GetConfig() if err != nil { @@ -342,6 +375,9 @@ func main() { httpTimeout := durEnv(envTriggerHTTPTimeoutSec, 30*time.Second, time.Second) saveInterval := durEnv(envWatermarkSaveSec, 5*time.Second, time.Second) pollLimit := intEnv(envTriggerPollLimit, 10000) + // #97: bounded lookback (0 = legacy strict HWM) + poison clamp. + triggerLookback := durEnvZeroOK(envTriggerLookbackSec, 300*time.Second, time.Second) + triggerMaxSkew := durEnv(envTriggerMaxSkewSec, time.Hour, time.Second) // Persistent watermark store keeps the trigger's kubescape_logs // cursor in forensic_db.trigger_watermark, so a restart on a busy // node doesn't replay the full table from event_time=0 (which @@ -368,6 +404,8 @@ func main() { WatermarkSaveInterval: saveInterval, PollLimit: pollLimit, HTTPTimeout: httpTimeout, + Lookback: triggerLookback, + MaxSkew: triggerMaxSkew, }) if err != nil { log.WithError(err).Fatal("failed to create trigger") @@ -805,6 +843,24 @@ func durEnv(key string, dflt, unit time.Duration) time.Duration { return time.Duration(n) * unit } +// durEnvZeroOK is durEnv with 0 as a VALID value (= feature disabled): +// unset / unparseable / negative → dflt; explicit "0" → 0. Used for the +// #97 lookback knob where 0 deliberately selects the legacy strict HWM, +// so it must be distinguishable from "not configured". +func durEnvZeroOK(key string, dflt, unit time.Duration) time.Duration { + v := strings.TrimSpace(os.Getenv(key)) + if v == "" { + return dflt + } + n, err := strconv.ParseInt(v, 10, 64) + if err != nil || n < 0 { + log.WithFields(log.Fields{"key": key, "value": v}). + Warn("invalid duration env; using default") + return dflt + } + return time.Duration(n) * unit +} + // intEnv reads a positive-integer-valued env var. Returns dflt on // missing / unparseable / non-positive. Same shape as durEnv but // without the unit multiplier — for counts (e.g. row limits). diff --git a/src/vizier/services/adaptive_export/internal/trigger/BUILD.bazel b/src/vizier/services/adaptive_export/internal/trigger/BUILD.bazel index 0445d9211f4..8ffeb29b8e4 100644 --- a/src/vizier/services/adaptive_export/internal/trigger/BUILD.bazel +++ b/src/vizier/services/adaptive_export/internal/trigger/BUILD.bazel @@ -21,6 +21,8 @@ go_library( name = "trigger", srcs = [ "clickhouse.go", + "dedup.go", + "metrics.go", "watermark.go", ], importpath = "px.dev/pixie/src/vizier/services/adaptive_export/internal/trigger", @@ -28,6 +30,8 @@ go_library( deps = [ "//src/vizier/services/adaptive_export/internal/chhttp", "//src/vizier/services/adaptive_export/internal/kubescape", + "@com_github_prometheus_client_golang//prometheus", + "@com_github_prometheus_client_golang//prometheus/promauto", "@com_github_sirupsen_logrus//:logrus", ], ) @@ -37,10 +41,15 @@ pl_go_test( srcs = [ "clickhouse_internal_test.go", "clickhouse_test.go", + "dedup_test.go", "fingerprint_bench_test.go", + "lookback_test.go", "oracle_test.go", "watermark_test.go", ], embed = [":trigger"], - deps = ["//src/vizier/services/adaptive_export/internal/kubescape"], + deps = [ + "//src/vizier/services/adaptive_export/internal/kubescape", + "@com_github_prometheus_client_golang//prometheus/testutil", + ], ) diff --git a/src/vizier/services/adaptive_export/internal/trigger/clickhouse.go b/src/vizier/services/adaptive_export/internal/trigger/clickhouse.go index 80e03f3b942..1f548e37cc2 100644 --- a/src/vizier/services/adaptive_export/internal/trigger/clickhouse.go +++ b/src/vizier/services/adaptive_export/internal/trigger/clickhouse.go @@ -84,13 +84,48 @@ type Config struct { // hardcoded to 5s, which under any backlog caused every poll to // time out mid-stream → watermark never advanced. HTTPTimeout time.Duration + + // Lookback (#97 / F8 / AE-9): when > 0, each poll re-scans + // [watermark-Lookback, ∞) instead of the strict [watermark, ∞) and + // dedupes re-seen rows by content fingerprint, so an out-of-order / + // clock-skewed / restart-buried row that lands within the window is + // still processed EXACTLY ONCE (no drop, no duplicate). Rows below + // watermark-Lookback stay dropped — the documented bound. 0 keeps + // the legacy strict high-water-mark behavior (anything below the + // watermark is dropped forever). Production default is 300s via + // ADAPTIVE_TRIGGER_LOOKBACK_SEC in cmd/main.go; the zero value here + // is legacy so existing callers/tests are unchanged. + Lookback time.Duration + + // MaxSkew is the wall-clock poison clamp (#97): a row whose + // NORMALIZED event_time is more than MaxSkew past now is still + // emitted once, but never advances the watermark, so a single + // corrupted/oversized timestamp (the 1.78e18 leftover of loadtest + // E8) cannot jump the cursor past all real data and silently halt + // the trigger. Also applied to the persisted watermark at load, so + // an ALREADY-poisoned cursor self-recovers on restart without the + // manual `ALTER TABLE trigger_watermark DELETE`. <=0 → 1h. + MaxSkew time.Duration + + // DedupMaxEntries caps the lookback dedup set (memory bound). An + // in-window fingerprint evicted by capacity may re-emit once, so + // size it >= the max rows expected per lookback window. + // <=0 → 4*PollLimit. + DedupMaxEntries int } +// defaultMaxSkew is the default wall-clock poison-clamp bound (#97): +// an event_time more than this far in the future is implausible. +const defaultMaxSkew = time.Hour + // ClickHouseHTTP polls forensic_db. over the ClickHouse HTTP // interface, scoped to a single node. type ClickHouseHTTP struct { cfg Config client *http.Client + // now is the wall clock used by the poison clamp (#97). + // Injectable for deterministic tests; time.Now in production. + now func() time.Time } // New validates Config and returns a ready trigger. @@ -143,9 +178,19 @@ func New(cfg Config) (*ClickHouseHTTP, error) { if cfg.HTTPTimeout <= 0 { cfg.HTTPTimeout = 30 * time.Second } + if cfg.Lookback < 0 { + return nil, fmt.Errorf("trigger: Lookback must be >= 0 (got %v)", cfg.Lookback) + } + if cfg.MaxSkew <= 0 { + cfg.MaxSkew = defaultMaxSkew + } + if cfg.DedupMaxEntries <= 0 { + cfg.DedupMaxEntries = 4 * cfg.PollLimit + } return &ClickHouseHTTP{ cfg: cfg, client: &http.Client{Timeout: cfg.HTTPTimeout}, + now: time.Now, }, nil } @@ -169,11 +214,15 @@ func (t *ClickHouseHTTP) Subscribe(ctx context.Context) (<-chan kubescape.Event, func (t *ClickHouseHTTP) run(ctx context.Context, out chan<- kubescape.Event) { defer close(out) // Watermark uses event_time as the cursor PLUS a set of row - // fingerprints already pushed at that exact event_time. This - // closes the race where two kubescape rows share the same - // event_time but the second arrives after our previous poll: the - // query is `event_time >= watermark` (inclusive) and we skip rows - // whose fingerprint we have already seen at the boundary. + // fingerprints already pushed. In legacy strict mode (Lookback==0) + // the query is `event_time >= watermark` (inclusive) and the + // fingerprint set covers only the exact boundary event_time — + // closing the race where two kubescape rows share the same + // event_time but the second arrives after our previous poll. With + // a bounded lookback (#97, the F8/AE-9 fix) the query starts at + // max(0, watermark-Lookback) and the fingerprint set is a bounded + // LRU over the whole re-scanned window, so out-of-order / skewed / + // restart-buried rows inside the window are captured exactly once. // // Cold-start order: persistent store > InitialWatermark > 0. // The persistent store is the production answer to "operator @@ -203,7 +252,39 @@ func (t *ClickHouseHTTP) run(ctx context.Context, out chan<- kubescape.Event) { // pre-fix persisted seconds watermark (or a non-seconds InitialWatermark) // is interpreted on the same scale as chNormEventTimeNanos in the SQL. watermark = normalizeEventTimeNanos(watermark) + maxSkewNS := uint64(t.cfg.MaxSkew.Nanoseconds()) + lookbackNS := uint64(t.cfg.Lookback.Nanoseconds()) + // Self-recovery from an ALREADY-poisoned persisted cursor (#97 T1): + // a pre-fix deployment could have persisted a far-future watermark + // (loadtest E8's leftover 1.78e18-style value). Clamp it to + // wall-clock so fresh rows flow again on restart WITHOUT the manual + // `ALTER TABLE trigger_watermark DELETE WHERE 1=1` + redeploy. + if nowNS := uint64(t.now().UnixNano()); watermark > nowNS+maxSkewNS { + log.WithFields(log.Fields{"watermark": watermark, "clamped_to": nowNS}). + Warn("trigger: persisted watermark is implausibly far in the future — clamping to wall-clock (poison recovery, #97)") + watermark = nowNS + } + wmGauge := metricWatermarkNS.WithLabelValues(t.cfg.Table, t.cfg.Hostname) + wmGauge.Set(float64(watermark)) + // Dedup state. Strict mode (Lookback==0) keeps the legacy exact + // boundary set; lookback mode dedupes the whole re-scanned window + // with a bounded LRU (#97). rejectedSeen exists only in strict mode: + // a clamp-rejected row never falls below the cursor, so without a + // fingerprint record it would re-emit on every poll. seenAtBoundary := map[string]bool{} + var seenInWindow *dedupLRU + var rejectedSeen *dedupLRU + if lookbackNS > 0 { + seenInWindow = newDedupLRU(t.cfg.DedupMaxEntries) + } else { + rejectedSeen = newDedupLRU(t.cfg.DedupMaxEntries) + } + // catchup lifts a poll's lower bound above the sliding lookback + // floor while an in-window backlog is wider than PollLimit: without + // it every poll would re-fetch the same fully-deduped first + // PollLimit rows and never reach deeper into the window. Cleared as + // soon as a poll returns under capacity (back to full-window scans). + var catchup uint64 ticker := time.NewTicker(t.cfg.PollInterval) defer ticker.Stop() @@ -253,7 +334,21 @@ func (t *ClickHouseHTTP) run(ctx context.Context, out chan<- kubescape.Event) { }() pollOnce := func() { - rows, maxSeen, err := t.fetchSince(ctx, watermark) + // Bounded lookback (#97): scan from max(0, watermark-Lookback) + // so rows that landed BELOW the cursor (out-of-order, clock + // skew, restart burial) are still fetched; the dedup LRU makes + // re-seen rows exactly-once. Lookback==0 → legacy strict HWM. + queryFrom := watermark + if lookbackNS > 0 { + queryFrom = 0 + if watermark > lookbackNS { + queryFrom = watermark - lookbackNS + } + if catchup > queryFrom { + queryFrom = catchup + } + } + rows, maxFetched, err := t.fetchSince(ctx, queryFrom) // Partial-read tolerance: when the body read is cut short by // HTTP timeout / connection reset, fetchSince returns the rows // it managed to parse + err. We still process those rows so @@ -267,6 +362,19 @@ func (t *ClickHouseHTTP) run(ctx context.Context, out chan<- kubescape.Event) { log.WithError(err).WithField("partial_rows", len(rows)). Warn("trigger: poll partial — advancing on what parsed") } + // Wall-clock poison clamp (#97): any normalized event_time past + // now+MaxSkew must never advance the cursor. acceptedMax is the + // advancement target — the max normalized event_time among rows + // that PASS the clamp. With no poison rows it equals maxFetched, + // so the monotonic happy path is byte-identical to before. + skewLimit := uint64(t.now().UnixNano()) + maxSkewNS + acceptedMax := uint64(0) + for _, row := range rows { + if evn := normalizeEventTimeNanos(row.EventTime); evn <= skewLimit && evn > acceptedMax { + acceptedMax = evn + } + } + wmAtPollStart := watermark nextSeen := map[string]bool{} // Periodic in-loop save: when pollOnce is draining a large // initial backlog, the watermark advances long before the @@ -276,45 +384,122 @@ func (t *ClickHouseHTTP) run(ctx context.Context, out chan<- kubescape.Event) { // with the time-based throttle inside flushWatermark, this // produces at most one persistent INSERT per WatermarkSaveInterval. const saveEveryN = 256 - skippedAtBoundary := 0 + skippedSeen := 0 + emitted := 0 for i, row := range rows { fp := rowFingerprint(row) // Cursor comparisons are in NORMALIZED nanos (F8): the raw // event_time unit is not enforced, so compare on the same scale - // as the SQL filter (chNormEventTimeNanos) and maxSeen. + // as the SQL filter (chNormEventTimeNanos) and acceptedMax. evn := normalizeEventTimeNanos(row.EventTime) - if evn == watermark && seenAtBoundary[fp] { - skippedAtBoundary++ - continue // already pushed in a prior poll at this exact boundary + if lookbackNS > 0 { + if seenInWindow.Contains(fp) { + skippedSeen++ + continue // already pushed in a prior scan of this window + } + } else { + if evn == watermark && seenAtBoundary[fp] { + skippedSeen++ + continue // already pushed in a prior poll at this exact boundary + } + if rejectedSeen.Contains(fp) { + continue // clamp-rejected row re-fetched (it never sinks below the cursor) + } } - ev, err := kubescape.Extract(row) - if err != nil { - log.WithError(err).Debug("trigger: skip incomplete row") + poison := evn > skewLimit + ev, exErr := kubescape.Extract(row) + if exErr != nil { + log.WithError(exErr).Debug("trigger: skip incomplete row") + // Register the fingerprint anyway (lookback / poison): + // the row can never become extractable, and without a + // record it would be re-fetched + re-logged every poll + // for as long as it stays above the scan floor. + if lookbackNS > 0 { + seenInWindow.Add(fp, evn) + } else if poison { + rejectedSeen.Add(fp, evn) + } continue } - // Promote the per-row (normalized) event_time into the watermark - // immediately so flushWatermark below can persist mid-drain. - if evn > watermark { - watermark = evn - dirty = true + if poison { + // Emit the row once (it may be a real anomaly with a + // mangled timestamp) but do NOT let it advance the + // cursor: one 1.78e18 row must not jump the watermark + // past all real seconds rows (F8 halt). + metricEventTimeRejected.Inc() + log.WithFields(log.Fields{ + "event_time": row.EventTime, + "normalized": evn, + "skew_limit": skewLimit, + }).Warn("trigger: event_time beyond wall-clock skew bound — processing row WITHOUT advancing watermark (poison clamp, #97)") + } else { + if evn < wmAtPollStart { + // A row the legacy strict HWM would have dropped — + // captured via the lookback (T2). Observable proof + // the fix is doing work (T3). + metricBelowWatermark.Inc() + } + // Promote the per-row (normalized) event_time into the watermark + // immediately so flushWatermark below can persist mid-drain. + if evn > watermark { + watermark = evn + dirty = true + wmGauge.Set(float64(watermark)) + } + } + if lookbackNS > 0 { + seenInWindow.Add(fp, evn) + } else if poison { + rejectedSeen.Add(fp, evn) } select { case out <- ev: case <-ctx.Done(): return } - if evn == maxSeen { + emitted++ + if !poison && evn == acceptedMax { nextSeen[fp] = true } if i > 0 && i%saveEveryN == 0 { flushWatermark() } } - if maxSeen > watermark { - watermark = maxSeen + if lookbackNS > 0 { + if acceptedMax > watermark { + watermark = acceptedMax + dirty = true + wmGauge.Set(float64(watermark)) + } + // Paging within the window: a saturated response means the + // window holds more rows than PollLimit — lift the floor so + // the next poll pages FORWARD instead of re-fetching the + // same deduped prefix forever. + if len(rows) >= t.cfg.PollLimit { + if emitted == 0 && skippedSeen == len(rows) { + // Every row in the saturated page was already seen — + // step past the page entirely (lookback analog of the + // legacy 1ns boundary escape). + catchup = maxFetched + 1 + } else if acceptedMax > catchup { + catchup = acceptedMax + } + } else { + catchup = 0 + } + // Entries below the sliding floor can never be re-fetched; + // evict them so the LRU stays at ~window size. + floor := uint64(0) + if watermark > lookbackNS { + floor = watermark - lookbackNS + } + seenInWindow.EvictBelow(floor) + } else if acceptedMax > watermark { + watermark = acceptedMax seenAtBoundary = nextSeen dirty = true - } else if maxSeen == watermark { + wmGauge.Set(float64(watermark)) + } else if acceptedMax == watermark { // no progress this tick — preserve boundary set, optionally extend for fp := range nextSeen { seenAtBoundary[fp] = true @@ -329,10 +514,11 @@ func (t *ClickHouseHTTP) run(ctx context.Context, out chan<- kubescape.Event) { // the next poll, which is acceptable: the fingerprint dedup already // tolerates boundary overlap, and we prefer forward progress over // an infinite loop. - if skippedAtBoundary > 0 && len(nextSeen) == 0 && len(rows) >= t.cfg.PollLimit { + if skippedSeen > 0 && len(nextSeen) == 0 && len(rows) >= t.cfg.PollLimit { watermark++ seenAtBoundary = map[string]bool{} dirty = true + wmGauge.Set(float64(watermark)) log.WithField("watermark", watermark). Warn("trigger: boundary paging escape — advanced watermark by 1ns to unblock poll") } diff --git a/src/vizier/services/adaptive_export/internal/trigger/dedup.go b/src/vizier/services/adaptive_export/internal/trigger/dedup.go new file mode 100644 index 00000000000..b1a0d587406 --- /dev/null +++ b/src/vizier/services/adaptive_export/internal/trigger/dedup.go @@ -0,0 +1,98 @@ +// Copyright 2018- The Pixie Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package trigger + +import "container/list" + +// dedupLRU is a bounded, insertion-ordered set of row fingerprints, +// each tagged with the row's normalized event_time (nanos). It is the +// #97 (F8/AE-9) extension of the old single-boundary `seenAtBoundary` +// map: with a bounded lookback the trigger re-fetches every row in +// [watermark-Lookback, watermark] on each poll, so dedup must cover the +// whole window, not just the exact watermark boundary. +// +// Eviction is two-fold: +// - EvictBelow(floor): entries whose event_time has slid below the +// lookback floor can never be returned by the SELECT again, so they +// are dropped eagerly to keep the set at ~window size. +// - capacity: Add evicts the OLDEST INSERTION when over max, bounding +// memory even if the window holds more rows than expected. An +// in-window entry evicted by capacity may cause one duplicate emit — +// the documented trade-off for bounded memory (size it >= the max +// rows per window; default 4*PollLimit). +// +// Not goroutine-safe; owned by the single poll loop. +type dedupLRU struct { + max int + ll *list.List // front = oldest insertion + items map[string]*list.Element +} + +type dedupEntry struct { + fp string + evn uint64 // normalized event_time (nanos) +} + +func newDedupLRU(max int) *dedupLRU { + if max <= 0 { + max = 1 + } + return &dedupLRU{max: max, ll: list.New(), items: map[string]*list.Element{}} +} + +// Contains reports whether fp was Added and not yet evicted. +func (d *dedupLRU) Contains(fp string) bool { + _, ok := d.items[fp] + return ok +} + +// Add records fp with its normalized event_time. No-op if already +// present. Evicts oldest insertions while over capacity. +func (d *dedupLRU) Add(fp string, evn uint64) { + if _, ok := d.items[fp]; ok { + return + } + d.items[fp] = d.ll.PushBack(dedupEntry{fp: fp, evn: evn}) + for d.ll.Len() > d.max { + d.removeElement(d.ll.Front()) + } +} + +// EvictBelow drops entries with evn < floor, popping from the oldest +// insertion. Insertion order tracks the poll's ORDER BY event_time, so +// in the common case this removes exactly the expired prefix. A late +// arrival (low evn inserted after a higher one) may survive behind a +// newer entry until capacity eviction — harmless: Contains on an +// expired fp only suppresses a row the SELECT can no longer return. +func (d *dedupLRU) EvictBelow(floor uint64) { + for e := d.ll.Front(); e != nil; { + if e.Value.(dedupEntry).evn >= floor { + return + } + next := e.Next() + d.removeElement(e) + e = next + } +} + +// Len returns the number of live entries. +func (d *dedupLRU) Len() int { return d.ll.Len() } + +func (d *dedupLRU) removeElement(e *list.Element) { + delete(d.items, e.Value.(dedupEntry).fp) + d.ll.Remove(e) +} diff --git a/src/vizier/services/adaptive_export/internal/trigger/dedup_test.go b/src/vizier/services/adaptive_export/internal/trigger/dedup_test.go new file mode 100644 index 00000000000..c139b21831c --- /dev/null +++ b/src/vizier/services/adaptive_export/internal/trigger/dedup_test.go @@ -0,0 +1,92 @@ +// Copyright 2018- The Pixie Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package trigger + +import "testing" + +func TestDedupLRU_AddContains(t *testing.T) { + d := newDedupLRU(8) + if d.Contains("a") { + t.Fatalf("empty LRU claims to contain a") + } + d.Add("a", 100) + d.Add("b", 200) + if !d.Contains("a") || !d.Contains("b") { + t.Fatalf("added fingerprints not found") + } + if d.Len() != 2 { + t.Fatalf("Len = %d, want 2", d.Len()) + } + // Duplicate Add is a no-op (no double-entry, no reorder). + d.Add("a", 100) + if d.Len() != 2 { + t.Fatalf("duplicate Add changed Len to %d", d.Len()) + } +} + +func TestDedupLRU_CapacityEvictsOldestInsertion(t *testing.T) { + d := newDedupLRU(3) + d.Add("a", 1) + d.Add("b", 2) + d.Add("c", 3) + d.Add("d", 4) // over capacity → "a" (oldest insertion) evicted + if d.Contains("a") { + t.Fatalf("oldest entry not evicted at capacity") + } + for _, fp := range []string{"b", "c", "d"} { + if !d.Contains(fp) { + t.Fatalf("entry %q evicted unexpectedly", fp) + } + } + if d.Len() != 3 { + t.Fatalf("Len = %d, want 3", d.Len()) + } +} + +func TestDedupLRU_EvictBelow(t *testing.T) { + d := newDedupLRU(8) + d.Add("a", 100) + d.Add("b", 200) + d.Add("c", 300) + d.EvictBelow(250) + if d.Contains("a") || d.Contains("b") { + t.Fatalf("entries below floor survived EvictBelow") + } + if !d.Contains("c") { + t.Fatalf("entry at/above floor was evicted") + } + // EvictBelow stops at the first entry >= floor (prefix semantics): + // a late arrival (low evn inserted AFTER a higher one) survives — + // documented as harmless. + d.Add("late", 50) + d.EvictBelow(250) + if !d.Contains("late") { + t.Fatalf("late-arrival entry behind a newer one should survive prefix eviction") + } +} + +func TestDedupLRU_ZeroCapacityIsSafe(t *testing.T) { + d := newDedupLRU(0) // clamped to 1 + d.Add("a", 1) + if !d.Contains("a") { + t.Fatalf("single entry not retained") + } + d.Add("b", 2) + if d.Contains("a") || !d.Contains("b") { + t.Fatalf("capacity-1 eviction wrong: a=%v b=%v", d.Contains("a"), d.Contains("b")) + } +} diff --git a/src/vizier/services/adaptive_export/internal/trigger/lookback_test.go b/src/vizier/services/adaptive_export/internal/trigger/lookback_test.go new file mode 100644 index 00000000000..8fa153001fa --- /dev/null +++ b/src/vizier/services/adaptive_export/internal/trigger/lookback_test.go @@ -0,0 +1,321 @@ +// Copyright 2018- The Pixie Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +// Bounded-lookback + wall-clock poison-clamp tests (#97 / F8 / AE-9). +// No live ClickHouse: a stub HTTP server implements the trigger's +// JSONEachRow contract INCLUDING the `>= ` watermark predicate, +// so re-poll semantics (the essence of lookback) are exercised for real. + +package trigger + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "regexp" + "sort" + "strconv" + "sync" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus/testutil" +) + +// fakeCH is a stub ClickHouse HTTP endpoint that stores rows and, like +// the real server, only returns rows whose NORMALIZED event_time is >= +// the bound parsed out of the trigger's SELECT. +type fakeCH struct { + mu sync.Mutex + rows []fakeRow + srv *httptest.Server +} + +type fakeRow struct { + eventTime uint64 // raw, unit-ambiguous — exactly like production + ruleID string + pid int +} + +var boundRE = regexp.MustCompile(`>= (\d+) ORDER`) + +func newFakeCH(t *testing.T) *fakeCH { + f := &fakeCH{} + f.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query().Get("query") + m := boundRE.FindStringSubmatch(q) + if m == nil { + t.Errorf("query without >= bound: %q", q) + w.WriteHeader(400) + return + } + bound, err := strconv.ParseUint(m[1], 10, 64) + if err != nil { + t.Errorf("unparseable bound in query %q: %v", q, err) + w.WriteHeader(400) + return + } + f.mu.Lock() + var out []fakeRow + for _, row := range f.rows { + if normalizeEventTimeNanos(row.eventTime) >= bound { + out = append(out, row) + } + } + f.mu.Unlock() + sort.Slice(out, func(i, j int) bool { + return normalizeEventTimeNanos(out[i].eventTime) < normalizeEventTimeNanos(out[j].eventTime) + }) + for _, row := range out { + fmt.Fprintf(w, + `{"RuleID":%q,"RuntimeK8sDetails":"{\"podName\":\"p-1\",\"podNamespace\":\"ns\"}","RuntimeProcessDetails":"{\"processTree\":{\"pid\":%d,\"comm\":\"c\"}}","event_time":"%d","hostname":"node-1"}`+"\n", + row.ruleID, row.pid, row.eventTime) + } + })) + return f +} + +func (f *fakeCH) add(r fakeRow) { + f.mu.Lock() + f.rows = append(f.rows, r) + f.mu.Unlock() +} + +func (f *fakeCH) close() { f.srv.Close() } + +// testBase is a fixed "now" for deterministic clamp behavior: +// 2026-05-29T… ≈ 1.7805e9 seconds. +const testBase = uint64(1_780_500_000) + +func fixedNow() time.Time { return time.Unix(int64(testBase), 0) } + +// newLookbackTrigger builds a trigger against the fake server with the +// #97 config (300s lookback) and a pinned wall clock. +func newLookbackTrigger(t *testing.T, f *fakeCH, hostname string, lookback time.Duration) *ClickHouseHTTP { + t.Helper() + tr, err := New(Config{ + Endpoint: f.srv.URL, + Hostname: hostname, + PollInterval: 20 * time.Millisecond, + Lookback: lookback, + }) + if err != nil { + t.Fatalf("New: %v", err) + } + tr.now = fixedNow // deterministic poison clamp + return tr +} + +// TestTrigger_LookbackCapturesLateArrivalExactlyOnce — T2: a row that +// lands BELOW the watermark but inside the lookback window is processed +// exactly once (no drop, no duplicate over many re-polls), and a row +// below watermark-lookback stays dropped (the documented bound). Also +// asserts ae_trigger_below_watermark_total increments (T3). +func TestTrigger_LookbackCapturesLateArrivalExactlyOnce(t *testing.T) { + f := newFakeCH(t) + defer f.close() + f.add(fakeRow{eventTime: testBase, ruleID: "R1", pid: 111}) // head row → watermark = testBase + + belowBefore := testutil.ToFloat64(metricBelowWatermark) + + tr := newLookbackTrigger(t, f, "node-lb", 300*time.Second) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ch, _ := tr.Subscribe(ctx) + + // Wait for the head row so the watermark is at testBase. + select { + case ev := <-ch: + if ev.Target.PID != 111 { + t.Fatalf("first event PID = %d, want 111", ev.Target.PID) + } + case <-time.After(500 * time.Millisecond): + t.Fatalf("timeout waiting for head row") + } + + // Late arrival 60s below the watermark (inside the 300s window) and + // one 400s below (outside the window). + f.add(fakeRow{eventTime: testBase - 60, ruleID: "R2", pid: 222}) + f.add(fakeRow{eventTime: testBase - 400, ruleID: "R3", pid: 333}) + + got := map[uint64]int{} + deadline := time.Now().Add(400 * time.Millisecond) // ~20 re-polls of the same window + for time.Now().Before(deadline) { + select { + case ev := <-ch: + got[ev.Target.PID]++ + case <-time.After(20 * time.Millisecond): + } + } + if got[222] != 1 { + t.Errorf("late-arrival row emitted %d times, want exactly 1 (T2)", got[222]) + } + if got[333] != 0 { + t.Errorf("row below watermark-lookback emitted %d times, want 0 (documented bound)", got[333]) + } + if got[111] != 0 { + t.Errorf("head row re-emitted %d times after initial delivery (window dedup failed)", got[111]) + } + if delta := testutil.ToFloat64(metricBelowWatermark) - belowBefore; delta < 1 { + t.Errorf("ae_trigger_below_watermark_total delta = %v, want >= 1", delta) + } +} + +// TestTrigger_PoisonRowDoesNotHalt — T1 (the F8 non-halt guarantee): a +// row carrying the real E8 poison timestamp (1.78e18-style far-future +// vs the pinned clock) is clamp-rejected from advancing the watermark, +// the reject metric increments, the watermark gauge stays wall-clock- +// bounded, and SUBSEQUENT seconds rows are still processed — no manual +// watermark reset needed. +func TestTrigger_PoisonRowDoesNotHalt(t *testing.T) { + // The exact leftover value from loadtest E8's poisoned watermark. + // Normalized it stays 1.781559e18 ns ≈ 12 days past the pinned + // clock (1.7805e9 s) — beyond the 1h MaxSkew. + const poisonET = uint64(1781559619170395824) + + f := newFakeCH(t) + defer f.close() + f.add(fakeRow{eventTime: testBase - 10, ruleID: "R1", pid: 111}) + + rejBefore := testutil.ToFloat64(metricEventTimeRejected) + + tr := newLookbackTrigger(t, f, "node-poison", 300*time.Second) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ch, _ := tr.Subscribe(ctx) + + select { + case ev := <-ch: + if ev.Target.PID != 111 { + t.Fatalf("first event PID = %d, want 111", ev.Target.PID) + } + case <-time.After(500 * time.Millisecond): + t.Fatalf("timeout waiting for head row") + } + + // Inject the poison row; it is emitted once (real anomaly, mangled + // timestamp) but must not advance the cursor. + f.add(fakeRow{eventTime: poisonET, ruleID: "RPOISON", pid: 666}) + select { + case ev := <-ch: + if ev.Target.PID != 666 { + t.Fatalf("expected poison row emission, got PID %d", ev.Target.PID) + } + case <-time.After(500 * time.Millisecond): + t.Fatalf("poison row was dropped entirely; want emitted-once-without-advance") + } + if delta := testutil.ToFloat64(metricEventTimeRejected) - rejBefore; delta < 1 { + t.Errorf("ae_trigger_event_time_rejected_total delta = %v, want >= 1", delta) + } + + // THE F8 guarantee: a fresh seconds row AFTER the poison must flow. + // Under the old strict HWM the cursor sat at 1.78e18 and this row + // was below it forever (25/25 ticks at n_anomalies=0 in E8). + f.add(fakeRow{eventTime: testBase + 5, ruleID: "R2", pid: 222}) + var got222 int + deadline := time.Now().Add(600 * time.Millisecond) + for time.Now().Before(deadline) && got222 == 0 { + select { + case ev := <-ch: + if ev.Target.PID == 222 { + got222++ + } + case <-time.After(20 * time.Millisecond): + } + } + if got222 != 1 { + t.Fatalf("post-poison seconds row emitted %d times, want 1 (T1 non-halt)", got222) + } + + // Watermark gauge stays wall-clock-bounded: it advanced to the real + // row (testBase+5 s), NOT to the poison value. + wantWM := float64(normalizeEventTimeNanos(testBase + 5)) + if got := testutil.ToFloat64(metricWatermarkNS.WithLabelValues("kubescape_logs", "node-poison")); got != wantWM { + t.Errorf("ae_trigger_watermark_ns = %v, want %v (wall-clock-bounded, not poison)", got, wantWM) + } +} + +// TestTrigger_PoisonPersistedWatermarkSelfRecovers — the E8 recovery +// scenario without the manual ALTER TABLE … DELETE: a pre-fix deployment +// left a far-future watermark behind; on start the trigger clamps it to +// wall-clock and fresh rows flow again. +func TestTrigger_PoisonPersistedWatermarkSelfRecovers(t *testing.T) { + const poisonWM = uint64(1781559619170395824) + + f := newFakeCH(t) + defer f.close() + f.add(fakeRow{eventTime: testBase, ruleID: "R1", pid: 111}) + + tr := newLookbackTrigger(t, f, "node-recover", 300*time.Second) + tr.cfg.InitialWatermark = poisonWM // simulates the poisoned persisted cursor + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ch, _ := tr.Subscribe(ctx) + + select { + case ev := <-ch: + if ev.Target.PID != 111 { + t.Fatalf("recovered event PID = %d, want 111", ev.Target.PID) + } + case <-time.After(500 * time.Millisecond): + t.Fatalf("fresh row not delivered — poisoned persisted watermark was not clamped (still halted)") + } +} + +// TestTrigger_LookbackZeroIsStrictHWM — T4: LOOKBACK=0 preserves the +// legacy strict high-water-mark exactly — the poll bound IS the +// watermark (no window subtraction) and a below-watermark row stays +// dropped. (The monotonic happy path itself is pinned by the existing +// clickhouse_test.go suite, which runs with the zero-value Lookback.) +func TestTrigger_LookbackZeroIsStrictHWM(t *testing.T) { + f := newFakeCH(t) + defer f.close() + f.add(fakeRow{eventTime: testBase, ruleID: "R1", pid: 111}) + + tr := newLookbackTrigger(t, f, "node-strict", 0) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ch, _ := tr.Subscribe(ctx) + + select { + case ev := <-ch: + if ev.Target.PID != 111 { + t.Fatalf("first event PID = %d, want 111", ev.Target.PID) + } + case <-time.After(500 * time.Millisecond): + t.Fatalf("timeout waiting for head row") + } + + // A late arrival below the watermark: with strict HWM the SELECT + // bound equals the watermark, so it is never fetched again → dropped. + f.add(fakeRow{eventTime: testBase - 60, ruleID: "R2", pid: 222}) + got := map[uint64]int{} + deadline := time.Now().Add(300 * time.Millisecond) + for time.Now().Before(deadline) { + select { + case ev := <-ch: + got[ev.Target.PID]++ + case <-time.After(20 * time.Millisecond): + } + } + if got[222] != 0 { + t.Errorf("strict mode emitted a below-watermark row %d times; want 0 (legacy behavior)", got[222]) + } + if got[111] != 0 { + t.Errorf("strict mode re-emitted the boundary row %d times; want 0", got[111]) + } +} diff --git a/src/vizier/services/adaptive_export/internal/trigger/metrics.go b/src/vizier/services/adaptive_export/internal/trigger/metrics.go new file mode 100644 index 00000000000..4fc9dee9eda --- /dev/null +++ b/src/vizier/services/adaptive_export/internal/trigger/metrics.go @@ -0,0 +1,57 @@ +// Copyright 2018- The Pixie Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package trigger + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +// Watermark observability (#97 / F8 / AE-9). Registered on the DEFAULT +// prometheus registry via promauto — the same pattern the rest of pixie +// uses (e.g. query_broker's queryExec* summaries) — and served by the +// shared services/metrics /metrics handler wired up in cmd/main.go. +// Before these existed a watermark halt was completely invisible: writes +// stopped, no error, no signal (loadtest E8). +var ( + // metricWatermarkNS tracks the trigger's current cursor in + // normalized unix NANOS, per (table, hostname). A flat gauge while + // kubescape rows keep arriving is the F8 silent-halt signature — + // alert on it. + metricWatermarkNS = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "ae_trigger_watermark_ns", + Help: "Current trigger high-water-mark cursor in normalized unix nanoseconds, per (table, hostname).", + }, []string{"table", "hostname"}) + + // metricBelowWatermark counts rows processed with a normalized + // event_time BELOW the poll-start watermark — i.e. out-of-order / + // clock-skewed / restart-buried rows the legacy strict HWM silently + // dropped and the bounded lookback now captures. + metricBelowWatermark = promauto.NewCounter(prometheus.CounterOpts{ + Name: "ae_trigger_below_watermark_total", + Help: "Rows seen with event_time below the prior watermark that the bounded lookback captured (strict HWM would have dropped them).", + }) + + // metricEventTimeRejected counts poison clamps: rows whose + // normalized event_time was implausibly far in the future + // (> now + MaxSkew) and were therefore barred from advancing the + // watermark. + metricEventTimeRejected = promauto.NewCounter(prometheus.CounterOpts{ + Name: "ae_trigger_event_time_rejected_total", + Help: "Rows whose normalized event_time exceeded now+max-skew and were rejected from advancing the watermark (poison clamp).", + }) +) From 29a67c31806640dae5ea6eccba07e15c39c91550 Mon Sep 17 00:00:00 2001 From: entlein Date: Sun, 16 Aug 2026 09:06:32 +0200 Subject: [PATCH 18/66] =?UTF-8?q?adaptive=5Fexport:=20secure-by-default=20?= =?UTF-8?q?control=20surface=20=E2=80=94=20TLS=20+=20auth=20ON=20(#96)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Flip the dx→AE control surface (:9100) to secure-by-default so the bearer JWT + control payloads no longer cross the CNI in cleartext. - TLS default-ON. Mounted /certs/server.{crt,key} (service-tls-certs) win; else AE self-generates an ephemeral in-memory ECDSA P-256 self-signed cert (1y, SAN localhost/127.0.0.1/::1/node) so TLS works with zero extra secrets. Plaintext ONLY via explicit CONTROL_INSECURE=true (loud WARN). - Auth default-ON whenever PL_JWT_SIGNING_KEY is present (drops the extra CONTROL_REQUIRE_AUTH gate). No key + no CONTROL_INSECURE => fail-closed: the control HTTP surface refuses to start; the rest of AE keeps running. - CONTROL_TLS / CONTROL_REQUIRE_AUTH become deprecated no-ops (warn if set); CONTROL_TLS_CERT/KEY kept as overrides; new CONTROL_INSECURE opt-out. control/tls.go: TLSConfig(cert,key,hosts) + selfSignedCert + certToPEM helper. control/tls_test.go: self-gen serves TLS /healthz, TLS rejects unauthenticated, mounted-cert load path, plaintext opt-out path. BUILD.bazel srcs updated. Manifests: AE deployment already mounts /certs + PL_JWT_SIGNING_KEY (no CONTROL_TLS to drop) — added a secure-by-default note. dx-daemon AE_CONTROL_ADDR http:// -> https:// (dx client skip-verifies). Stacks on #92 (fix/ae-protocol-export-pxexport); does not touch #97 code. --- .../bootstrap/adaptive_export_deployment.yaml | 5 + k8s/vizier/dx/dx-daemon.yaml | 4 +- .../services/adaptive_export/cmd/main.go | 144 ++++++++----- .../internal/control/BUILD.bazel | 10 +- .../adaptive_export/internal/control/tls.go | 128 ++++++++++++ .../internal/control/tls_test.go | 194 ++++++++++++++++++ 6 files changed, 431 insertions(+), 54 deletions(-) create mode 100644 src/vizier/services/adaptive_export/internal/control/tls.go create mode 100644 src/vizier/services/adaptive_export/internal/control/tls_test.go diff --git a/k8s/vizier/bootstrap/adaptive_export_deployment.yaml b/k8s/vizier/bootstrap/adaptive_export_deployment.yaml index 3e6609594b5..251ceed69ce 100644 --- a/k8s/vizier/bootstrap/adaptive_export_deployment.yaml +++ b/k8s/vizier/bootstrap/adaptive_export_deployment.yaml @@ -49,6 +49,11 @@ spec: value: "kubescape_logs" - name: EXPORT_MODE value: "never" + # Control surface is secure-by-default (#96): TLS + bearer-JWT auth are ON + # out of the box. The service-tls-certs keypair mounted at /certs below is + # used for TLS (else AE self-generates an ephemeral in-memory cert), and + # PL_JWT_SIGNING_KEY above turns on auth. CONTROL_TLS / CONTROL_REQUIRE_AUTH + # are deprecated no-ops; set CONTROL_INSECURE=true only to opt out (dev). - name: CONTROL_ADDR value: ":9100" - name: ADAPTIVE_PUSH_PIXIE_ROWS diff --git a/k8s/vizier/dx/dx-daemon.yaml b/k8s/vizier/dx/dx-daemon.yaml index 8b4eadf1bbc..ea6479af766 100644 --- a/k8s/vizier/dx/dx-daemon.yaml +++ b/k8s/vizier/dx/dx-daemon.yaml @@ -30,7 +30,9 @@ spec: - { name: NODE_NAME, valueFrom: { fieldRef: { fieldPath: spec.nodeName } } } - { name: HOST_IP, valueFrom: { fieldRef: { fieldPath: status.hostIP } } } - { name: DX_RECEIVER_TLS, value: "1" } - - { name: AE_CONTROL_ADDR, value: "http://adaptive-export-control.pl.svc.cluster.local:9100" } + # AE control surface is TLS-by-default (#96); the dx client TLS-skip-verifies + # the in-cluster (self-signed/shared) cert and attaches its bearer JWT. + - { name: AE_CONTROL_ADDR, value: "https://adaptive-export-control.pl.svc.cluster.local:9100" } - { name: PX_API_KEY, valueFrom: { secretKeyRef: { name: dx-pixie-auth, key: api-key, optional: true } } } - { name: PX_CLUSTER_ID, valueFrom: { secretKeyRef: { name: dx-pixie-auth, key: cluster-id, optional: true } } } - { name: PX_CLOUD_ADDR, valueFrom: { secretKeyRef: { name: dx-pixie-auth, key: cloud-addr, optional: true } } } diff --git a/src/vizier/services/adaptive_export/cmd/main.go b/src/vizier/services/adaptive_export/cmd/main.go index 1a33a0cae90..38e46e3260e 100644 --- a/src/vizier/services/adaptive_export/cmd/main.go +++ b/src/vizier/services/adaptive_export/cmd/main.go @@ -721,63 +721,105 @@ func main() { // control surface: when CONTROL_ADDR is set, the per-node controller // steers this AE's activeSet (Upsert/Remove) over HTTP. Off by default so // the existing trigger→controller→activeSet flow is unchanged. + // + // Secure-by-default (#96): the control surface serves TLS and requires a + // bearer JWT out of the box. dx skip-verifies the in-cluster (self-signed) + // cert and attaches the service JWT it already mints. The ONLY way to run + // plaintext / no-auth is an explicit CONTROL_INSECURE=true (loud warning); + // without a signing key AND without CONTROL_INSECURE the control surface + // fails closed and does not start (everything else keeps running). + // + // CONTROL_TLS / CONTROL_REQUIRE_AUTH are deprecated no-ops (secure is the + // default now); they are honored for back-compat but no longer required. if addr := os.Getenv("CONTROL_ADDR"); addr != "" { - // Wire the controller as the /query runner: dx OrderQuery → one-shot pixie - // capture written to forensic_db (write⊇read; entlein/dx#93). When the - // operator-side querier is disabled (no PushPixieTables), OrderQuery returns - // an error and /query 502s — start/stop + dx_evidence_graph still work. - ctrlSrv := control.New(activeSet, ctl) - ctrlSrv.SetGraphWriter(applier) // dx_evidence_graph ingest → ClickHouse - ctrlSrv.SetManifestWriter(applier) // dx_evidence_manifest ingest → ClickHouse - // Bearer-JWT auth on the control surface (CodeRabbit: protect control - // endpoints). Same shared lib + signing key the broker/PEM use — dx - // attaches the service JWT it already mints. Default-OFF so this can - // merge before dx sends the bearer; flip CONTROL_REQUIRE_AUTH=true once - // dx is updated + PL_JWT_SIGNING_KEY is mounted. Safe incremental rollout. - if key := os.Getenv("PL_JWT_SIGNING_KEY"); key != "" && os.Getenv("CONTROL_REQUIRE_AUTH") == "true" { - ctrlSrv.SetAuth(key, "vizier") - log.Info("control surface: bearer-JWT auth ENABLED (audience=vizier)") - } else { - log.Warn("control surface: auth DISABLED (set CONTROL_REQUIRE_AUTH=true + PL_JWT_SIGNING_KEY)") + insecure := strings.EqualFold(os.Getenv("CONTROL_INSECURE"), "true") + signingKey := os.Getenv("PL_JWT_SIGNING_KEY") + + if _, ok := os.LookupEnv("CONTROL_TLS"); ok { + log.Warn("CONTROL_TLS is deprecated (TLS is default-ON; use CONTROL_INSECURE=true to opt out)") } - // Wrap in an http.Server with explicit timeouts so a slow client - // can't pin a goroutine on the control surface (CodeRabbit - // r3379377432). The control plane is small/idempotent JSON, so - // short read/write budgets are fine. - httpSrv := &http.Server{ - Addr: addr, - Handler: ctrlSrv.Handler(), - ReadHeaderTimeout: 5 * time.Second, - ReadTimeout: 15 * time.Second, - WriteTimeout: 30 * time.Second, - IdleTimeout: 60 * time.Second, + if _, ok := os.LookupEnv("CONTROL_REQUIRE_AUTH"); ok { + log.Warn("CONTROL_REQUIRE_AUTH is deprecated (auth is default-ON when PL_JWT_SIGNING_KEY is set)") } - go func() { - log.WithField("addr", addr).Info("control surface listening") - // CONTROL_TLS=true → serve TLS so the bearer JWT + control payloads - // don't cross the CNI in cleartext (auth without TLS leaks the token). - // Cert/key from the service-tls-certs secret the broker/PEM already use - // (mounted /certs); dx skip-verifies. Default-OFF for incremental rollout. - var err error - if os.Getenv("CONTROL_TLS") == "true" { - cert := os.Getenv("CONTROL_TLS_CERT") - if cert == "" { - cert = "/certs/server.crt" - } - key := os.Getenv("CONTROL_TLS_KEY") - if key == "" { - key = "/certs/server.key" - } - log.WithField("cert", cert).Info("control surface: TLS ENABLED") - err = httpSrv.ListenAndServeTLS(cert, key) + + switch { + case signingKey == "" && !insecure: + // Fail-closed: refuse to expose an unauthenticated control surface + // silently. The operator's export/attribution paths keep running; + // only this HTTP surface is withheld. + log.Error("control surface: REFUSING to start — no PL_JWT_SIGNING_KEY and CONTROL_INSECURE not set. " + + "Set PL_JWT_SIGNING_KEY to enable bearer-JWT auth (recommended), or CONTROL_INSECURE=true to run " + + "plaintext without auth (NOT for production).") + default: + // Wire the controller as the /query runner: dx OrderQuery → one-shot pixie + // capture written to forensic_db (write⊇read; entlein/dx#93). When the + // operator-side querier is disabled (no PushPixieTables), OrderQuery returns + // an error and /query 502s — start/stop + dx_evidence_graph still work. + ctrlSrv := control.New(activeSet, ctl) + ctrlSrv.SetGraphWriter(applier) // dx_evidence_graph ingest → ClickHouse + ctrlSrv.SetManifestWriter(applier) // dx_evidence_manifest ingest → ClickHouse + // Bearer-JWT auth default-ON whenever a signing key is present. Same + // shared lib + signing key the broker/PEM use — dx attaches the service + // JWT it already mints. No key is only reachable with CONTROL_INSECURE. + if signingKey != "" { + ctrlSrv.SetAuth(signingKey, "vizier") + log.Info("control surface: bearer-JWT auth ENABLED (audience=vizier)") } else { - log.Warn("control surface: TLS DISABLED — bearer JWT crosses the CNI in cleartext (set CONTROL_TLS=true)") - err = httpSrv.ListenAndServe() + log.Warn("control surface: auth DISABLED — no PL_JWT_SIGNING_KEY (CONTROL_INSECURE=true set)") } - if err != nil && err != http.ErrServerClosed { - log.WithError(err).Error("control surface stopped") + // Wrap in an http.Server with explicit timeouts so a slow client + // can't pin a goroutine on the control surface (CodeRabbit + // r3379377432). The control plane is small/idempotent JSON, so + // short read/write budgets are fine. + httpSrv := &http.Server{ + Addr: addr, + Handler: ctrlSrv.Handler(), + ReadHeaderTimeout: 5 * time.Second, + ReadTimeout: 15 * time.Second, + WriteTimeout: 30 * time.Second, + IdleTimeout: 60 * time.Second, } - }() + go func() { + log.WithField("addr", addr).Info("control surface listening") + var err error + if insecure { + // Explicit opt-out only. The bearer JWT + control payloads + // then cross the CNI in cleartext — never the silent default. + log.Warn("control surface: INSECURE (plaintext) — CONTROL_INSECURE set; " + + "bearer JWT + control payloads cross the CNI in cleartext") + err = httpSrv.ListenAndServe() + } else { + // TLS default-ON. Mounted keypair wins (service-tls-certs the + // broker/PEM already carry, /certs/server.{crt,key}); else an + // ephemeral in-memory self-signed cert so TLS works with zero + // extra secrets (dx skip-verifies). + certFile := os.Getenv("CONTROL_TLS_CERT") + if certFile == "" { + certFile = "/certs/server.crt" + } + keyFile := os.Getenv("CONTROL_TLS_KEY") + if keyFile == "" { + keyFile = "/certs/server.key" + } + tlsCfg, selfSigned, terr := control.TLSConfig(certFile, keyFile, hostname) + if terr != nil { + log.WithError(terr).Error("control surface: TLS setup failed — control surface not started") + return + } + httpSrv.TLSConfig = tlsCfg + if selfSigned { + log.Info("control surface: TLS ENABLED (self-signed)") + } else { + log.WithField("cert", certFile).Info("control surface: TLS ENABLED (mounted cert)") + } + // Certs are already in TLSConfig → empty file args. + err = httpSrv.ListenAndServeTLS("", "") + } + if err != nil && err != http.ErrServerClosed { + log.WithError(err).Error("control surface stopped") + } + }() + } } sigCh := make(chan os.Signal, 1) diff --git a/src/vizier/services/adaptive_export/internal/control/BUILD.bazel b/src/vizier/services/adaptive_export/internal/control/BUILD.bazel index c22b1b8ba71..3b39c27bef6 100644 --- a/src/vizier/services/adaptive_export/internal/control/BUILD.bazel +++ b/src/vizier/services/adaptive_export/internal/control/BUILD.bazel @@ -19,7 +19,10 @@ load("//bazel:pl_build_system.bzl", "pl_go_test") go_library( name = "control", - srcs = ["server.go"], + srcs = [ + "server.go", + "tls.go", + ], importpath = "px.dev/pixie/src/vizier/services/adaptive_export/internal/control", visibility = ["//src/vizier/services/adaptive_export:__subpackages__"], deps = [ @@ -31,7 +34,10 @@ go_library( pl_go_test( name = "control_test", - srcs = ["server_test.go"], + srcs = [ + "server_test.go", + "tls_test.go", + ], embed = [":control"], deps = [ "//src/shared/services/utils", diff --git a/src/vizier/services/adaptive_export/internal/control/tls.go b/src/vizier/services/adaptive_export/internal/control/tls.go new file mode 100644 index 00000000000..3e994e3bf7b --- /dev/null +++ b/src/vizier/services/adaptive_export/internal/control/tls.go @@ -0,0 +1,128 @@ +// Copyright 2018- The Pixie Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package control + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "math/big" + "net" + "os" + "time" +) + +// TLSConfig builds the server-side *tls.Config for the control surface. +// +// If BOTH certFile and keyFile exist and load, the mounted keypair is used +// (the shared service-tls-certs the broker/PEM already carry). Otherwise an +// ephemeral in-memory self-signed cert is generated so TLS works with zero +// extra secrets — dx skip-verifies the in-cluster cert, so a self-signed cert +// is sufficient to stop the bearer JWT crossing the CNI in cleartext. +// +// The bool return reports whether the cert was self-generated (true) vs +// loaded from disk (false), for the caller's boot log. +func TLSConfig(certFile, keyFile string, hostnames ...string) (*tls.Config, bool, error) { + if certFile != "" && keyFile != "" && fileExists(certFile) && fileExists(keyFile) { + cert, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return nil, false, fmt.Errorf("load mounted keypair %s/%s: %w", certFile, keyFile, err) + } + return &tls.Config{Certificates: []tls.Certificate{cert}, MinVersion: tls.VersionTLS12}, false, nil + } + cert, err := selfSignedCert(hostnames...) + if err != nil { + return nil, false, err + } + return &tls.Config{Certificates: []tls.Certificate{cert}, MinVersion: tls.VersionTLS12}, true, nil +} + +func fileExists(p string) bool { + fi, err := os.Stat(p) + return err == nil && !fi.IsDir() +} + +// selfSignedCert mints an ephemeral in-memory self-signed certificate: +// ECDSA P-256, 1y validity, SAN covering localhost + 127.0.0.1 + ::1 and any +// extra hostnames (the pod/node name). Nothing is written to disk; the key +// lives only in the returned tls.Certificate. +func selfSignedCert(hostnames ...string) (tls.Certificate, error) { + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return tls.Certificate{}, fmt.Errorf("generate ecdsa key: %w", err) + } + serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + if err != nil { + return tls.Certificate{}, fmt.Errorf("generate serial: %w", err) + } + now := time.Now() + tmpl := x509.Certificate{ + SerialNumber: serial, + Subject: pkix.Name{CommonName: "adaptive-export-control"}, + NotBefore: now.Add(-time.Hour), + NotAfter: now.AddDate(1, 0, 0), // 1y validity + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + DNSNames: []string{"localhost"}, + IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback}, + } + for _, h := range hostnames { + if h == "" { + continue + } + if ip := net.ParseIP(h); ip != nil { + tmpl.IPAddresses = append(tmpl.IPAddresses, ip) + } else { + tmpl.DNSNames = append(tmpl.DNSNames, h) + } + } + der, err := x509.CreateCertificate(rand.Reader, &tmpl, &tmpl, &key.PublicKey, key) + if err != nil { + return tls.Certificate{}, fmt.Errorf("create certificate: %w", err) + } + return tls.Certificate{ + Certificate: [][]byte{der}, + PrivateKey: key, + Leaf: &tmpl, + }, nil +} + +// certToPEM renders a tls.Certificate (as produced by selfSignedCert, holding a +// single DER cert + an *ecdsa.PrivateKey) as PEM cert + PEM key bytes — the +// on-disk shape of a mounted /certs/server.{crt,key} keypair. +func certToPEM(cert tls.Certificate) (certPEM, keyPEM []byte, err error) { + if len(cert.Certificate) == 0 { + return nil, nil, fmt.Errorf("certToPEM: empty certificate chain") + } + certPEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: cert.Certificate[0]}) + ec, ok := cert.PrivateKey.(*ecdsa.PrivateKey) + if !ok { + return nil, nil, fmt.Errorf("certToPEM: private key is not *ecdsa.PrivateKey") + } + der, err := x509.MarshalECPrivateKey(ec) + if err != nil { + return nil, nil, fmt.Errorf("marshal ec private key: %w", err) + } + keyPEM = pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: der}) + return certPEM, keyPEM, nil +} diff --git a/src/vizier/services/adaptive_export/internal/control/tls_test.go b/src/vizier/services/adaptive_export/internal/control/tls_test.go new file mode 100644 index 00000000000..e6f53680c83 --- /dev/null +++ b/src/vizier/services/adaptive_export/internal/control/tls_test.go @@ -0,0 +1,194 @@ +// Copyright 2018- The Pixie Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package control + +import ( + "crypto/tls" + "net" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + "time" + + jwtutils "px.dev/pixie/src/shared/services/utils" +) + +// serveTLS starts the control server over TLS on 127.0.0.1:0 using the given +// *tls.Config and returns the base https URL + a shutdown func. +func serveTLS(t *testing.T, cfg *tls.Config, srv *Server) (string, func()) { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + httpSrv := &http.Server{Handler: srv.Handler(), TLSConfig: cfg} + go func() { _ = httpSrv.ServeTLS(ln, "", "") }() + return "https://" + ln.Addr().String(), func() { _ = httpSrv.Close() } +} + +func skipVerifyClient() *http.Client { + return &http.Client{ + Timeout: 3 * time.Second, + Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}, //nolint:gosec // test: dx skip-verifies the in-cluster self-signed cert + } +} + +// TestTLSConfigSelfSigned: with no mounted cert files, TLSConfig self-generates +// an in-memory cert (the default secure path when /certs is absent). +func TestTLSConfigSelfSigned(t *testing.T) { + cfg, selfSigned, err := TLSConfig("/no/such/cert.crt", "/no/such/key.key", "some-pod") + if err != nil { + t.Fatalf("TLSConfig self-gen: %v", err) + } + if !selfSigned { + t.Fatal("expected selfSigned=true when cert files are absent") + } + if cfg == nil || len(cfg.Certificates) != 1 { + t.Fatalf("expected exactly one in-memory certificate, got %+v", cfg) + } +} + +// TestTLSServesHealthz: the server serves TLS by default (self-gen path) and a +// TLS client can reach /healthz. This is T1's "no cleartext by default". +func TestTLSServesHealthz(t *testing.T) { + cfg, selfSigned, err := TLSConfig("", "", "localhost") + if err != nil { + t.Fatalf("TLSConfig: %v", err) + } + if !selfSigned { + t.Fatal("expected self-signed cert") + } + base, stop := serveTLS(t, cfg, New(&fakeExporter{}, nil)) + defer stop() + + resp, err := skipVerifyClient().Get(base + "/healthz") + if err != nil { + t.Fatalf("TLS GET /healthz: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("healthz over TLS = %d, want 200", resp.StatusCode) + } +} + +// TestTLSRejectsUnauthenticated: over TLS with a signing key configured, an +// unauthenticated control request is rejected (401). This is T1's "requires +// the bearer JWT when a signing key is present" — verified end-to-end on the +// real TLS listener, not just the handler. +func TestTLSRejectsUnauthenticated(t *testing.T) { + const key = "0123456789abcdef0123456789abcdef" + srv := New(&fakeExporter{}, nil) + srv.SetAuth(key, "vizier") + + cfg, _, err := TLSConfig("", "", "localhost") + if err != nil { + t.Fatalf("TLSConfig: %v", err) + } + base, stop := serveTLS(t, cfg, srv) + defer stop() + client := skipVerifyClient() + + // No bearer → 401. + resp, err := client.Post(base+"/export/start", "application/json", strings.NewReader(`{"pod":"p","t_end":1}`)) + if err != nil { + t.Fatalf("TLS POST: %v", err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusUnauthorized { + t.Fatalf("unauthenticated over TLS = %d, want 401", resp.StatusCode) + } + + // Valid bearer → not 401. + good, err := jwtutils.SignJWTClaims(jwtutils.GenerateJWTForService("dx", "vizier"), key) + if err != nil { + t.Fatalf("mint token: %v", err) + } + req, _ := http.NewRequest(http.MethodPost, base+"/export/start", strings.NewReader(`{"namespace":"n","pod":"p","t_end":1}`)) + req.Header.Set("Authorization", "Bearer "+good) + resp2, err := client.Do(req) + if err != nil { + t.Fatalf("TLS POST authed: %v", err) + } + resp2.Body.Close() + if resp2.StatusCode == http.StatusUnauthorized { + t.Fatal("valid bearer wrongly rejected over TLS") + } +} + +// TestTLSConfigMountedCert: when cert+key files exist, TLSConfig loads them +// (selfSigned=false) — the /certs/server.{crt,key} shared-cert path. +func TestTLSConfigMountedCert(t *testing.T) { + dir := t.TempDir() + certPath := filepath.Join(dir, "server.crt") + keyPath := filepath.Join(dir, "server.key") + writePEMKeypair(t, certPath, keyPath) + + cfg, selfSigned, err := TLSConfig(certPath, keyPath, "localhost") + if err != nil { + t.Fatalf("TLSConfig mounted: %v", err) + } + if selfSigned { + t.Fatal("expected selfSigned=false when cert files exist") + } + if cfg == nil || len(cfg.Certificates) != 1 { + t.Fatalf("expected one loaded certificate, got %+v", cfg) + } +} + +// TestPlaintextPathServes: the CONTROL_INSECURE opt-out serves plain HTTP. This +// mirrors main.go's insecure branch (httpSrv.ListenAndServe with the same +// handler) — a plaintext client reaches /healthz. +func TestPlaintextPathServes(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + httpSrv := &http.Server{Handler: New(&fakeExporter{}, nil).Handler()} + go func() { _ = httpSrv.Serve(ln) }() + defer httpSrv.Close() + + resp, err := (&http.Client{Timeout: 3 * time.Second}).Get("http://" + ln.Addr().String() + "/healthz") + if err != nil { + t.Fatalf("plaintext GET /healthz: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("plaintext healthz = %d, want 200", resp.StatusCode) + } +} + +// writePEMKeypair mints a self-signed cert via the same helper and writes it as +// PEM cert+key files, so the mounted-cert load path can be exercised. +func writePEMKeypair(t *testing.T, certPath, keyPath string) { + t.Helper() + cert, err := selfSignedCert("localhost") + if err != nil { + t.Fatalf("selfSignedCert: %v", err) + } + certPEM, keyPEM, err := certToPEM(cert) + if err != nil { + t.Fatalf("certToPEM: %v", err) + } + if err := os.WriteFile(certPath, certPEM, 0o600); err != nil { + t.Fatalf("write cert: %v", err) + } + if err := os.WriteFile(keyPath, keyPEM, 0o600); err != nil { + t.Fatalf("write key: %v", err) + } +} From 7f4ef7d51e9554d2e2fcf9bcb63aa1eb6ce2c570 Mon Sep 17 00:00:00 2001 From: entlein Date: Sun, 16 Aug 2026 14:05:19 +0200 Subject: [PATCH 19/66] ci: release manifest robust to lightweight tags (fix jq 'timestamp: ,' error) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit %(taggerdate:raw) is empty for a lightweight release tag → create_manifest_update emits 'timestamp: ,' → jq syntax error → the vizier release-metadata step fails even though the image built + pushed. Fall back to the tagged commit's committer date. --- ci/artifact_utils.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/ci/artifact_utils.sh b/ci/artifact_utils.sh index a1eec1a7760..e6d7c0dca26 100644 --- a/ci/artifact_utils.sh +++ b/ci/artifact_utils.sh @@ -107,7 +107,15 @@ create_manifest_update() { tag_name="release/${component}/v${version}" # actions/checkout doesn't get the tag annotation properly. git fetch origin tag "${tag_name}" -f - timestamp="$(git tag -l --format "%(taggerdate:raw)" "${tag_name}" | awk '{print $1}' | jq '. | todate')" + # taggerdate is empty for a LIGHTWEIGHT tag → produces `timestamp: ,` → jq syntax + # error → release-metadata step fails even though the image built fine. Fall back to + # the tagged commit's committer date so the manifest is well-formed regardless of how + # the release tag was cut (annotated vs lightweight). + raw_ts="$(git tag -l --format "%(taggerdate:raw)" "${tag_name}" | awk '{print $1}')" + if [ -z "${raw_ts}" ]; then + raw_ts="$(git log -1 --format="%ct" "${tag_name}")" + fi + timestamp="$(printf '%s' "${raw_ts}" | jq '. | todate')" jq -s \ "[{name: \"${component}\", artifact: [{timestamp: ${timestamp}, commitHash: \"${commit_hash}\", versionStr: \"${version}\", availableArtifactMirrors: .}]}]" \ From 417b7a6a57a111d1eb5dbbc1f1cd4522e1a7f7cf Mon Sep 17 00:00:00 2001 From: entlein Date: Sun, 16 Aug 2026 14:26:51 +0200 Subject: [PATCH 20/66] AE skaffold: aeprod63 -> aeprod64 (F8 watermark #97 + control secure-by-default #96) --- k8s/vizier/adaptive_export/kustomization.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/vizier/adaptive_export/kustomization.yaml b/k8s/vizier/adaptive_export/kustomization.yaml index 186a589c5b0..b55c10dbf0e 100644 --- a/k8s/vizier/adaptive_export/kustomization.yaml +++ b/k8s/vizier/adaptive_export/kustomization.yaml @@ -7,4 +7,4 @@ resources: images: - name: vizier-adaptive_export_image newName: ghcr.io/k8sstormcenter/vizier-adaptive_export_image - newTag: 0.14.19-aeprod63 + newTag: 0.14.19-aeprod64 From 573e630fdb7838261733c1b8e61c34a943d17247 Mon Sep 17 00:00:00 2001 From: entlein Date: Sun, 16 Aug 2026 15:42:39 +0200 Subject: [PATCH 21/66] pxl_scripts: dx/evidence_graph 3-level-zoom Live View bundle Standalone GraphWidget bundle (no src/ui changes) rendering the dx evidence graph with drill-down: graph edges -> investigation manifest -> consulted raw forensic rows. Reads forensic_db in ClickHouse via px.DataFrame(clickhouse_dsn). - evidence_graph: severity-weighted pod->pod edges; px.Pod() stamps ST_POD_NAME so nodes deep-link to px/pod via the widget's built-in deepLinkURLFromSemanticType. - investigation_detail: manifest row(s), case_window bounds via px.pluck_int64. - consulted_rows: demo.md 'H3 dc_snoop reconstruction for the alert pod. - vis.json: Graph over evidence_graph (edgeWeightColumn=confidence, edgeColorColumn=max_severity, edgeHoverInfo=investigation_id/condition/criteria/ edge_kind), plus manifest + consulted-rows Table widgets; investigation_id var is the zoom. - README: 3-level zoom, load-into-UI steps, clickhouse_dsn feasibility (YES) + templated-read / hostname-partition / ns-start_time caveats. Static-validated only; needs live-UI validation on a cluster carrying forensic_db. --- src/pxl_scripts/dx/evidence_graph/README.md | 107 ++++++++++++++ .../dx/evidence_graph/evidence_graph.pxl | 103 +++++++++++++ .../dx/evidence_graph/manifest.yaml | 9 ++ src/pxl_scripts/dx/evidence_graph/vis.json | 138 ++++++++++++++++++ 4 files changed, 357 insertions(+) create mode 100644 src/pxl_scripts/dx/evidence_graph/README.md create mode 100644 src/pxl_scripts/dx/evidence_graph/evidence_graph.pxl create mode 100644 src/pxl_scripts/dx/evidence_graph/manifest.yaml create mode 100644 src/pxl_scripts/dx/evidence_graph/vis.json diff --git a/src/pxl_scripts/dx/evidence_graph/README.md b/src/pxl_scripts/dx/evidence_graph/README.md new file mode 100644 index 00000000000..29a3e21380b --- /dev/null +++ b/src/pxl_scripts/dx/evidence_graph/README.md @@ -0,0 +1,107 @@ +# DX Evidence Graph — 3-level zoom (`dx/evidence_graph`) + +A **standalone** Pixie Live View bundle (PxL + `vis.json`, no Pixie UI source +changes) that renders the dx evidence graph in Pixie's existing `GraphWidget` and +lets an analyst zoom from an investigation down to the individual forensic rows dx +consulted. + +## What it shows — the three levels + +| Level | Widget | PxL func | Content | +|------|--------|----------|---------| +| 1 | Graph | `evidence_graph` | Severity-weighted, all-protocol **pod → pod** edge list for the malignant (ruled-in) investigations. Edge weight = `confidence`, colour = `max_severity`, label = `edge_kind`, hover = investigation_id / condition / criteria / num_findings. | +| 2 | Table | `investigation_detail` | The **manifest** row(s) for the zoomed investigation: `verdict`, `condition`, `confidence`/`posterior`, case-window bounds (`win_lo`/`win_hi`, plucked from the `case_window` JSON), `evidence_hash`, raw `findings`. | +| 3 | Table | `consulted_rows` | The **§H reconstruction**: the raw `dc_snoop` (default) process rows for the alert pod in the window — the individual rows dx consulted. Repoint `raw_table` at `redis_events` / `kubescape_logs` for the other §H tables. | + +## The drill-down model + +- **Pod-node double-click → `px/pod` (built-in, no code change).** `evidence_graph` + stamps the `from_entity`/`to_entity` node columns with the pod semantic type via + `px.Pod(...)` (registered as a `STRING → ST_POD_NAME` cast in + `src/carnot/planner/objects/pixie_module.cc:560`). The GraphWidget's built-in + `doubleClickCallback` → `deepLinkURLFromSemanticType` (`graph.tsx` ~line 170) then + deep-links any `ST_POD_NAME` node to `px/pod`. We rely on that path; nothing under + `src/ui` is modified. Caveat: the column is stamped pod-typed even when an endpoint + resolves to a service/IP (pod > service > ip fallback), so a non-pod node double-click + deep-links to `px/pod?pod=` — harmless, and in the demo every endpoint is a pod. +- **Investigation zoom = the `investigation_id` script var.** Copy an `investigation_id` + from a graph-edge hover into the `investigation_id` variable (and set `pod_filter` to + the alert pod). Levels 2 and 3 re-run scoped to that investigation. `investigation_filter` + independently narrows the graph itself. + +## How to load it into a running Pixie UI (no rebuild) + +This is a self-contained scripts bundle — deploy it without touching the UI: + +1. **Custom Live View (fastest).** In the Live UI, open the script editor (the + `` **Scratch Pad** / "Edit script" pane), paste `evidence_graph.pxl` into the + **PxL** tab and `vis.json` into the **Vis Spec** tab, set the variables + (at minimum `clickhouse_dsn`), and Run. +2. **Bundled script.** The directory (`evidence_graph.pxl` + `vis.json` + + `manifest.yaml`) is globbed into `bundle-oss.json` by + `src/pxl_scripts/BUILD.bazel` (the `**/*.pxl|json|yaml` filegroup), so it ships as + the script id **`dx/evidence_graph`** wherever that bundle is served. No registry + edit is required. +3. **`px` CLI.** `px run -f evidence_graph.pxl` (table output) for a non-UI smoke test. + +Set the `clickhouse_dsn` variable to your forensic_db DSN (default: +`ingest_writer:changeme-ingest@clickhouse-forensic-soc-db.clickhouse.svc.cluster.local:9000/forensic_db`). + +## Feasibility: can a PxL Live View read ClickHouse via `clickhouse_dsn`? — **YES** + +Confirmed against the fork, not assumed: + +- `px.DataFrame(table, clickhouse_dsn=..., start_time=...)` is a first-class reader: + the arg is registered on the DataFrame op + (`src/carnot/planner/objects/dataframe.cc:189-197,558-559`) and executed by the PEM + through `ClickHouseSourceNode` (`src/carnot/exec/clickhouse_source_node.cc`). +- It is **already in production use** by the shipped `px/dx_evidence_graph` bundle + reading these exact tables, and `schema.sql` documents the tables as *"read by the + Pixie dx_evidence_graph UI via px.DataFrame(clickhouse_dsn=...)"*. +- The reader maps `String / Int8..64 / UInt8..64 / Float32/64 / DateTime / DateTime64` + → Pixie types (`clickhouse_source_node.cc:112-370`). Every column projected here is + in that set. + +So this bundle is built directly on `clickhouse_dsn`. **No alternative execution path +is needed.** + +### Real constraints (documented, not blockers) + +1. **Templated read, not arbitrary SQL.** The reader issues + `SELECT … FROM
WHERE >= … [AND hostname = ] ORDER BY LIMIT …`. + It **cannot** run the `JSONExtract*` / `ARRAY JOIN` SQL from demo.md §H. The bundle + therefore re-implements the reconstruction in PxL: JSON columns + (`case_window`, `findings`) are parsed with `px.pluck_int64` / `px.pluck`, and + row-scoping is done with PxL filters (`px.contains`) instead of a SQL join. +2. **`hostname` partition filter.** When a `hostname` column exists the reader appends + `AND hostname = ` (`clickhouse_source_node.cc:429-434`). + Rows are only visible from the PEM whose host wrote them — a multi-node caveat. + `dc_snoop` is node-scoped so this is expected; for the dx tables ensure the reading + PEM matches the writing host (the shipped `px/dx_evidence_graph` operates under the + same rule). +3. **`start_time` is a no-op on nanosecond `event_time` tables.** The reader converts + `start_time` ns→seconds (`clickhouse_source_node.cc:69-75`) and compares it to + `event_time`. For tables whose `event_time` is `UInt64` **nanoseconds** + (`dx_evidence_graph`, `dx_evidence_manifest`, `kubescape_logs`) the seconds-scale + threshold is always ≤ the nanosecond values, so **all** in-TTL rows return (no time + narrowing — bounded by the 30-day TTL + `LIMIT`). On `dc_snoop` (`DateTime64(9)`) + `start_time` filtering does apply. This is why Level 3's exact window is read from + the **manifest** (`win_lo`/`win_hi`, Level 2) rather than from `start_time`. + +## Needs live-UI validation on a cluster with the dx evidence tables + +Statically validated here: `vis.json` is valid JSON; every variable is referenced by +a `globalFunc`; every widget binds to a declared `globalFunc`; func names and per-arg +names match the PxL signatures; projected columns exist in `schema.sql`; the `px.Pod` +/ `px.pluck_int64` / `px.contains` builtins used all exist in the fork. + +Cannot be run against a live PEM from here — validate on a cluster carrying +`forensic_db` (e.g. a PG with the SOC stack + AE): + +- All three funcs **compile and execute** through the PEM's `clickhouse_dsn` reader. +- `px.Pod(...)` produces `ST_POD_NAME` nodes and a **double-click deep-links to `px/pod`**. +- `px.pluck_int64(case_window, 'lo'|'hi')` returns the correct window bounds (Level 2). +- Level 3 `raw_table` swaps (`redis_events`, `kubescape_logs`) project without a + column-name error (their schemas differ from `dc_snoop`; adjust the projection if so). +- Graph rendering: `edgeColorColumn`/`edgeThresholds` colour by `max_severity`, hover + shows the investigation fields. diff --git a/src/pxl_scripts/dx/evidence_graph/evidence_graph.pxl b/src/pxl_scripts/dx/evidence_graph/evidence_graph.pxl new file mode 100644 index 00000000000..0055e3314d5 --- /dev/null +++ b/src/pxl_scripts/dx/evidence_graph/evidence_graph.pxl @@ -0,0 +1,103 @@ +# Copyright 2018- The Pixie Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 + +''' DX Evidence Graph (3-level zoom) + +Renders the dx evidence graph in Pixie's built-in GraphWidget and lets an analyst +zoom from the whole investigation set -> one investigation's manifest -> the raw +forensic rows dx consulted. + +The evidence tables live in ClickHouse (forensic_db), not in Pixie's socket_tracer. +They are read through the fork's `px.DataFrame(table, clickhouse_dsn=...)` reader +(src/carnot/exec/clickhouse_source_node.cc), which the PEM executes like any other +source. Every table read here carries the required `event_time` + `hostname` +columns, so the reader's templated WHERE/ORDER-BY works. +''' +import px + +# Level 1 ----------------------------------------------------------------------- +# The severity-weighted, all-protocol pod->pod edge list for the malignant +# (ruled-in) investigations. requestor/responder are stamped with the pod +# semantic type (px.Pod) so a double-click on a node deep-links to px/pod via the +# GraphWidget's built-in deepLinkURLFromSemanticType -- NO Pixie UI change. + + +def evidence_graph(start_time: str, clickhouse_dsn: str, table: str, investigation_filter: str): + df = px.DataFrame(table, clickhouse_dsn=clickhouse_dsn, start_time=start_time) + + # Best display name per endpoint: pod > service > ip (matches px/dx_evidence_graph). + df.requestor = px.select(df.requestor_pod == '', + px.select(df.requestor_service == '', df.requestor_ip, df.requestor_service), + df.requestor_pod) + df.responder = px.select(df.responder_pod == '', + px.select(df.responder_service == '', df.responder_ip, df.responder_service), + df.responder_pod) + + # Stamp ST_POD_NAME so graph nodes are drill-able to px/pod out of the box. + df.from_entity = px.Pod(df.requestor) + df.to_entity = px.Pod(df.responder) + + # Zoom: keep only the selected investigation. Empty filter = all malignant edges. + df = df[px.contains(df.investigation_id, investigation_filter)] + + return df[['from_entity', 'to_entity', + 'confidence', 'max_severity', 'weight', + 'condition', 'edge_kind', 'criteria', 'num_findings', + 'investigation_id']] + + +# Level 2 ----------------------------------------------------------------------- +# The manifest row(s) for the selected investigation: verdict, case-window +# bounds, evidence_hash, and the raw findings JSON (the completeness contract). +# case_window is a JSON string column ({"lo":,"hi":}); pluck the bounds. + + +def investigation_detail(start_time: str, clickhouse_dsn: str, manifest_table: str, investigation_id: str): + df = px.DataFrame(manifest_table, clickhouse_dsn=clickhouse_dsn, start_time=start_time) + + # Zoom filter. Empty investigation_id = every manifest in the window. + df = df[px.contains(df.investigation_id, investigation_id)] + + df.win_lo = px.pluck_int64(df.case_window, 'lo') + df.win_hi = px.pluck_int64(df.case_window, 'hi') + + return df[['investigation_id', 'verdict', 'condition', + 'confidence', 'posterior', 'catalog_version', + 'win_lo', 'win_hi', 'evidence_hash', 'findings']] + + +# Level 3 ----------------------------------------------------------------------- +# The raw forensic rows dx consulted for the selected investigation -- the §H +# reconstruction. Adapts demo.md §H3 (dc_snoop process rows for the alert pod +# within the case window). Because px.DataFrame(clickhouse_dsn) issues a +# templated per-table read (not arbitrary ClickHouse SQL), the case-window bound +# is approximated by the Live View `start_time` and the pod is selected with +# `pod_filter`; the exact case_window (win_lo/win_hi) is shown in Level 2. +# Point `raw_table` at dc_snoop (default), redis_events, or kubescape_logs to +# reconstruct the other §H tables (their projected columns differ -- see README). + + +def consulted_rows(start_time: str, clickhouse_dsn: str, raw_table: str, pod_filter: str, investigation_id: str): + df = px.DataFrame(raw_table, clickhouse_dsn=clickhouse_dsn, start_time=start_time) + + # Scope to the alert pod (the investigation's subject). Empty = all pods. + df = df[px.contains(df.pod, pod_filter)] + + # Carry the selected investigation for context / cross-reference. + df.investigation_id = investigation_id + + return df[['time_', 'namespace', 'pod', 'container', + 'comm', 'pid', 'file', 'investigation_id']] diff --git a/src/pxl_scripts/dx/evidence_graph/manifest.yaml b/src/pxl_scripts/dx/evidence_graph/manifest.yaml new file mode 100644 index 00000000000..d09d7f61d7e --- /dev/null +++ b/src/pxl_scripts/dx/evidence_graph/manifest.yaml @@ -0,0 +1,9 @@ +--- +short: DX Evidence Graph (3-level zoom) +long: > + Renders the dx evidence graph in Pixie's built-in GraphWidget and lets an analyst zoom + from the whole malignant investigation set into one investigation's manifest and the raw + forensic rows dx consulted. Level 1: severity-weighted pod-to-pod edges (nodes deep-link to + px/pod). Level 2: the manifest (verdict, case window, evidence_hash). Level 3: the §H + reconstruction of the consulted dc_snoop / redis_events / kubescape_logs rows. Reads + forensic_db in ClickHouse via px.DataFrame(clickhouse_dsn=...). diff --git a/src/pxl_scripts/dx/evidence_graph/vis.json b/src/pxl_scripts/dx/evidence_graph/vis.json new file mode 100644 index 00000000000..2db3923b641 --- /dev/null +++ b/src/pxl_scripts/dx/evidence_graph/vis.json @@ -0,0 +1,138 @@ +{ + "variables": [ + { + "name": "start_time", + "type": "PX_STRING", + "description": "Start time of the window (approximates the case window for Level 3).", + "defaultValue": "-15m" + }, + { + "name": "clickhouse_dsn", + "type": "PX_STRING", + "description": "ClickHouse DSN for forensic_db: user:pass@host:port/db.", + "defaultValue": "ingest_writer:changeme-ingest@clickhouse-forensic-soc-db.clickhouse.svc.cluster.local:9000/forensic_db" + }, + { + "name": "table", + "type": "PX_STRING", + "description": "Evidence-graph edge table (default: rule-ins-only malignant view).", + "defaultValue": "dx_evidence_graph_malignant" + }, + { + "name": "investigation_filter", + "type": "PX_STRING", + "description": "Substring to narrow the graph to one investigation_id. Empty = all malignant edges.", + "defaultValue": "" + }, + { + "name": "manifest_table", + "type": "PX_STRING", + "description": "Manifest table (verdict / case-window / evidence_hash).", + "defaultValue": "dx_evidence_manifest" + }, + { + "name": "investigation_id", + "type": "PX_STRING", + "description": "ZOOM: the investigation_id to expand into its manifest + consulted rows. Copy it from a graph edge hover. Empty = all.", + "defaultValue": "" + }, + { + "name": "raw_table", + "type": "PX_STRING", + "description": "Level-3 forensic table to reconstruct: dc_snoop (default), redis_events, or kubescape_logs.", + "defaultValue": "dc_snoop" + }, + { + "name": "pod_filter", + "type": "PX_POD", + "description": "ZOOM: the alert pod whose consulted rows to show (substring match). Empty = all pods in window.", + "defaultValue": "" + } + ], + "globalFuncs": [ + { + "outputName": "dx_graph", + "func": { + "name": "evidence_graph", + "args": [ + {"name": "start_time", "variable": "start_time"}, + {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, + {"name": "table", "variable": "table"}, + {"name": "investigation_filter", "variable": "investigation_filter"} + ] + } + }, + { + "outputName": "dx_manifest", + "func": { + "name": "investigation_detail", + "args": [ + {"name": "start_time", "variable": "start_time"}, + {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, + {"name": "manifest_table", "variable": "manifest_table"}, + {"name": "investigation_id", "variable": "investigation_id"} + ] + } + }, + { + "outputName": "dx_consulted", + "func": { + "name": "consulted_rows", + "args": [ + {"name": "start_time", "variable": "start_time"}, + {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, + {"name": "raw_table", "variable": "raw_table"}, + {"name": "pod_filter", "variable": "pod_filter"}, + {"name": "investigation_id", "variable": "investigation_id"} + ] + } + } + ], + "widgets": [ + { + "name": "Evidence Graph (double-click a pod node -> px/pod)", + "position": {"x": 0, "y": 0, "w": 12, "h": 5}, + "globalFuncOutputName": "dx_graph", + "displaySpec": { + "@type": "types.px.dev/px.vispb.Graph", + "adjacencyList": { + "fromColumn": "from_entity", + "toColumn": "to_entity" + }, + "edgeWeightColumn": "confidence", + "edgeColorColumn": "max_severity", + "edgeLabelColumn": "edge_kind", + "edgeThresholds": { + "mediumThreshold": 3, + "highThreshold": 4 + }, + "edgeHoverInfo": [ + "investigation_id", + "condition", + "criteria", + "edge_kind", + "confidence", + "max_severity", + "num_findings" + ], + "edgeLength": 500 + } + }, + { + "name": "Level 2 - Investigation Manifest (verdict / case-window / evidence_hash)", + "position": {"x": 0, "y": 5, "w": 12, "h": 3}, + "globalFuncOutputName": "dx_manifest", + "displaySpec": { + "@type": "types.px.dev/px.vispb.Table" + } + }, + { + "name": "Level 3 - Consulted Raw Rows (dc_snoop / redis_events / kubescape_logs)", + "position": {"x": 0, "y": 8, "w": 12, "h": 4}, + "globalFuncOutputName": "dx_consulted", + "displaySpec": { + "@type": "types.px.dev/px.vispb.Table" + } + } + ] +} From 465e4d923d92a3675f49f576e83b4fedcfec56f0 Mon Sep 17 00:00:00 2001 From: entlein Date: Sun, 16 Aug 2026 15:54:36 +0200 Subject: [PATCH 22/66] AE skaffold: aeprod64 -> aeprod65 (clean green release: #96 #97 + jq fix) --- k8s/vizier/adaptive_export/kustomization.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/vizier/adaptive_export/kustomization.yaml b/k8s/vizier/adaptive_export/kustomization.yaml index b55c10dbf0e..d5c054b445a 100644 --- a/k8s/vizier/adaptive_export/kustomization.yaml +++ b/k8s/vizier/adaptive_export/kustomization.yaml @@ -7,4 +7,4 @@ resources: images: - name: vizier-adaptive_export_image newName: ghcr.io/k8sstormcenter/vizier-adaptive_export_image - newTag: 0.14.19-aeprod64 + newTag: 0.14.19-aeprod65 From 0f5dbcc21585ad3b5788403e6a6e0c13edeabe00 Mon Sep 17 00:00:00 2001 From: entlein Date: Sun, 16 Aug 2026 16:11:32 +0200 Subject: [PATCH 23/66] pxl_scripts(px/dx_evidence_graph): fold in 3-level-zoom enhancements MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Enhance the existing widget in place (drop the parallel dx/evidence_graph bundle): - drill-able pod nodes (px.Pod -> ST_POD_NAME -> double-click deep-links to px/pod via the GraphWidget's built-in deepLinkURLFromSemanticType; NO src/ui change) - L2 investigation_detail: the manifest (verdict / case_window / evidence_hash / findings) - L3 consulted_rows: the raw forensic rows dx considered (demo.md §H reconstruction) - investigation_id vis variable = the zoom key; keeps the forensic_analyst read DSN. Needs live-UI validation on a cluster with populated dx_evidence_graph/manifest. --- .../dx/evidence_graph/evidence_graph.pxl | 103 ------------- .../dx/evidence_graph/manifest.yaml | 9 -- src/pxl_scripts/dx/evidence_graph/vis.json | 138 ------------------ .../dx_evidence_graph}/README.md | 4 +- .../dx_evidence_graph/dx_evidence_graph.pxl | 54 ++++++- src/pxl_scripts/px/dx_evidence_graph/vis.json | 99 +++++++++++-- 6 files changed, 139 insertions(+), 268 deletions(-) delete mode 100644 src/pxl_scripts/dx/evidence_graph/evidence_graph.pxl delete mode 100644 src/pxl_scripts/dx/evidence_graph/manifest.yaml delete mode 100644 src/pxl_scripts/dx/evidence_graph/vis.json rename src/pxl_scripts/{dx/evidence_graph => px/dx_evidence_graph}/README.md (97%) diff --git a/src/pxl_scripts/dx/evidence_graph/evidence_graph.pxl b/src/pxl_scripts/dx/evidence_graph/evidence_graph.pxl deleted file mode 100644 index 0055e3314d5..00000000000 --- a/src/pxl_scripts/dx/evidence_graph/evidence_graph.pxl +++ /dev/null @@ -1,103 +0,0 @@ -# Copyright 2018- The Pixie Authors. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# SPDX-License-Identifier: Apache-2.0 - -''' DX Evidence Graph (3-level zoom) - -Renders the dx evidence graph in Pixie's built-in GraphWidget and lets an analyst -zoom from the whole investigation set -> one investigation's manifest -> the raw -forensic rows dx consulted. - -The evidence tables live in ClickHouse (forensic_db), not in Pixie's socket_tracer. -They are read through the fork's `px.DataFrame(table, clickhouse_dsn=...)` reader -(src/carnot/exec/clickhouse_source_node.cc), which the PEM executes like any other -source. Every table read here carries the required `event_time` + `hostname` -columns, so the reader's templated WHERE/ORDER-BY works. -''' -import px - -# Level 1 ----------------------------------------------------------------------- -# The severity-weighted, all-protocol pod->pod edge list for the malignant -# (ruled-in) investigations. requestor/responder are stamped with the pod -# semantic type (px.Pod) so a double-click on a node deep-links to px/pod via the -# GraphWidget's built-in deepLinkURLFromSemanticType -- NO Pixie UI change. - - -def evidence_graph(start_time: str, clickhouse_dsn: str, table: str, investigation_filter: str): - df = px.DataFrame(table, clickhouse_dsn=clickhouse_dsn, start_time=start_time) - - # Best display name per endpoint: pod > service > ip (matches px/dx_evidence_graph). - df.requestor = px.select(df.requestor_pod == '', - px.select(df.requestor_service == '', df.requestor_ip, df.requestor_service), - df.requestor_pod) - df.responder = px.select(df.responder_pod == '', - px.select(df.responder_service == '', df.responder_ip, df.responder_service), - df.responder_pod) - - # Stamp ST_POD_NAME so graph nodes are drill-able to px/pod out of the box. - df.from_entity = px.Pod(df.requestor) - df.to_entity = px.Pod(df.responder) - - # Zoom: keep only the selected investigation. Empty filter = all malignant edges. - df = df[px.contains(df.investigation_id, investigation_filter)] - - return df[['from_entity', 'to_entity', - 'confidence', 'max_severity', 'weight', - 'condition', 'edge_kind', 'criteria', 'num_findings', - 'investigation_id']] - - -# Level 2 ----------------------------------------------------------------------- -# The manifest row(s) for the selected investigation: verdict, case-window -# bounds, evidence_hash, and the raw findings JSON (the completeness contract). -# case_window is a JSON string column ({"lo":,"hi":}); pluck the bounds. - - -def investigation_detail(start_time: str, clickhouse_dsn: str, manifest_table: str, investigation_id: str): - df = px.DataFrame(manifest_table, clickhouse_dsn=clickhouse_dsn, start_time=start_time) - - # Zoom filter. Empty investigation_id = every manifest in the window. - df = df[px.contains(df.investigation_id, investigation_id)] - - df.win_lo = px.pluck_int64(df.case_window, 'lo') - df.win_hi = px.pluck_int64(df.case_window, 'hi') - - return df[['investigation_id', 'verdict', 'condition', - 'confidence', 'posterior', 'catalog_version', - 'win_lo', 'win_hi', 'evidence_hash', 'findings']] - - -# Level 3 ----------------------------------------------------------------------- -# The raw forensic rows dx consulted for the selected investigation -- the §H -# reconstruction. Adapts demo.md §H3 (dc_snoop process rows for the alert pod -# within the case window). Because px.DataFrame(clickhouse_dsn) issues a -# templated per-table read (not arbitrary ClickHouse SQL), the case-window bound -# is approximated by the Live View `start_time` and the pod is selected with -# `pod_filter`; the exact case_window (win_lo/win_hi) is shown in Level 2. -# Point `raw_table` at dc_snoop (default), redis_events, or kubescape_logs to -# reconstruct the other §H tables (their projected columns differ -- see README). - - -def consulted_rows(start_time: str, clickhouse_dsn: str, raw_table: str, pod_filter: str, investigation_id: str): - df = px.DataFrame(raw_table, clickhouse_dsn=clickhouse_dsn, start_time=start_time) - - # Scope to the alert pod (the investigation's subject). Empty = all pods. - df = df[px.contains(df.pod, pod_filter)] - - # Carry the selected investigation for context / cross-reference. - df.investigation_id = investigation_id - - return df[['time_', 'namespace', 'pod', 'container', - 'comm', 'pid', 'file', 'investigation_id']] diff --git a/src/pxl_scripts/dx/evidence_graph/manifest.yaml b/src/pxl_scripts/dx/evidence_graph/manifest.yaml deleted file mode 100644 index d09d7f61d7e..00000000000 --- a/src/pxl_scripts/dx/evidence_graph/manifest.yaml +++ /dev/null @@ -1,9 +0,0 @@ ---- -short: DX Evidence Graph (3-level zoom) -long: > - Renders the dx evidence graph in Pixie's built-in GraphWidget and lets an analyst zoom - from the whole malignant investigation set into one investigation's manifest and the raw - forensic rows dx consulted. Level 1: severity-weighted pod-to-pod edges (nodes deep-link to - px/pod). Level 2: the manifest (verdict, case window, evidence_hash). Level 3: the §H - reconstruction of the consulted dc_snoop / redis_events / kubescape_logs rows. Reads - forensic_db in ClickHouse via px.DataFrame(clickhouse_dsn=...). diff --git a/src/pxl_scripts/dx/evidence_graph/vis.json b/src/pxl_scripts/dx/evidence_graph/vis.json deleted file mode 100644 index 2db3923b641..00000000000 --- a/src/pxl_scripts/dx/evidence_graph/vis.json +++ /dev/null @@ -1,138 +0,0 @@ -{ - "variables": [ - { - "name": "start_time", - "type": "PX_STRING", - "description": "Start time of the window (approximates the case window for Level 3).", - "defaultValue": "-15m" - }, - { - "name": "clickhouse_dsn", - "type": "PX_STRING", - "description": "ClickHouse DSN for forensic_db: user:pass@host:port/db.", - "defaultValue": "ingest_writer:changeme-ingest@clickhouse-forensic-soc-db.clickhouse.svc.cluster.local:9000/forensic_db" - }, - { - "name": "table", - "type": "PX_STRING", - "description": "Evidence-graph edge table (default: rule-ins-only malignant view).", - "defaultValue": "dx_evidence_graph_malignant" - }, - { - "name": "investigation_filter", - "type": "PX_STRING", - "description": "Substring to narrow the graph to one investigation_id. Empty = all malignant edges.", - "defaultValue": "" - }, - { - "name": "manifest_table", - "type": "PX_STRING", - "description": "Manifest table (verdict / case-window / evidence_hash).", - "defaultValue": "dx_evidence_manifest" - }, - { - "name": "investigation_id", - "type": "PX_STRING", - "description": "ZOOM: the investigation_id to expand into its manifest + consulted rows. Copy it from a graph edge hover. Empty = all.", - "defaultValue": "" - }, - { - "name": "raw_table", - "type": "PX_STRING", - "description": "Level-3 forensic table to reconstruct: dc_snoop (default), redis_events, or kubescape_logs.", - "defaultValue": "dc_snoop" - }, - { - "name": "pod_filter", - "type": "PX_POD", - "description": "ZOOM: the alert pod whose consulted rows to show (substring match). Empty = all pods in window.", - "defaultValue": "" - } - ], - "globalFuncs": [ - { - "outputName": "dx_graph", - "func": { - "name": "evidence_graph", - "args": [ - {"name": "start_time", "variable": "start_time"}, - {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, - {"name": "table", "variable": "table"}, - {"name": "investigation_filter", "variable": "investigation_filter"} - ] - } - }, - { - "outputName": "dx_manifest", - "func": { - "name": "investigation_detail", - "args": [ - {"name": "start_time", "variable": "start_time"}, - {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, - {"name": "manifest_table", "variable": "manifest_table"}, - {"name": "investigation_id", "variable": "investigation_id"} - ] - } - }, - { - "outputName": "dx_consulted", - "func": { - "name": "consulted_rows", - "args": [ - {"name": "start_time", "variable": "start_time"}, - {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, - {"name": "raw_table", "variable": "raw_table"}, - {"name": "pod_filter", "variable": "pod_filter"}, - {"name": "investigation_id", "variable": "investigation_id"} - ] - } - } - ], - "widgets": [ - { - "name": "Evidence Graph (double-click a pod node -> px/pod)", - "position": {"x": 0, "y": 0, "w": 12, "h": 5}, - "globalFuncOutputName": "dx_graph", - "displaySpec": { - "@type": "types.px.dev/px.vispb.Graph", - "adjacencyList": { - "fromColumn": "from_entity", - "toColumn": "to_entity" - }, - "edgeWeightColumn": "confidence", - "edgeColorColumn": "max_severity", - "edgeLabelColumn": "edge_kind", - "edgeThresholds": { - "mediumThreshold": 3, - "highThreshold": 4 - }, - "edgeHoverInfo": [ - "investigation_id", - "condition", - "criteria", - "edge_kind", - "confidence", - "max_severity", - "num_findings" - ], - "edgeLength": 500 - } - }, - { - "name": "Level 2 - Investigation Manifest (verdict / case-window / evidence_hash)", - "position": {"x": 0, "y": 5, "w": 12, "h": 3}, - "globalFuncOutputName": "dx_manifest", - "displaySpec": { - "@type": "types.px.dev/px.vispb.Table" - } - }, - { - "name": "Level 3 - Consulted Raw Rows (dc_snoop / redis_events / kubescape_logs)", - "position": {"x": 0, "y": 8, "w": 12, "h": 4}, - "globalFuncOutputName": "dx_consulted", - "displaySpec": { - "@type": "types.px.dev/px.vispb.Table" - } - } - ] -} diff --git a/src/pxl_scripts/dx/evidence_graph/README.md b/src/pxl_scripts/px/dx_evidence_graph/README.md similarity index 97% rename from src/pxl_scripts/dx/evidence_graph/README.md rename to src/pxl_scripts/px/dx_evidence_graph/README.md index 29a3e21380b..ea27bccd455 100644 --- a/src/pxl_scripts/dx/evidence_graph/README.md +++ b/src/pxl_scripts/px/dx_evidence_graph/README.md @@ -1,4 +1,4 @@ -# DX Evidence Graph — 3-level zoom (`dx/evidence_graph`) +# DX Evidence Graph — 3-level zoom (`px/dx_evidence_graph`) A **standalone** Pixie Live View bundle (PxL + `vis.json`, no Pixie UI source changes) that renders the dx evidence graph in Pixie's existing `GraphWidget` and @@ -40,7 +40,7 @@ This is a self-contained scripts bundle — deploy it without touching the UI: 2. **Bundled script.** The directory (`evidence_graph.pxl` + `vis.json` + `manifest.yaml`) is globbed into `bundle-oss.json` by `src/pxl_scripts/BUILD.bazel` (the `**/*.pxl|json|yaml` filegroup), so it ships as - the script id **`dx/evidence_graph`** wherever that bundle is served. No registry + the script id **`px/dx_evidence_graph`** wherever that bundle is served. No registry edit is required. 3. **`px` CLI.** `px run -f evidence_graph.pxl` (table output) for a non-UI smoke test. diff --git a/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl b/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl index c5860a94a2c..f78e176c3db 100644 --- a/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl +++ b/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl @@ -16,19 +16,69 @@ import px +# 3-level zoom over the dx evidence graph (all reads via the ClickHouse forensic_db): +# L1 evidence graph -> pod->pod malignant edges (this func) +# L2 investigation -> the manifest row (verdict / case-window / evidence_hash) +# L3 consulted rows -> the raw forensic rows dx considered (the §H reconstruction) +# The `investigation_id` vis variable is the zoom key for L2/L3. Graph nodes are +# stamped px.Pod so a double-click deep-links to px/pod via the GraphWidget's +# built-in deepLinkURLFromSemanticType -- no Pixie UI change. -def dx_evidence_graph(start_time: str, clickhouse_dsn: str, table: str): + +# Level 1 ----------------------------------------------------------------------- +def dx_evidence_graph(start_time: str, clickhouse_dsn: str, table: str, investigation_filter: str): df = px.DataFrame(table, clickhouse_dsn=clickhouse_dsn, start_time=start_time) + # Best display name per endpoint: pod > service > ip. df.requestor = px.select(df.requestor_pod == '', px.select(df.requestor_service == '', df.requestor_ip, df.requestor_service), df.requestor_pod) df.responder = px.select(df.responder_pod == '', px.select(df.responder_service == '', df.responder_ip, df.responder_service), df.responder_pod) - return df[['requestor', 'responder', + # Stamp ST_POD_NAME so graph nodes are drill-able to px/pod out of the box. + df.from_entity = px.Pod(df.requestor) + df.to_entity = px.Pod(df.responder) + # Zoom: narrow the graph to one investigation. Empty filter = all malignant edges. + df = df[px.contains(df.investigation_id, investigation_filter)] + return df[['from_entity', 'to_entity', + 'requestor', 'responder', 'requestor_pod', 'responder_pod', 'requestor_service', 'responder_service', 'requestor_ip', 'responder_ip', 'weight', 'max_severity', 'confidence', 'edge_kind', 'condition', 'criteria', 'num_findings', 'investigation_id']] + + +# Level 2 ----------------------------------------------------------------------- +# The manifest row(s) for the selected investigation: verdict, case-window bounds, +# evidence_hash, and the raw findings JSON (the completeness contract). case_window +# is a JSON string column ({"lo":,"hi":}); pluck the bounds. +def investigation_detail(start_time: str, clickhouse_dsn: str, manifest_table: str, investigation_id: str): + df = px.DataFrame(manifest_table, clickhouse_dsn=clickhouse_dsn, start_time=start_time) + # Zoom filter. Empty investigation_id = every manifest in the window. + df = df[px.contains(df.investigation_id, investigation_id)] + df.win_lo = px.pluck_int64(df.case_window, 'lo') + df.win_hi = px.pluck_int64(df.case_window, 'hi') + return df[['investigation_id', 'verdict', 'condition', + 'confidence', 'posterior', 'catalog_version', + 'win_lo', 'win_hi', 'evidence_hash', 'findings']] + + +# Level 3 ----------------------------------------------------------------------- +# The raw forensic rows dx consulted for the selected investigation -- the §H +# reconstruction. Adapts demo.md §H3 (dc_snoop process rows for the alert pod). +# px.DataFrame(clickhouse_dsn) is a templated per-table read (not arbitrary SQL), +# so the case-window bound is approximated by the Live View `start_time` and the +# pod is selected with `pod_filter`; the exact case_window (win_lo/win_hi) is +# shown in Level 2. Point `raw_table` at dc_snoop (default), redis_events, or +# kubescape_logs to reconstruct the other §H tables (their projected columns +# differ -- see README). +def consulted_rows(start_time: str, clickhouse_dsn: str, raw_table: str, pod_filter: str, investigation_id: str): + df = px.DataFrame(raw_table, clickhouse_dsn=clickhouse_dsn, start_time=start_time) + # Scope to the alert pod (the investigation's subject). Empty = all pods. + df = df[px.contains(df.pod, pod_filter)] + # Carry the selected investigation for context / cross-reference. + df.investigation_id = investigation_id + return df[['time_', 'namespace', 'pod', 'container', + 'comm', 'pid', 'file', 'investigation_id']] diff --git a/src/pxl_scripts/px/dx_evidence_graph/vis.json b/src/pxl_scripts/px/dx_evidence_graph/vis.json index 90befd97383..5ff8d5993d8 100644 --- a/src/pxl_scripts/px/dx_evidence_graph/vis.json +++ b/src/pxl_scripts/px/dx_evidence_graph/vis.json @@ -3,20 +3,50 @@ { "name": "start_time", "type": "PX_STRING", - "description": "Start time of the window.", + "description": "Start time of the window (approximates the case window for Level 3).", "defaultValue": "-15m" }, { "name": "clickhouse_dsn", "type": "PX_STRING", - "description": "ClickHouse DSN: user:pass@host:port/db.", + "description": "ClickHouse DSN for forensic_db: user:pass@host:port/db.", "defaultValue": "forensic_analyst:changeme-analyst@clickhouse-forensic-soc-db.clickhouse.svc.cluster.local:9000/forensic_db" }, { "name": "table", "type": "PX_STRING", - "description": "dx_evidence_graph", + "description": "Evidence-graph edge table (default: rule-ins-only malignant view).", "defaultValue": "dx_evidence_graph_malignant" + }, + { + "name": "investigation_filter", + "type": "PX_STRING", + "description": "Substring to narrow the graph to one investigation_id. Empty = all malignant edges.", + "defaultValue": "" + }, + { + "name": "manifest_table", + "type": "PX_STRING", + "description": "Manifest table (verdict / case-window / evidence_hash).", + "defaultValue": "dx_evidence_manifest" + }, + { + "name": "investigation_id", + "type": "PX_STRING", + "description": "ZOOM: the investigation_id to expand into its manifest + consulted rows. Copy it from a graph edge hover. Empty = all.", + "defaultValue": "" + }, + { + "name": "raw_table", + "type": "PX_STRING", + "description": "Level-3 forensic table to reconstruct: dc_snoop (default), redis_events, or kubescape_logs.", + "defaultValue": "dc_snoop" + }, + { + "name": "pod_filter", + "type": "PX_POD", + "description": "ZOOM: the alert pod whose consulted rows to show (substring match). Empty = all pods in window.", + "defaultValue": "" } ], "globalFuncs": [ @@ -27,23 +57,49 @@ "args": [ {"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, - {"name": "table", "variable": "table"} + {"name": "table", "variable": "table"}, + {"name": "investigation_filter", "variable": "investigation_filter"} + ] + } + }, + { + "outputName": "dx_manifest", + "func": { + "name": "investigation_detail", + "args": [ + {"name": "start_time", "variable": "start_time"}, + {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, + {"name": "manifest_table", "variable": "manifest_table"}, + {"name": "investigation_id", "variable": "investigation_id"} + ] + } + }, + { + "outputName": "dx_consulted", + "func": { + "name": "consulted_rows", + "args": [ + {"name": "start_time", "variable": "start_time"}, + {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, + {"name": "raw_table", "variable": "raw_table"}, + {"name": "pod_filter", "variable": "pod_filter"}, + {"name": "investigation_id", "variable": "investigation_id"} ] } } ], "widgets": [ { - "name": "DX Evidence Graph", + "name": "DX Evidence Graph (double-click a pod node -> px/pod)", "position": {"x": 0, "y": 0, "w": 12, "h": 5}, "globalFuncOutputName": "dx_graph", "displaySpec": { "@type": "types.px.dev/px.vispb.Graph", "adjacencyList": { - "fromColumn": "requestor", - "toColumn": "responder" + "fromColumn": "from_entity", + "toColumn": "to_entity" }, - "edgeWeightColumn": "weight", + "edgeWeightColumn": "confidence", "edgeColorColumn": "max_severity", "edgeLabelColumn": "edge_kind", "edgeThresholds": { @@ -51,25 +107,40 @@ "highThreshold": 4 }, "edgeHoverInfo": [ - "edge_kind", + "investigation_id", "condition", "criteria", - "weight", - "max_severity", + "edge_kind", "confidence", - "num_findings", - "investigation_id" + "max_severity", + "num_findings" ], "edgeLength": 500 } }, { "name": "Edges", - "position": {"x": 0, "y": 5, "w": 12, "h": 4}, + "position": {"x": 0, "y": 5, "w": 12, "h": 3}, "globalFuncOutputName": "dx_graph", "displaySpec": { "@type": "types.px.dev/px.vispb.Table" } + }, + { + "name": "Level 2 - Investigation Manifest (verdict / case-window / evidence_hash)", + "position": {"x": 0, "y": 8, "w": 12, "h": 3}, + "globalFuncOutputName": "dx_manifest", + "displaySpec": { + "@type": "types.px.dev/px.vispb.Table" + } + }, + { + "name": "Level 3 - Consulted Raw Rows (dc_snoop / redis_events / kubescape_logs)", + "position": {"x": 0, "y": 11, "w": 12, "h": 4}, + "globalFuncOutputName": "dx_consulted", + "displaySpec": { + "@type": "types.px.dev/px.vispb.Table" + } } ] } From 0a473332f0ad390b5ba270b4690f37f22200d478 Mon Sep 17 00:00:00 2001 From: entlein Date: Sun, 16 Aug 2026 19:59:38 +0200 Subject: [PATCH 24/66] pxl_scripts(px/dx_evidence_graph): fix reads validated live against forensic_db L3 consulted_rows referenced df.pod/namespace/container which exist in NO raw table (dc_snoop has only time_,pid,comm,t,file,hostname,event_time) -> the 'Column pod not found' compile error that killed the whole view. Project the event_time+hostname intersection (present in every raw_table), filter by hostname (host_filter, was pod_filter/PX_POD). raw_table default dc_snoop -> kubescape_logs: the px ClickHouseSourceNode only returns rows for UInt64 event_time; dc_snoop/redis_events/conn_stats are DateTime64 and read back 0 despite millions in CH. kubescape_logs (and the UInt64 dx_evidence_* graph/manifest tables) are the readable ones. --- .../dx_evidence_graph/dx_evidence_graph.pxl | 37 ++++++++++++------- src/pxl_scripts/px/dx_evidence_graph/vis.json | 14 +++---- 2 files changed, 31 insertions(+), 20 deletions(-) diff --git a/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl b/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl index f78e176c3db..f0472b180bc 100644 --- a/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl +++ b/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl @@ -67,18 +67,29 @@ def investigation_detail(start_time: str, clickhouse_dsn: str, manifest_table: s # Level 3 ----------------------------------------------------------------------- # The raw forensic rows dx consulted for the selected investigation -- the §H -# reconstruction. Adapts demo.md §H3 (dc_snoop process rows for the alert pod). -# px.DataFrame(clickhouse_dsn) is a templated per-table read (not arbitrary SQL), -# so the case-window bound is approximated by the Live View `start_time` and the -# pod is selected with `pod_filter`; the exact case_window (win_lo/win_hi) is -# shown in Level 2. Point `raw_table` at dc_snoop (default), redis_events, or -# kubescape_logs to reconstruct the other §H tables (their projected columns -# differ -- see README). -def consulted_rows(start_time: str, clickhouse_dsn: str, raw_table: str, pod_filter: str, investigation_id: str): +# reconstruction. px.DataFrame(clickhouse_dsn) is a templated per-table read (not +# arbitrary SQL), so the case-window bound is approximated by the Live View +# `start_time` and rows are scoped by `host_filter` (the node). The exact +# case_window (win_lo/win_hi) is shown in Level 2. Point `raw_table` at dc_snoop +# (default), redis_events, or kubescape_logs. +# +# DEFENSIVE PROJECTION (validated live against forensic_db): the three raw tables +# share ONLY `event_time` + `hostname` -- dc_snoop has no pod/namespace/container +# column, redis_events keys off upid, and kubescape_logs has neither `time_` nor +# `investigation_id`. Projecting strictly that intersection guarantees the default +# view renders for ANY raw_table instead of failing to compile on a missing column. +# +# READABILITY (validated live): the px ClickHouseSourceNode only returns rows for +# tables whose event_time is UInt64. kubescape_logs is UInt64 and reads back fine; +# dc_snoop / redis_events / conn_stats use DateTime64 and read back 0 rows despite +# millions present in CH. So the default raw_table is kubescape_logs -- the one that +# actually renders. dc_snoop et al. need the Pixie-side DateTime64 read fix first. +def consulted_rows(start_time: str, clickhouse_dsn: str, raw_table: str, host_filter: str, investigation_id: str): df = px.DataFrame(raw_table, clickhouse_dsn=clickhouse_dsn, start_time=start_time) - # Scope to the alert pod (the investigation's subject). Empty = all pods. - df = df[px.contains(df.pod, pod_filter)] - # Carry the selected investigation for context / cross-reference. + # Scope by hostname (the node) -- the only filterable column present in every + # raw_table. Empty = all nodes in the window. + df = df[px.contains(df.hostname, host_filter)] + # Carry the selected investigation for context. kubescape_logs has no + # investigation_id of its own, so this is a stamped constant, not a read. df.investigation_id = investigation_id - return df[['time_', 'namespace', 'pod', 'container', - 'comm', 'pid', 'file', 'investigation_id']] + return df[['event_time', 'hostname', 'investigation_id']] diff --git a/src/pxl_scripts/px/dx_evidence_graph/vis.json b/src/pxl_scripts/px/dx_evidence_graph/vis.json index 5ff8d5993d8..39b8df9942b 100644 --- a/src/pxl_scripts/px/dx_evidence_graph/vis.json +++ b/src/pxl_scripts/px/dx_evidence_graph/vis.json @@ -39,13 +39,13 @@ { "name": "raw_table", "type": "PX_STRING", - "description": "Level-3 forensic table to reconstruct: dc_snoop (default), redis_events, or kubescape_logs.", - "defaultValue": "dc_snoop" + "description": "Level-3 forensic table to reconstruct. Default kubescape_logs -- the only UInt64 event_time table px can read via clickhouse_dsn. dc_snoop/redis_events/conn_stats use DateTime64 and currently read back 0 rows (Pixie ClickHouseSourceNode limitation).", + "defaultValue": "kubescape_logs" }, { - "name": "pod_filter", - "type": "PX_POD", - "description": "ZOOM: the alert pod whose consulted rows to show (substring match). Empty = all pods in window.", + "name": "host_filter", + "type": "PX_STRING", + "description": "ZOOM: node/hostname substring to scope Level-3 consulted rows (the only column present in every raw_table). Empty = all nodes in window.", "defaultValue": "" } ], @@ -82,7 +82,7 @@ {"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "raw_table", "variable": "raw_table"}, - {"name": "pod_filter", "variable": "pod_filter"}, + {"name": "host_filter", "variable": "host_filter"}, {"name": "investigation_id", "variable": "investigation_id"} ] } @@ -135,7 +135,7 @@ } }, { - "name": "Level 3 - Consulted Raw Rows (dc_snoop / redis_events / kubescape_logs)", + "name": "Level 3 - Consulted Raw Rows (event_time + hostname; raw_table: dc_snoop / redis_events / kubescape_logs)", "position": {"x": 0, "y": 11, "w": 12, "h": 4}, "globalFuncOutputName": "dx_consulted", "displaySpec": { From aa3d30a32529117e72703cb09fd989bb76a22b03 Mon Sep 17 00:00:00 2001 From: entlein Date: Sun, 16 Aug 2026 21:15:52 +0200 Subject: [PATCH 25/66] pxl_scripts(px/dx_evidence_graph): L3 shows the REAL anomaly records, not event_time/hostname The defensive event_time+hostname+investigation_id projection returned content-free rows (a nanosecond int + node name + blank) -- looked like random fields. Fix: consulted_rows pins to kubescape_logs (the one px-readable table) and px.plucks the real evidence from the RuntimeK8sDetails / RuntimeProcessDetails JSON columns: namespace/pod/container + comm/cmdline + RuleID + the alert message. Drops the raw_table var (dc_snoop/redis_events are DateTime64-unreadable anyway). Validated live: redis/redis-master-0 R0001/R0002/R0006/R0008/R0010/R0011 with real messages. --- .../dx_evidence_graph/dx_evidence_graph.pxl | 47 +++++++++---------- src/pxl_scripts/px/dx_evidence_graph/vis.json | 12 +---- 2 files changed, 23 insertions(+), 36 deletions(-) diff --git a/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl b/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl index f0472b180bc..c63729d0fe2 100644 --- a/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl +++ b/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl @@ -66,30 +66,25 @@ def investigation_detail(start_time: str, clickhouse_dsn: str, manifest_table: s # Level 3 ----------------------------------------------------------------------- -# The raw forensic rows dx consulted for the selected investigation -- the §H -# reconstruction. px.DataFrame(clickhouse_dsn) is a templated per-table read (not -# arbitrary SQL), so the case-window bound is approximated by the Live View -# `start_time` and rows are scoped by `host_filter` (the node). The exact -# case_window (win_lo/win_hi) is shown in Level 2. Point `raw_table` at dc_snoop -# (default), redis_events, or kubescape_logs. -# -# DEFENSIVE PROJECTION (validated live against forensic_db): the three raw tables -# share ONLY `event_time` + `hostname` -- dc_snoop has no pod/namespace/container -# column, redis_events keys off upid, and kubescape_logs has neither `time_` nor -# `investigation_id`. Projecting strictly that intersection guarantees the default -# view renders for ANY raw_table instead of failing to compile on a missing column. -# -# READABILITY (validated live): the px ClickHouseSourceNode only returns rows for -# tables whose event_time is UInt64. kubescape_logs is UInt64 and reads back fine; -# dc_snoop / redis_events / conn_stats use DateTime64 and read back 0 rows despite -# millions present in CH. So the default raw_table is kubescape_logs -- the one that -# actually renders. dc_snoop et al. need the Pixie-side DateTime64 read fix first. -def consulted_rows(start_time: str, clickhouse_dsn: str, raw_table: str, host_filter: str, investigation_id: str): - df = px.DataFrame(raw_table, clickhouse_dsn=clickhouse_dsn, start_time=start_time) - # Scope by hostname (the node) -- the only filterable column present in every - # raw_table. Empty = all nodes in the window. +# The actual kubescape anomaly records dx consulted -- the real forensic evidence: +# RuleID + the offending pod/container + the process (comm/cmdline) + the alert +# message. Read from forensic_db.kubescape_logs (the one px-readable UInt64 table; +# dc_snoop/redis_events/conn_stats are DateTime64 and read back 0 rows via the +# ClickHouseSourceNode). The pod & process detail live inside two JSON columns +# (RuntimeK8sDetails, RuntimeProcessDetails), so we px.pluck the fields out rather +# than returning opaque event_time/hostname. Scoped by `host_filter` (node); +# empty = all nodes in the window. +def consulted_rows(start_time: str, clickhouse_dsn: str, host_filter: str): + df = px.DataFrame('kubescape_logs', clickhouse_dsn=clickhouse_dsn, start_time=start_time) df = df[px.contains(df.hostname, host_filter)] - # Carry the selected investigation for context. kubescape_logs has no - # investigation_id of its own, so this is a stamped constant, not a read. - df.investigation_id = investigation_id - return df[['event_time', 'hostname', 'investigation_id']] + # pod / container from the k8s-details JSON + df.namespace = px.pluck(df.RuntimeK8sDetails, 'podNamespace') + df.pod = px.pluck(df.RuntimeK8sDetails, 'podName') + df.container = px.pluck(df.RuntimeK8sDetails, 'containerName') + # offending process from the process-details JSON + df.comm = px.pluck(df.RuntimeProcessDetails, 'comm') + df.cmdline = px.pluck(df.RuntimeProcessDetails, 'cmdline') + df.rule = df.RuleID + df.alert = df.message + return df[['event_time', 'namespace', 'pod', 'container', + 'rule', 'comm', 'cmdline', 'alert']] diff --git a/src/pxl_scripts/px/dx_evidence_graph/vis.json b/src/pxl_scripts/px/dx_evidence_graph/vis.json index 39b8df9942b..21b873313e4 100644 --- a/src/pxl_scripts/px/dx_evidence_graph/vis.json +++ b/src/pxl_scripts/px/dx_evidence_graph/vis.json @@ -36,12 +36,6 @@ "description": "ZOOM: the investigation_id to expand into its manifest + consulted rows. Copy it from a graph edge hover. Empty = all.", "defaultValue": "" }, - { - "name": "raw_table", - "type": "PX_STRING", - "description": "Level-3 forensic table to reconstruct. Default kubescape_logs -- the only UInt64 event_time table px can read via clickhouse_dsn. dc_snoop/redis_events/conn_stats use DateTime64 and currently read back 0 rows (Pixie ClickHouseSourceNode limitation).", - "defaultValue": "kubescape_logs" - }, { "name": "host_filter", "type": "PX_STRING", @@ -81,9 +75,7 @@ "args": [ {"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, - {"name": "raw_table", "variable": "raw_table"}, - {"name": "host_filter", "variable": "host_filter"}, - {"name": "investigation_id", "variable": "investigation_id"} + {"name": "host_filter", "variable": "host_filter"} ] } } @@ -135,7 +127,7 @@ } }, { - "name": "Level 3 - Consulted Raw Rows (event_time + hostname; raw_table: dc_snoop / redis_events / kubescape_logs)", + "name": "Level 3 - Consulted anomaly records (kubescape_logs: RuleID / pod / container / process / cmdline / alert)", "position": {"x": 0, "y": 11, "w": 12, "h": 4}, "globalFuncOutputName": "dx_consulted", "displaySpec": { From cef667b4d8ed0f6a28b2deadba9fae7a4a2ae084 Mon Sep 17 00:00:00 2001 From: entlein Date: Mon, 17 Aug 2026 19:48:07 +0200 Subject: [PATCH 26/66] pxl_scripts(px/dx_evidence_graph): join on uniqueID + surface process Re-point L1/L2 to the deterministic uniqueID join (dx rc15 carries the kubescape uniqueID into the manifest seed). Graph edges = subject_pod ->[process]-> target (process now surfaced: cat/ln/getent); consulted findings join on uniqueID, not the lossy RuleID@timestamp. Reads the dx_kubescape_anomalies + dx_anomaly_findings views (deduped). --- .../dx_evidence_graph/dx_evidence_graph.pxl | 76 ++----------- src/pxl_scripts/px/dx_evidence_graph/vis.json | 104 ++++++------------ 2 files changed, 45 insertions(+), 135 deletions(-) diff --git a/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl b/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl index c63729d0fe2..435b8e92ef3 100644 --- a/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl +++ b/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl @@ -16,75 +16,17 @@ import px -# 3-level zoom over the dx evidence graph (all reads via the ClickHouse forensic_db): -# L1 evidence graph -> pod->pod malignant edges (this func) -# L2 investigation -> the manifest row (verdict / case-window / evidence_hash) -# L3 consulted rows -> the raw forensic rows dx considered (the §H reconstruction) -# The `investigation_id` vis variable is the zoom key for L2/L3. Graph nodes are -# stamped px.Pod so a double-click deep-links to px/pod via the GraphWidget's -# built-in deepLinkURLFromSemanticType -- no Pixie UI change. - -# Level 1 ----------------------------------------------------------------------- -def dx_evidence_graph(start_time: str, clickhouse_dsn: str, table: str, investigation_filter: str): +def evidence_graph(start_time: str, clickhouse_dsn: str, table: str): df = px.DataFrame(table, clickhouse_dsn=clickhouse_dsn, start_time=start_time) - # Best display name per endpoint: pod > service > ip. - df.requestor = px.select(df.requestor_pod == '', - px.select(df.requestor_service == '', df.requestor_ip, df.requestor_service), - df.requestor_pod) - df.responder = px.select(df.responder_pod == '', - px.select(df.responder_service == '', df.responder_ip, df.responder_service), - df.responder_pod) - # Stamp ST_POD_NAME so graph nodes are drill-able to px/pod out of the box. - df.from_entity = px.Pod(df.requestor) - df.to_entity = px.Pod(df.responder) - # Zoom: narrow the graph to one investigation. Empty filter = all malignant edges. - df = df[px.contains(df.investigation_id, investigation_filter)] + df.from_entity = px.Pod(df.subject_pod) + df.to_entity = df.target return df[['from_entity', 'to_entity', - 'requestor', 'responder', - 'requestor_pod', 'responder_pod', - 'requestor_service', 'responder_service', - 'requestor_ip', 'responder_ip', - 'weight', 'max_severity', 'confidence', - 'edge_kind', 'condition', 'criteria', 'num_findings', - 'investigation_id']] - + 'uniqueID', 'rule', 'process', 'target', 'target_kind', + 'severity', 'alert', 'subject_pod']] -# Level 2 ----------------------------------------------------------------------- -# The manifest row(s) for the selected investigation: verdict, case-window bounds, -# evidence_hash, and the raw findings JSON (the completeness contract). case_window -# is a JSON string column ({"lo":,"hi":}); pluck the bounds. -def investigation_detail(start_time: str, clickhouse_dsn: str, manifest_table: str, investigation_id: str): - df = px.DataFrame(manifest_table, clickhouse_dsn=clickhouse_dsn, start_time=start_time) - # Zoom filter. Empty investigation_id = every manifest in the window. - df = df[px.contains(df.investigation_id, investigation_id)] - df.win_lo = px.pluck_int64(df.case_window, 'lo') - df.win_hi = px.pluck_int64(df.case_window, 'hi') - return df[['investigation_id', 'verdict', 'condition', - 'confidence', 'posterior', 'catalog_version', - 'win_lo', 'win_hi', 'evidence_hash', 'findings']] - -# Level 3 ----------------------------------------------------------------------- -# The actual kubescape anomaly records dx consulted -- the real forensic evidence: -# RuleID + the offending pod/container + the process (comm/cmdline) + the alert -# message. Read from forensic_db.kubescape_logs (the one px-readable UInt64 table; -# dc_snoop/redis_events/conn_stats are DateTime64 and read back 0 rows via the -# ClickHouseSourceNode). The pod & process detail live inside two JSON columns -# (RuntimeK8sDetails, RuntimeProcessDetails), so we px.pluck the fields out rather -# than returning opaque event_time/hostname. Scoped by `host_filter` (node); -# empty = all nodes in the window. -def consulted_rows(start_time: str, clickhouse_dsn: str, host_filter: str): - df = px.DataFrame('kubescape_logs', clickhouse_dsn=clickhouse_dsn, start_time=start_time) - df = df[px.contains(df.hostname, host_filter)] - # pod / container from the k8s-details JSON - df.namespace = px.pluck(df.RuntimeK8sDetails, 'podNamespace') - df.pod = px.pluck(df.RuntimeK8sDetails, 'podName') - df.container = px.pluck(df.RuntimeK8sDetails, 'containerName') - # offending process from the process-details JSON - df.comm = px.pluck(df.RuntimeProcessDetails, 'comm') - df.cmdline = px.pluck(df.RuntimeProcessDetails, 'cmdline') - df.rule = df.RuleID - df.alert = df.message - return df[['event_time', 'namespace', 'pod', 'container', - 'rule', 'comm', 'cmdline', 'alert']] +def consulted_findings(start_time: str, clickhouse_dsn: str, table: str, uniqueID: str): + df = px.DataFrame(table, clickhouse_dsn=clickhouse_dsn, start_time=start_time) + df = df[px.contains(df.uniqueID, uniqueID)] + return df[['uniqueID', 'vector', 'source', 'src_table', 'detail']] diff --git a/src/pxl_scripts/px/dx_evidence_graph/vis.json b/src/pxl_scripts/px/dx_evidence_graph/vis.json index 21b873313e4..c63e6e86f89 100644 --- a/src/pxl_scripts/px/dx_evidence_graph/vis.json +++ b/src/pxl_scripts/px/dx_evidence_graph/vis.json @@ -3,8 +3,8 @@ { "name": "start_time", "type": "PX_STRING", - "description": "Start time of the window (approximates the case window for Level 3).", - "defaultValue": "-15m" + "description": "Start of the window.", + "defaultValue": "-30m" }, { "name": "clickhouse_dsn", @@ -13,33 +13,21 @@ "defaultValue": "forensic_analyst:changeme-analyst@clickhouse-forensic-soc-db.clickhouse.svc.cluster.local:9000/forensic_db" }, { - "name": "table", + "name": "graph_table", "type": "PX_STRING", - "description": "Evidence-graph edge table (default: rule-ins-only malignant view).", - "defaultValue": "dx_evidence_graph_malignant" + "description": "L1 graph source: one edge per deduped kubescape anomaly (subject_pod -> target).", + "defaultValue": "dx_kubescape_anomalies" }, { - "name": "investigation_filter", + "name": "findings_table", "type": "PX_STRING", - "description": "Substring to narrow the graph to one investigation_id. Empty = all malignant edges.", - "defaultValue": "" - }, - { - "name": "manifest_table", - "type": "PX_STRING", - "description": "Manifest table (verdict / case-window / evidence_hash).", - "defaultValue": "dx_evidence_manifest" - }, - { - "name": "investigation_id", - "type": "PX_STRING", - "description": "ZOOM: the investigation_id to expand into its manifest + consulted rows. Copy it from a graph edge hover. Empty = all.", - "defaultValue": "" + "description": "L2 source: the corroborating findings dx consulted, keyed by uniqueID.", + "defaultValue": "dx_anomaly_findings" }, { - "name": "host_filter", + "name": "uniqueID", "type": "PX_STRING", - "description": "ZOOM: node/hostname substring to scope Level-3 consulted rows (the only column present in every raw_table). Empty = all nodes in window.", + "description": "ZOOM: the anomaly uniqueID to expand into its consulted findings. Copy it from a graph-edge hover or the anomalies table. Empty = all findings in the window.", "defaultValue": "" } ], @@ -47,43 +35,31 @@ { "outputName": "dx_graph", "func": { - "name": "dx_evidence_graph", + "name": "evidence_graph", "args": [ {"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, - {"name": "table", "variable": "table"}, - {"name": "investigation_filter", "variable": "investigation_filter"} + {"name": "table", "variable": "graph_table"} ] } }, { - "outputName": "dx_manifest", + "outputName": "dx_findings", "func": { - "name": "investigation_detail", + "name": "consulted_findings", "args": [ {"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, - {"name": "manifest_table", "variable": "manifest_table"}, - {"name": "investigation_id", "variable": "investigation_id"} - ] - } - }, - { - "outputName": "dx_consulted", - "func": { - "name": "consulted_rows", - "args": [ - {"name": "start_time", "variable": "start_time"}, - {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, - {"name": "host_filter", "variable": "host_filter"} + {"name": "table", "variable": "findings_table"}, + {"name": "uniqueID", "variable": "uniqueID"} ] } } ], "widgets": [ { - "name": "DX Evidence Graph (double-click a pod node -> px/pod)", - "position": {"x": 0, "y": 0, "w": 12, "h": 5}, + "name": "DX Evidence Graph (subject pod -> target; double-click a pod node -> px/pod)", + "position": {"x": 0, "y": 0, "w": 12, "h": 6}, "globalFuncOutputName": "dx_graph", "displaySpec": { "@type": "types.px.dev/px.vispb.Graph", @@ -91,45 +67,37 @@ "fromColumn": "from_entity", "toColumn": "to_entity" }, - "edgeWeightColumn": "confidence", - "edgeColorColumn": "max_severity", - "edgeLabelColumn": "edge_kind", + "edgeWeightColumn": "severity", + "edgeColorColumn": "severity", + "edgeLabelColumn": "rule", "edgeThresholds": { - "mediumThreshold": 3, - "highThreshold": 4 + "mediumThreshold": 5, + "highThreshold": 8 }, "edgeHoverInfo": [ - "investigation_id", - "condition", - "criteria", - "edge_kind", - "confidence", - "max_severity", - "num_findings" + "uniqueID", + "rule", + "process", + "target", + "target_kind", + "severity", + "alert" ], "edgeLength": 500 } }, { - "name": "Edges", - "position": {"x": 0, "y": 5, "w": 12, "h": 3}, - "globalFuncOutputName": "dx_graph", + "name": "Consulted findings (zoom: set uniqueID) - the evidence dx corroborated with", + "position": {"x": 0, "y": 6, "w": 12, "h": 4}, + "globalFuncOutputName": "dx_findings", "displaySpec": { "@type": "types.px.dev/px.vispb.Table" } }, { - "name": "Level 2 - Investigation Manifest (verdict / case-window / evidence_hash)", - "position": {"x": 0, "y": 8, "w": 12, "h": 3}, - "globalFuncOutputName": "dx_manifest", - "displaySpec": { - "@type": "types.px.dev/px.vispb.Table" - } - }, - { - "name": "Level 3 - Consulted anomaly records (kubescape_logs: RuleID / pod / container / process / cmdline / alert)", - "position": {"x": 0, "y": 11, "w": 12, "h": 4}, - "globalFuncOutputName": "dx_consulted", + "name": "Anomalies (deduped kubescape_logs -> the graph edges; copy a uniqueID to zoom)", + "position": {"x": 0, "y": 10, "w": 12, "h": 4}, + "globalFuncOutputName": "dx_graph", "displaySpec": { "@type": "types.px.dev/px.vispb.Table" } From e322caf6b2d8c7e4e5961a0aea8c85856e5e4b87 Mon Sep 17 00:00:00 2001 From: entlein Date: Mon, 17 Aug 2026 20:41:16 +0200 Subject: [PATCH 27/66] pxl_scripts(px/dx_evidence_graph): L2 shows the actual attack cmdline per anomaly dx's consulted findings reference benign background rows (proven: closest redis row is PING/CLIENT LIST, never the attack), so they can't surface the payload. The real command lives in kubescape_logs RuntimeProcessDetails.processTree.cmdline, keyed by uniqueID. Re-point consulted_findings at the dx_anomaly_findings view (now built from that process tree): rule / comm / parent / the actual cmdline / alert -- e.g. R0010 -> '/usr/bin/cat /etc/shadow', R1008 -> 'sh -c getent hosts xmr.pool.minergate.com'. --- src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl | 2 +- src/pxl_scripts/px/dx_evidence_graph/vis.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl b/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl index 435b8e92ef3..300cab5fefc 100644 --- a/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl +++ b/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl @@ -29,4 +29,4 @@ def evidence_graph(start_time: str, clickhouse_dsn: str, table: str): def consulted_findings(start_time: str, clickhouse_dsn: str, table: str, uniqueID: str): df = px.DataFrame(table, clickhouse_dsn=clickhouse_dsn, start_time=start_time) df = df[px.contains(df.uniqueID, uniqueID)] - return df[['uniqueID', 'vector', 'source', 'src_table', 'detail']] + return df[['uniqueID', 'rule', 'comm', 'parent', 'cmdline', 'alert']] diff --git a/src/pxl_scripts/px/dx_evidence_graph/vis.json b/src/pxl_scripts/px/dx_evidence_graph/vis.json index c63e6e86f89..b3848f4f80a 100644 --- a/src/pxl_scripts/px/dx_evidence_graph/vis.json +++ b/src/pxl_scripts/px/dx_evidence_graph/vis.json @@ -87,7 +87,7 @@ } }, { - "name": "Consulted findings (zoom: set uniqueID) - the evidence dx corroborated with", + "name": "Anomaly detail (zoom: set uniqueID) - rule / process / parent / the actual cmdline / alert", "position": {"x": 0, "y": 6, "w": 12, "h": 4}, "globalFuncOutputName": "dx_findings", "displaySpec": { From c591ab4d14055a1730461a70e507d02034ceca72 Mon Sep 17 00:00:00 2001 From: entlein Date: Mon, 17 Aug 2026 23:00:04 +0200 Subject: [PATCH 28/66] px/dx_evidence_graph: L2 = full pre-correlation record set; dx skaffold -> rc18 L2 (consulted_records) now joins each finding to its actual record: exact (time_) join for pixie redis_events/conn_stats (rc18 makes finding.time_ == source.time_), row_identity content for dc_snoop/dns/process. Reveals the COMPLETE evidence set per anomaly -- attack (cat /etc/shadow, anomalous.dns.query, mnt_payload/drifted_bob) AND benign (PING, 127.0.0.1, proc/self/stat) -- the pre-correlation completeness guarantee. Bump k8s/vizier/dx/dx-daemon.yaml to dx rc18 (the per-row timestamp fix chain rc15-18). --- k8s/vizier/dx/dx-daemon.yaml | 2 +- .../px/dx_evidence_graph/dx_evidence_graph.pxl | 4 ++-- src/pxl_scripts/px/dx_evidence_graph/vis.json | 16 ++++++++-------- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/k8s/vizier/dx/dx-daemon.yaml b/k8s/vizier/dx/dx-daemon.yaml index ea6479af766..fdc0e2c944b 100644 --- a/k8s/vizier/dx/dx-daemon.yaml +++ b/k8s/vizier/dx/dx-daemon.yaml @@ -23,7 +23,7 @@ spec: # (WITHOUT -tiny — -tiny's pclntab stripping SIGSEGV'd under load). Passes the # obfuscation gate AND survives the kill-chain (restarts=0, 4 rounds). Carries the # evidence-manifest + DX_FOREST_PUSHDOWN code. - image: docker.io/entlein/dx-daemon:0.4.0-ssotforest-rc14 + image: docker.io/entlein/dx-daemon:0.4.0-ssotforest-rc18 ports: - { name: findings, containerPort: 9099, hostPort: 9099 } env: diff --git a/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl b/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl index 300cab5fefc..201b8c0ddb1 100644 --- a/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl +++ b/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl @@ -26,7 +26,7 @@ def evidence_graph(start_time: str, clickhouse_dsn: str, table: str): 'severity', 'alert', 'subject_pod']] -def consulted_findings(start_time: str, clickhouse_dsn: str, table: str, uniqueID: str): +def consulted_records(start_time: str, clickhouse_dsn: str, table: str, uniqueID: str): df = px.DataFrame(table, clickhouse_dsn=clickhouse_dsn, start_time=start_time) df = df[px.contains(df.uniqueID, uniqueID)] - return df[['uniqueID', 'rule', 'comm', 'parent', 'cmdline', 'alert']] + return df[['uniqueID', 'vector', 'source', 'src_table', 'record']] diff --git a/src/pxl_scripts/px/dx_evidence_graph/vis.json b/src/pxl_scripts/px/dx_evidence_graph/vis.json index b3848f4f80a..4642e0e4e05 100644 --- a/src/pxl_scripts/px/dx_evidence_graph/vis.json +++ b/src/pxl_scripts/px/dx_evidence_graph/vis.json @@ -19,15 +19,15 @@ "defaultValue": "dx_kubescape_anomalies" }, { - "name": "findings_table", + "name": "records_table", "type": "PX_STRING", - "description": "L2 source: the corroborating findings dx consulted, keyed by uniqueID.", + "description": "L2 source: the pre-correlation consulted-records set (all evidence dx collected per anomaly).", "defaultValue": "dx_anomaly_findings" }, { "name": "uniqueID", "type": "PX_STRING", - "description": "ZOOM: the anomaly uniqueID to expand into its consulted findings. Copy it from a graph-edge hover or the anomalies table. Empty = all findings in the window.", + "description": "ZOOM: the anomaly uniqueID to expand into its consulted records. Copy it from a graph-edge hover or the anomalies table. Empty = all records in the window.", "defaultValue": "" } ], @@ -44,13 +44,13 @@ } }, { - "outputName": "dx_findings", + "outputName": "dx_records", "func": { - "name": "consulted_findings", + "name": "consulted_records", "args": [ {"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, - {"name": "table", "variable": "findings_table"}, + {"name": "table", "variable": "records_table"}, {"name": "uniqueID", "variable": "uniqueID"} ] } @@ -87,9 +87,9 @@ } }, { - "name": "Anomaly detail (zoom: set uniqueID) - rule / process / parent / the actual cmdline / alert", + "name": "Consulted records (zoom: set uniqueID) - the full pre-correlation evidence set per anomaly", "position": {"x": 0, "y": 6, "w": 12, "h": 4}, - "globalFuncOutputName": "dx_findings", + "globalFuncOutputName": "dx_records", "displaySpec": { "@type": "types.px.dev/px.vispb.Table" } From 75eec19fd95c20086510fa632baea2a2db0be6fb Mon Sep 17 00:00:00 2001 From: entlein Date: Tue, 18 Aug 2026 19:16:14 +0200 Subject: [PATCH 29/66] ae/pxl: stamp hostname on socket_tracer exports (#136 pushdown) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PodEnrichPxL set namespace + pod on the native (socket_tracer) tables but NOT hostname — only the stack_trace branch stamped it. So conn_stats / http_events / dns_events / redis_events (and every protocol table) landed in ClickHouse with an EMPTY hostname, even though hostname is the LEADING ORDER BY column on all of them. Consequences that this fixes: * px reads of these tables filter WHERE hostname=; empty hostname matched nothing (the reads only worked via a join that sourced hostname elsewhere). * the #136 order-UUID pre-correlation views could not expose a real hostname without an order-JOIN, and that join blocked the (hostname, event_time) primary-key pushdown (validated on rig 6a841cf7, CH 24.8). Fix: PodEnrichPxL's native path also emits df.hostname = px.upid_to_node_name(df.upid) — the same UDF stack_trace already uses; valid on any upid-bearing table. Dark-vector tables (raw pid, no upid) are unchanged; their node stamping is a separate follow-up. Tests: statement-count oracles +1 line. --- .../services/adaptive_export/internal/pxl/compile.go | 7 ++++++- .../adaptive_export/internal/pxl/queryfor_test.go | 9 +++++---- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/src/vizier/services/adaptive_export/internal/pxl/compile.go b/src/vizier/services/adaptive_export/internal/pxl/compile.go index cdd21c5313c..af81a209d8b 100644 --- a/src/vizier/services/adaptive_export/internal/pxl/compile.go +++ b/src/vizier/services/adaptive_export/internal/pxl/compile.go @@ -113,7 +113,12 @@ func PodEnrichPxL(table string) string { "df = df.merge(proc, how='left', left_on=['pid'], right_on=['pid'], suffixes=['', '_x'])\n" } return "df.namespace = px.upid_to_namespace(df.upid)\n" + - "df.pod = px.upid_to_pod_name(df.upid)\n" + "df.pod = px.upid_to_pod_name(df.upid)\n" + + // hostname = the capture node — the leading ORDER BY column on every + // socket_tracer table. AE left it empty (only stack_trace stamped it), so + // px reads of these tables (and the #136 order-UUID views) could not filter + // by hostname and the pushdown prefix (hostname,event_time) was unusable. + "df.hostname = px.upid_to_node_name(df.upid)\n" } // Render fills a CompilePassthrough template with the precise [sliceStart, diff --git a/src/vizier/services/adaptive_export/internal/pxl/queryfor_test.go b/src/vizier/services/adaptive_export/internal/pxl/queryfor_test.go index f42c779bd5b..6ca0023fd61 100644 --- a/src/vizier/services/adaptive_export/internal/pxl/queryfor_test.go +++ b/src/vizier/services/adaptive_export/internal/pxl/queryfor_test.go @@ -258,14 +258,15 @@ func TestEscapePxL_TableDriven(t *testing.T) { // df = df[df.time_ < ...] 1 // df.namespace = px.upid_to_namespace(...) 1 // df.pod = px.upid_to_pod_name(...) 1 +// df.hostname = px.upid_to_node_name(...) 1 // df = df[df.namespace == '...'] 1 // df = df[df.pod == '...'] 1 // px.display(df, '...') 1 -// (trailing newline → empty 11th split) 1 +// (trailing newline → empty 12th split) 1 // -// Total: 10 statements + trailing empty == strings.Split == 11 entries. +// Total: 11 statements + trailing empty == strings.Split == 12 entries. func TestQueryFor_RejectsInjectionInTargetFields(t *testing.T) { - const wantLines = 11 + const wantLines = 12 cases := []struct { name string @@ -337,7 +338,7 @@ func TestQueryFor_PodOnlyRegexEscapesQuoteMetaInjection(t *testing.T) { if err != nil { t.Fatalf("QueryFor: %v", err) } - if strings.Contains(q, "exec(") || strings.Count(q, "\n") > 9 { + if strings.Contains(q, "exec(") || strings.Count(q, "\n") > 10 { t.Fatalf("pod-only path injection succeeded:\n%s", q) } } From 49b0c2dc1594eafd2e114c5ee7760f3d4cce5345 Mon Sep 17 00:00:00 2001 From: entlein Date: Tue, 18 Aug 2026 22:36:36 +0200 Subject: [PATCH 30/66] ae: dx_order_seeds table + dc_snoop dark-path hostname (#136) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two AE changes for the order-UUID pre-correlation dashboard: 1. dx_order_seeds table (schema.sql + KnownTables + OperatorOwnedTables): dx INSERTs one row per referral (evidence-loss fix — dx coalesces same-pod anomalies so most write no manifest). The dx_anomaly_orders view windows every uniqueID from this. ReplacingMergeTree ORDER BY (unique_id, rule_id), 30d TTL. 2. PodEnrichPxL dark-vector branch stamps hostname via px.upid_to_node_name on the same process_stats upid pod/ns already come from, so dc_snoop (and the other dark tables) carry hostname and become px-readable. Transient pids that miss process_stats resolve blank — same accepted limitation as pod/ns. --- .../internal/clickhouse/apply.go | 3 +++ .../internal/clickhouse/ddl.go | 4 ++++ .../internal/clickhouse/schema.sql | 20 +++++++++++++++++++ .../adaptive_export/internal/pxl/compile.go | 10 ++++++++-- 4 files changed, 35 insertions(+), 2 deletions(-) diff --git a/src/vizier/services/adaptive_export/internal/clickhouse/apply.go b/src/vizier/services/adaptive_export/internal/clickhouse/apply.go index 411b79f443d..0c09c0a564d 100644 --- a/src/vizier/services/adaptive_export/internal/clickhouse/apply.go +++ b/src/vizier/services/adaptive_export/internal/clickhouse/apply.go @@ -83,6 +83,9 @@ var OperatorOwnedTables = []string{ // dx consulted. Created on boot so POST /dx/evidence_manifest has a target. // Independent of dx_evidence_graph. Not a pixie table → not in PixieTables(). "dx_evidence_manifest", + // dx per-referral order seeds (#136 evidence-loss fix) — created on boot so + // dx's direct INSERT has a target. Not a pixie table → not in PixieTables(). + "dx_order_seeds", } // Applier applies operator-owned DDL to a ClickHouse cluster over the diff --git a/src/vizier/services/adaptive_export/internal/clickhouse/ddl.go b/src/vizier/services/adaptive_export/internal/clickhouse/ddl.go index 0eb315882e9..24e720fa458 100644 --- a/src/vizier/services/adaptive_export/internal/clickhouse/ddl.go +++ b/src/vizier/services/adaptive_export/internal/clickhouse/ddl.go @@ -84,6 +84,10 @@ var KnownTables = []string{ // the evidence rows dx consulted (POST /dx/evidence_manifest). NOT a pixie // table. Independent of dx_evidence_graph. "dx_evidence_manifest", + // operator-owned dx per-referral order seeds (#136 evidence-loss fix). dx + // INSERTs one row per anomaly so dx_anomaly_orders can window every uniqueID. + // NOT a pixie table. + "dx_order_seeds", } // ErrUnknownTable is returned by DDL / Columns when asked for a table diff --git a/src/vizier/services/adaptive_export/internal/clickhouse/schema.sql b/src/vizier/services/adaptive_export/internal/clickhouse/schema.sql index 6fa8ed7f002..bfec589f2dc 100644 --- a/src/vizier/services/adaptive_export/internal/clickhouse/schema.sql +++ b/src/vizier/services/adaptive_export/internal/clickhouse/schema.sql @@ -567,6 +567,26 @@ CREATE TABLE IF NOT EXISTS forensic_db.dx_evidence_manifest ( TTL toDateTime(fromUnixTimestamp64Nano(event_time)) + INTERVAL 30 DAY DELETE SETTINGS index_granularity = 8192; +-- dx_order_seeds — one row per kubescape referral dx sees (entlein/dx#136 +-- evidence-loss fix). dx coalesces same-pod anomalies into one investigation, so +-- most anomalies write no manifest; this table records EVERY anomaly's identity so +-- the dx_anomaly_orders view can give each uniqueID its own consulted window +-- (event_time ± 300s). dx INSERTs (POST-less, direct CH); AE owns the DDL. +-- ReplacingMergeTree ORDER BY (unique_id, rule_id) dedups re-fires but keeps +-- co-fired rules. NOT a pixie table. +CREATE TABLE IF NOT EXISTS forensic_db.dx_order_seeds ( + unique_id String, + rule_id String, + pod String, + event_time UInt64, + hostname String, + case_key String +) ENGINE = ReplacingMergeTree() + ORDER BY (unique_id, rule_id) + PARTITION BY toYYYYMM(fromUnixTimestamp64Nano(event_time)) + TTL toDateTime(fromUnixTimestamp64Nano(event_time)) + INTERVAL 30 DAY DELETE + SETTINGS index_granularity = 8192; + -- ── dx dark-vector tracepoint tables (entlein/dx#126) ──────────────────────── -- Fed by AE-owned bpftrace UpsertTracepoint probes (constantly enabled, no TTL). -- Emit raw kernel pid+comm (NOT upid); namespace/pod enriched at pull time via a diff --git a/src/vizier/services/adaptive_export/internal/pxl/compile.go b/src/vizier/services/adaptive_export/internal/pxl/compile.go index af81a209d8b..06d5f79b1b3 100644 --- a/src/vizier/services/adaptive_export/internal/pxl/compile.go +++ b/src/vizier/services/adaptive_export/internal/pxl/compile.go @@ -108,9 +108,15 @@ func PodEnrichPxL(table string) string { return "proc = px.DataFrame(table='process_stats', start_time='" + darkProcStatsWindow + "')\n" + "proc.pod = proc.ctx['pod']\n" + "proc.namespace = proc.ctx['namespace']\n" + + // node resolved from the SAME process_stats upid the pod/ns come from, so + // dark-vector rows (dc_snoop et al.) carry hostname and become px-readable + // (#136). Transient attack pids that miss process_stats resolve blank — + // the same accepted limitation as pod/ns above. + "proc.node = px.upid_to_node_name(proc.upid)\n" + "proc.pid = px.upid_to_pid(proc.upid)\n" + - "proc = proc.groupby(['pod', 'namespace', 'pid']).agg()\n" + - "df = df.merge(proc, how='left', left_on=['pid'], right_on=['pid'], suffixes=['', '_x'])\n" + "proc = proc.groupby(['pod', 'namespace', 'node', 'pid']).agg()\n" + + "df = df.merge(proc, how='left', left_on=['pid'], right_on=['pid'], suffixes=['', '_x'])\n" + + "df.hostname = df.node\n" } return "df.namespace = px.upid_to_namespace(df.upid)\n" + "df.pod = px.upid_to_pod_name(df.upid)\n" + From 57268e1fbb75209764344a1652c94a3784ddd2da Mon Sep 17 00:00:00 2001 From: entlein Date: Tue, 18 Aug 2026 22:38:29 +0200 Subject: [PATCH 31/66] px/dx_evidence_graph: multi-panel pre-correlation dashboard (#136) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Kill-chain graph + one panel per protocol (kubescape/redis/conn/http/dns/pgsql/ mysql), each interval-joined (pod + row_time in [lo,hi]) to the order_id selected from the Orders panel. Orders now come from dx_anomaly_orders backed by dx_order_seeds (one order per anomaly — no evidence lost to coalescing). --- .../dx_evidence_graph/dx_evidence_graph.pxl | 54 +++++++- src/pxl_scripts/px/dx_evidence_graph/vis.json | 121 ++++-------------- 2 files changed, 70 insertions(+), 105 deletions(-) diff --git a/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl b/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl index 201b8c0ddb1..0f026350a0d 100644 --- a/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl +++ b/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl @@ -17,16 +17,56 @@ import px +# _consulted: interval-join one protocol's rows to a chosen order — equi-join on +# pod + row_time within the order's [lo,hi] window. This is the pre-correlation +# record set per protocol for the selected anomaly (order_id). +def _consulted(start_time: str, clickhouse_dsn: str, view: str, order_id: str): + orders = px.DataFrame('dx_anomaly_orders', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + orders = orders[orders.order_id == order_id] + orders = orders[['order_id', 'pod', 'lo', 'hi']] + src = px.DataFrame(view, clickhouse_dsn=clickhouse_dsn, start_time=start_time) + j = src.merge(orders, how='inner', left_on=['pod'], right_on=['pod'], suffixes=['', '_ord']) + j = j[j.row_time >= j.lo] + j = j[j.row_time <= j.hi] + return j + + def evidence_graph(start_time: str, clickhouse_dsn: str, table: str): df = px.DataFrame(table, clickhouse_dsn=clickhouse_dsn, start_time=start_time) df.from_entity = px.Pod(df.subject_pod) df.to_entity = df.target - return df[['from_entity', 'to_entity', - 'uniqueID', 'rule', 'process', 'target', 'target_kind', - 'severity', 'alert', 'subject_pod']] + return df[['from_entity', 'to_entity', 'uniqueID', 'rule', 'process', + 'target', 'target_kind', 'severity', 'alert', 'subject_pod']] -def consulted_records(start_time: str, clickhouse_dsn: str, table: str, uniqueID: str): - df = px.DataFrame(table, clickhouse_dsn=clickhouse_dsn, start_time=start_time) - df = df[px.contains(df.uniqueID, uniqueID)] - return df[['uniqueID', 'vector', 'source', 'src_table', 'record']] +def kubescape(start_time: str, clickhouse_dsn: str, order_id: str): + return _consulted(start_time, clickhouse_dsn, 'dx_src__kubescape_logs', order_id) + + +def redis(start_time: str, clickhouse_dsn: str, order_id: str): + return _consulted(start_time, clickhouse_dsn, 'dx_src__redis_events', order_id) + + +def conn(start_time: str, clickhouse_dsn: str, order_id: str): + return _consulted(start_time, clickhouse_dsn, 'dx_src__conn_stats', order_id) + + +def http(start_time: str, clickhouse_dsn: str, order_id: str): + return _consulted(start_time, clickhouse_dsn, 'dx_src__http_events', order_id) + + +def dns(start_time: str, clickhouse_dsn: str, order_id: str): + return _consulted(start_time, clickhouse_dsn, 'dx_src__dns_events', order_id) + + +def pgsql(start_time: str, clickhouse_dsn: str, order_id: str): + return _consulted(start_time, clickhouse_dsn, 'dx_src__pgsql_events', order_id) + + +def mysql(start_time: str, clickhouse_dsn: str, order_id: str): + return _consulted(start_time, clickhouse_dsn, 'dx_src__mysql_events', order_id) + + +def orders(start_time: str, clickhouse_dsn: str): + df = px.DataFrame('dx_anomaly_orders', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + return df[['uniqueID', 'rule', 'pod', 'order_id', 'lo', 'hi']] diff --git a/src/pxl_scripts/px/dx_evidence_graph/vis.json b/src/pxl_scripts/px/dx_evidence_graph/vis.json index 4642e0e4e05..7ef6512953b 100644 --- a/src/pxl_scripts/px/dx_evidence_graph/vis.json +++ b/src/pxl_scripts/px/dx_evidence_graph/vis.json @@ -1,106 +1,31 @@ { "variables": [ - { - "name": "start_time", - "type": "PX_STRING", - "description": "Start of the window.", - "defaultValue": "-30m" - }, - { - "name": "clickhouse_dsn", - "type": "PX_STRING", - "description": "ClickHouse DSN for forensic_db: user:pass@host:port/db.", - "defaultValue": "forensic_analyst:changeme-analyst@clickhouse-forensic-soc-db.clickhouse.svc.cluster.local:9000/forensic_db" - }, - { - "name": "graph_table", - "type": "PX_STRING", - "description": "L1 graph source: one edge per deduped kubescape anomaly (subject_pod -> target).", - "defaultValue": "dx_kubescape_anomalies" - }, - { - "name": "records_table", - "type": "PX_STRING", - "description": "L2 source: the pre-correlation consulted-records set (all evidence dx collected per anomaly).", - "defaultValue": "dx_anomaly_findings" - }, - { - "name": "uniqueID", - "type": "PX_STRING", - "description": "ZOOM: the anomaly uniqueID to expand into its consulted records. Copy it from a graph-edge hover or the anomalies table. Empty = all records in the window.", - "defaultValue": "" - } + {"name": "start_time", "type": "PX_STRING", "description": "Window start.", "defaultValue": "-6h"}, + {"name": "clickhouse_dsn", "type": "PX_STRING", "description": "forensic_db DSN: user:pass@host:port/db.", "defaultValue": "forensic_analyst:changeme-analyst@clickhouse-forensic-soc-db.clickhouse.svc.cluster.local:9000/forensic_db"}, + {"name": "graph_table", "type": "PX_STRING", "description": "L1 kill-chain graph source.", "defaultValue": "dx_kubescape_anomalies"}, + {"name": "order_id", "type": "PX_STRING", "description": "Pick an order_id from the Orders panel at the bottom; every protocol panel snaps to that order's consulted records.", "defaultValue": ""} ], "globalFuncs": [ - { - "outputName": "dx_graph", - "func": { - "name": "evidence_graph", - "args": [ - {"name": "start_time", "variable": "start_time"}, - {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, - {"name": "table", "variable": "graph_table"} - ] - } - }, - { - "outputName": "dx_records", - "func": { - "name": "consulted_records", - "args": [ - {"name": "start_time", "variable": "start_time"}, - {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, - {"name": "table", "variable": "records_table"}, - {"name": "uniqueID", "variable": "uniqueID"} - ] - } - } + {"outputName": "g_graph", "func": {"name": "evidence_graph", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "table", "variable": "graph_table"}]}}, + {"outputName": "g_kube", "func": {"name": "kubescape", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "order_id", "variable": "order_id"}]}}, + {"outputName": "g_redis", "func": {"name": "redis", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "order_id", "variable": "order_id"}]}}, + {"outputName": "g_conn", "func": {"name": "conn", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "order_id", "variable": "order_id"}]}}, + {"outputName": "g_http", "func": {"name": "http", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "order_id", "variable": "order_id"}]}}, + {"outputName": "g_dns", "func": {"name": "dns", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "order_id", "variable": "order_id"}]}}, + {"outputName": "g_pgsql", "func": {"name": "pgsql", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "order_id", "variable": "order_id"}]}}, + {"outputName": "g_mysql", "func": {"name": "mysql", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "order_id", "variable": "order_id"}]}}, + {"outputName": "g_orders", "func": {"name": "orders", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}]}} ], "widgets": [ - { - "name": "DX Evidence Graph (subject pod -> target; double-click a pod node -> px/pod)", - "position": {"x": 0, "y": 0, "w": 12, "h": 6}, - "globalFuncOutputName": "dx_graph", - "displaySpec": { - "@type": "types.px.dev/px.vispb.Graph", - "adjacencyList": { - "fromColumn": "from_entity", - "toColumn": "to_entity" - }, - "edgeWeightColumn": "severity", - "edgeColorColumn": "severity", - "edgeLabelColumn": "rule", - "edgeThresholds": { - "mediumThreshold": 5, - "highThreshold": 8 - }, - "edgeHoverInfo": [ - "uniqueID", - "rule", - "process", - "target", - "target_kind", - "severity", - "alert" - ], - "edgeLength": 500 - } - }, - { - "name": "Consulted records (zoom: set uniqueID) - the full pre-correlation evidence set per anomaly", - "position": {"x": 0, "y": 6, "w": 12, "h": 4}, - "globalFuncOutputName": "dx_records", - "displaySpec": { - "@type": "types.px.dev/px.vispb.Table" - } - }, - { - "name": "Anomalies (deduped kubescape_logs -> the graph edges; copy a uniqueID to zoom)", - "position": {"x": 0, "y": 10, "w": 12, "h": 4}, - "globalFuncOutputName": "dx_graph", - "displaySpec": { - "@type": "types.px.dev/px.vispb.Table" - } - } + {"name": "Kill-chain graph (subject pod -> target)", "position": {"x": 0, "y": 0, "w": 12, "h": 4}, "globalFuncOutputName": "g_graph", + "displaySpec": {"@type": "types.px.dev/px.vispb.Graph", "adjacencyList": {"fromColumn": "from_entity", "toColumn": "to_entity"}, "edgeWeightColumn": "severity", "edgeColorColumn": "severity", "edgeLabelColumn": "rule", "edgeThresholds": {"mediumThreshold": 5, "highThreshold": 8}, "edgeHoverInfo": ["uniqueID", "rule", "process", "target", "severity"], "edgeLength": 500}}, + {"name": "kubescape_logs (anomalies) for order", "position": {"x": 0, "y": 4, "w": 12, "h": 4}, "globalFuncOutputName": "g_kube", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, + {"name": "redis_events", "position": {"x": 0, "y": 8, "w": 6, "h": 4}, "globalFuncOutputName": "g_redis", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, + {"name": "conn_stats", "position": {"x": 6, "y": 8, "w": 6, "h": 4}, "globalFuncOutputName": "g_conn", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, + {"name": "http_events", "position": {"x": 0, "y": 12, "w": 6, "h": 4}, "globalFuncOutputName": "g_http", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, + {"name": "dns_events", "position": {"x": 6, "y": 12, "w": 6, "h": 4}, "globalFuncOutputName": "g_dns", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, + {"name": "pgsql_events", "position": {"x": 0, "y": 16, "w": 6, "h": 4}, "globalFuncOutputName": "g_pgsql", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, + {"name": "mysql_events", "position": {"x": 6, "y": 16, "w": 6, "h": 4}, "globalFuncOutputName": "g_mysql", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, + {"name": "ORDERS (pick an order_id -> paste into the order_id variable)", "position": {"x": 0, "y": 20, "w": 12, "h": 4}, "globalFuncOutputName": "g_orders", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}} ] } From 3f6b89ff0cfe00658285d62256bf3f5c842eebfb Mon Sep 17 00:00:00 2001 From: constanze Date: Tue, 18 Aug 2026 20:41:08 +0000 Subject: [PATCH 32/66] test(clickhouse): dx_order_seeds joins the OperatorOwnedTables tail guard The ordering guard pins the operator's own write targets as the trailing slice of OperatorOwnedTables. Adding dx_order_seeds shifted that window by one, so the guard compared a slice starting at trigger_watermark and failed on 49b0c2dc1. Extend want with the new table. Also clears every pre-existing golangci finding in this tree and regenerates controller/BUILD.bazel with gazelle, so arc lint is clean over src/vizier/services/adaptive_export/: named returns in controller.go and control/tls.go, deadlineErr -> errDeadline, predeclared 'max' in trigger/dedup.go, order_chunk_test.go missing from the test srcs. The 165 remaining repo lint errors are yamllint flow-style findings in the k8s/ and skaffold/ manifests, untouched here. --- .../adaptive_export/internal/clickhouse/apply_test.go | 2 +- .../services/adaptive_export/internal/control/tls.go | 6 +++--- .../adaptive_export/internal/controller/BUILD.bazel | 2 ++ .../adaptive_export/internal/controller/controller.go | 4 ++-- .../internal/controller/order_chunk_test.go | 10 +++++----- .../services/adaptive_export/internal/trigger/dedup.go | 8 ++++---- 6 files changed, 17 insertions(+), 15 deletions(-) diff --git a/src/vizier/services/adaptive_export/internal/clickhouse/apply_test.go b/src/vizier/services/adaptive_export/internal/clickhouse/apply_test.go index 3c834d11a80..32e4a796237 100644 --- a/src/vizier/services/adaptive_export/internal/clickhouse/apply_test.go +++ b/src/vizier/services/adaptive_export/internal/clickhouse/apply_test.go @@ -228,7 +228,7 @@ func TestOperatorOwnedTables_DoesNotIncludeKubescape(t *testing.T) { // plugin can auto-DDL them with the wrong schema), then the operator's // own write targets in declared order. func TestOperatorOwnedTables_TrailingOperatorTables(t *testing.T) { - want := []string{"adaptive_attribution", "trigger_watermark", "ae_reconcile", "dx_evidence_graph", "dx_evidence_graph_malignant", "dx_evidence_manifest"} + want := []string{"adaptive_attribution", "trigger_watermark", "ae_reconcile", "dx_evidence_graph", "dx_evidence_graph_malignant", "dx_evidence_manifest", "dx_order_seeds"} got := OperatorOwnedTables[len(OperatorOwnedTables)-len(want):] for i, w := range want { if got[i] != w { diff --git a/src/vizier/services/adaptive_export/internal/control/tls.go b/src/vizier/services/adaptive_export/internal/control/tls.go index 3e994e3bf7b..cc59c8455c1 100644 --- a/src/vizier/services/adaptive_export/internal/control/tls.go +++ b/src/vizier/services/adaptive_export/internal/control/tls.go @@ -110,11 +110,11 @@ func selfSignedCert(hostnames ...string) (tls.Certificate, error) { // certToPEM renders a tls.Certificate (as produced by selfSignedCert, holding a // single DER cert + an *ecdsa.PrivateKey) as PEM cert + PEM key bytes — the // on-disk shape of a mounted /certs/server.{crt,key} keypair. -func certToPEM(cert tls.Certificate) (certPEM, keyPEM []byte, err error) { +func certToPEM(cert tls.Certificate) ([]byte, []byte, error) { if len(cert.Certificate) == 0 { return nil, nil, fmt.Errorf("certToPEM: empty certificate chain") } - certPEM = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: cert.Certificate[0]}) + certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: cert.Certificate[0]}) ec, ok := cert.PrivateKey.(*ecdsa.PrivateKey) if !ok { return nil, nil, fmt.Errorf("certToPEM: private key is not *ecdsa.PrivateKey") @@ -123,6 +123,6 @@ func certToPEM(cert tls.Certificate) (certPEM, keyPEM []byte, err error) { if err != nil { return nil, nil, fmt.Errorf("marshal ec private key: %w", err) } - keyPEM = pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: der}) + keyPEM := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: der}) return certPEM, keyPEM, nil } diff --git a/src/vizier/services/adaptive_export/internal/controller/BUILD.bazel b/src/vizier/services/adaptive_export/internal/controller/BUILD.bazel index 6024b7ce5c7..dc4bdca23b1 100644 --- a/src/vizier/services/adaptive_export/internal/controller/BUILD.bazel +++ b/src/vizier/services/adaptive_export/internal/controller/BUILD.bazel @@ -36,12 +36,14 @@ pl_go_test( name = "controller_test", srcs = [ "controller_test.go", + "order_chunk_test.go", "order_query_test.go", ], embed = [":controller"], deps = [ "//src/vizier/services/adaptive_export/internal/anomaly", "//src/vizier/services/adaptive_export/internal/kubescape", + "//src/vizier/services/adaptive_export/internal/reconcile", "//src/vizier/services/adaptive_export/internal/sink", ], ) diff --git a/src/vizier/services/adaptive_export/internal/controller/controller.go b/src/vizier/services/adaptive_export/internal/controller/controller.go index 1ccf6c4f725..01e317359c0 100644 --- a/src/vizier/services/adaptive_export/internal/controller/controller.go +++ b/src/vizier/services/adaptive_export/internal/controller/controller.go @@ -359,7 +359,7 @@ func (c *Controller) OrderQuery(target anomaly.Target, table string, start, end // captureSpan captures [start,end) for one table, subdividing a transient failure into // half-spans down to orderMinChunk. Bounded by maxOrderSplitDepth + the breaker so a // saturated PEM isn't stormed; overlapping retries dedupe in the ReplacingMergeTree tables. -func (c *Controller) captureSpan(target anomaly.Target, table string, start, end time.Time, queryID string, depth int) (readCount, wroteCount int, err error) { +func (c *Controller) captureSpan(target anomaly.Target, table string, start, end time.Time, queryID string, depth int) (int, int, error) { r, w, e := c.orderQuerySlice(target, table, start, end, queryID) if e == nil || !isRetriableSpanErr(e) || end.Sub(start) <= orderMinChunk { return r, w, e @@ -387,7 +387,7 @@ func (c *Controller) captureSpan(target anomaly.Target, table string, start, end // orderQuerySlice runs one (target, table, [start,end)) capture and writes the rows. // It records no reconcile row — the OrderQuery driver aggregates and records once. -func (c *Controller) orderQuerySlice(target anomaly.Target, table string, start, end time.Time, queryID string) (readCount, wroteCount int, err error) { +func (c *Controller) orderQuerySlice(target anomaly.Target, table string, start, end time.Time, queryID string) (int, int, error) { now := c.clock.Now() q, qerr := pxl.QueryFor(table, target, start, end, now) if qerr != nil { diff --git a/src/vizier/services/adaptive_export/internal/controller/order_chunk_test.go b/src/vizier/services/adaptive_export/internal/controller/order_chunk_test.go index a949767cf90..d9d0acb4921 100644 --- a/src/vizier/services/adaptive_export/internal/controller/order_chunk_test.go +++ b/src/vizier/services/adaptive_export/internal/controller/order_chunk_test.go @@ -84,7 +84,7 @@ func chunkCtl(snk Sink, q PixieQuerier, rec reconcile.Recorder, chunk time.Durat return c } -var deadlineErr = errors.New("rpc error: code = DeadlineExceeded desc = context deadline exceeded") +var errDeadline = errors.New("rpc error: code = DeadlineExceeded desc = context deadline exceeded") // A wide window is walked in OrderChunk-sized slices: one pixie query per chunk, // each writing its rows. 180s window / 60s chunk = 3 bounded queries. @@ -134,7 +134,7 @@ func TestOrderQuerySingleReconcileRowPerTable(t *testing.T) { // call, so the initial full-chunk query subdivides and the halves succeed. func TestCaptureSpanSubdividesOnTransientError(t *testing.T) { snk := newRecordingSink() - q := &countingQuerier{rows: []map[string]any{{"comm": "getent"}}, failN: 1, failErr: deadlineErr} + q := &countingQuerier{rows: []map[string]any{{"comm": "getent"}}, failN: 1, failErr: errDeadline} end := canonicalEventTime start := end.Add(-8 * time.Second) // single 60s chunk covers it → one initial query if err := chunkCtl(snk, q, reconcile.Nop{}, 60*time.Second). @@ -171,7 +171,7 @@ func TestCaptureSpanDoesNotSplitNonTransient(t *testing.T) { // the error instead of looping forever — the recursion terminates at the floor. func TestCaptureSpanTerminatesAtMinChunk(t *testing.T) { snk := newRecordingSink() - q := &countingQuerier{failAll: true, failErr: deadlineErr} + q := &countingQuerier{failAll: true, failErr: errDeadline} end := canonicalEventTime start := end.Add(-4 * time.Second) // 4s → 2s → 1s (floor), bounded call count err := chunkCtl(snk, q, reconcile.Nop{}, 60*time.Second). @@ -191,7 +191,7 @@ func TestCaptureSpanTerminatesAtMinChunk(t *testing.T) { // subdividing after orderBreakerTrip consecutive timeouts) bound it hard. func TestOrderQueryCircuitBreakerBoundsStorm(t *testing.T) { snk := newRecordingSink() - q := &countingQuerier{failAll: true, failErr: deadlineErr} + q := &countingQuerier{failAll: true, failErr: errDeadline} end := canonicalEventTime start := end.Add(-600 * time.Second) // 10 chunks @ 60s, all time out _ = chunkCtl(snk, q, reconcile.Nop{}, 60*time.Second). @@ -210,7 +210,7 @@ func TestOrderQueryCircuitBreakerBoundsStorm(t *testing.T) { // still subdivides and recovers (breaker reset by the success). func TestCircuitBreakerResetsOnSuccess(t *testing.T) { snk := newRecordingSink() - q := &countingQuerier{rows: []map[string]any{{"comm": "cat"}}, failN: 1, failErr: deadlineErr} + q := &countingQuerier{rows: []map[string]any{{"comm": "cat"}}, failN: 1, failErr: errDeadline} end := canonicalEventTime start := end.Add(-8 * time.Second) if err := chunkCtl(snk, q, reconcile.Nop{}, 60*time.Second). diff --git a/src/vizier/services/adaptive_export/internal/trigger/dedup.go b/src/vizier/services/adaptive_export/internal/trigger/dedup.go index b1a0d587406..ca1c5dda9a0 100644 --- a/src/vizier/services/adaptive_export/internal/trigger/dedup.go +++ b/src/vizier/services/adaptive_export/internal/trigger/dedup.go @@ -47,11 +47,11 @@ type dedupEntry struct { evn uint64 // normalized event_time (nanos) } -func newDedupLRU(max int) *dedupLRU { - if max <= 0 { - max = 1 +func newDedupLRU(capacity int) *dedupLRU { + if capacity <= 0 { + capacity = 1 } - return &dedupLRU{max: max, ll: list.New(), items: map[string]*list.Element{}} + return &dedupLRU{max: capacity, ll: list.New(), items: map[string]*list.Element{}} } // Contains reports whether fp was Added and not yet evicted. From 133a2263529f2ea9de40aab376d0e64510308d33 Mon Sep 17 00:00:00 2001 From: entlein Date: Wed, 19 Aug 2026 11:36:54 +0200 Subject: [PATCH 33/66] ae: bake the order-UUID pre-correlation views into the image (#136) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A clean deploy now provisions the whole px/dx_evidence_graph dashboard with zero hand-applied SQL. schema.sql gains 10 CREATE VIEW IF NOT EXISTS statements (dx_anomaly_orders, dx_kubescape_anomalies, dx_src__{kubescape_logs, redis_events, conn_stats, http_events, dns_events, pgsql_events, mysql_events, dc_snoop}); registered in KnownTables + OperatorOwnedTables (appended last). Ordering (Apply is fatal on a missing base): every base table is created before its view — the socket_tracer tables + dc_snoop + dx_order_seeds already boot first, and kubescape_logs is now ENSURED in OperatorOwnedTables (CREATE TABLE IF NOT EXISTS, idempotent with the soc installer) before its two views. px read contract upheld per view (event_time UInt64 + hostname, no Bool cols). Tail-guard test extended with the 10 views. Views validated live on 6a841cf7 before it dropped. --- .../internal/clickhouse/apply.go | 18 +++++ .../internal/clickhouse/apply_test.go | 3 +- .../internal/clickhouse/ddl.go | 13 +++ .../internal/clickhouse/schema.sql | 81 +++++++++++++++++++ 4 files changed, 114 insertions(+), 1 deletion(-) diff --git a/src/vizier/services/adaptive_export/internal/clickhouse/apply.go b/src/vizier/services/adaptive_export/internal/clickhouse/apply.go index 0c09c0a564d..d6ff32d8b33 100644 --- a/src/vizier/services/adaptive_export/internal/clickhouse/apply.go +++ b/src/vizier/services/adaptive_export/internal/clickhouse/apply.go @@ -33,6 +33,11 @@ import ( // reference any pixie table during creation (it does not, but the // invariant is cheap to keep). var OperatorOwnedTables = []string{ + // kubescape_logs — normally soc-owned, but ENSURED here (CREATE TABLE IF NOT + // EXISTS is idempotent; whoever boots first wins, both use the canonical schema) + // so the two order-UUID views over it (dx_kubescape_anomalies, dx_src__kubescape_ + // logs, appended last) can be created at boot without a fatal missing-base error. + "kubescape_logs", // 12 pixie socket_tracer tables — created BEFORE Pixie's retention // plugin gets a chance to auto-DDL them (which would omit our // namespace + pod columns and break analyst JOINs). @@ -86,6 +91,19 @@ var OperatorOwnedTables = []string{ // dx per-referral order seeds (#136 evidence-loss fix) — created on boot so // dx's direct INSERT has a target. Not a pixie table → not in PixieTables(). "dx_order_seeds", + // order-UUID pre-correlation VIEWS (#136) — created LAST, after every base + // table above exists (kubescape_logs, the socket_tracer tables, dc_snoop, + // dx_order_seeds). Read by the px/dx_evidence_graph dashboard. Not pixie tables. + "dx_anomaly_orders", + "dx_kubescape_anomalies", + "dx_src__kubescape_logs", + "dx_src__redis_events", + "dx_src__conn_stats", + "dx_src__http_events", + "dx_src__dns_events", + "dx_src__pgsql_events", + "dx_src__mysql_events", + "dx_src__dc_snoop", } // Applier applies operator-owned DDL to a ClickHouse cluster over the diff --git a/src/vizier/services/adaptive_export/internal/clickhouse/apply_test.go b/src/vizier/services/adaptive_export/internal/clickhouse/apply_test.go index 32e4a796237..4dcc56b55ff 100644 --- a/src/vizier/services/adaptive_export/internal/clickhouse/apply_test.go +++ b/src/vizier/services/adaptive_export/internal/clickhouse/apply_test.go @@ -228,7 +228,8 @@ func TestOperatorOwnedTables_DoesNotIncludeKubescape(t *testing.T) { // plugin can auto-DDL them with the wrong schema), then the operator's // own write targets in declared order. func TestOperatorOwnedTables_TrailingOperatorTables(t *testing.T) { - want := []string{"adaptive_attribution", "trigger_watermark", "ae_reconcile", "dx_evidence_graph", "dx_evidence_graph_malignant", "dx_evidence_manifest", "dx_order_seeds"} + want := []string{"adaptive_attribution", "trigger_watermark", "ae_reconcile", "dx_evidence_graph", "dx_evidence_graph_malignant", "dx_evidence_manifest", "dx_order_seeds", + "dx_anomaly_orders", "dx_kubescape_anomalies", "dx_src__kubescape_logs", "dx_src__redis_events", "dx_src__conn_stats", "dx_src__http_events", "dx_src__dns_events", "dx_src__pgsql_events", "dx_src__mysql_events", "dx_src__dc_snoop"} got := OperatorOwnedTables[len(OperatorOwnedTables)-len(want):] for i, w := range want { if got[i] != w { diff --git a/src/vizier/services/adaptive_export/internal/clickhouse/ddl.go b/src/vizier/services/adaptive_export/internal/clickhouse/ddl.go index 24e720fa458..49d3450c332 100644 --- a/src/vizier/services/adaptive_export/internal/clickhouse/ddl.go +++ b/src/vizier/services/adaptive_export/internal/clickhouse/ddl.go @@ -88,6 +88,19 @@ var KnownTables = []string{ // INSERTs one row per anomaly so dx_anomaly_orders can window every uniqueID. // NOT a pixie table. "dx_order_seeds", + // order-UUID pre-correlation views (#136) read by the px/dx_evidence_graph + // dashboard. VIEWS, created after their base tables (kubescape_logs ensured + // first). NOT pixie tables. Order matches schema.sql (appended at the end). + "dx_anomaly_orders", + "dx_kubescape_anomalies", + "dx_src__kubescape_logs", + "dx_src__redis_events", + "dx_src__conn_stats", + "dx_src__http_events", + "dx_src__dns_events", + "dx_src__pgsql_events", + "dx_src__mysql_events", + "dx_src__dc_snoop", } // ErrUnknownTable is returned by DDL / Columns when asked for a table diff --git a/src/vizier/services/adaptive_export/internal/clickhouse/schema.sql b/src/vizier/services/adaptive_export/internal/clickhouse/schema.sql index bfec589f2dc..55ce2764560 100644 --- a/src/vizier/services/adaptive_export/internal/clickhouse/schema.sql +++ b/src/vizier/services/adaptive_export/internal/clickhouse/schema.sql @@ -710,3 +710,84 @@ CREATE TABLE IF NOT EXISTS forensic_db.creds_change ( hostname String, event_time DateTime64(9, 'UTC') ) ENGINE = ReplacingMergeTree ORDER BY (time_, pid, comm, old_uid, new_uid, pod); + +-- ── Order-UUID pre-correlation views (entlein/dx#136) ──────────────────────── +-- The px/dx_evidence_graph multi-panel dashboard reads these. Each is created on +-- boot AFTER its base table (Apply is fatal on a missing base): all bases are +-- OperatorOwned, and kubescape_logs is ensured in OperatorOwnedTables just before +-- these views. px read contract: expose event_time UInt64 + hostname + NO Bool cols; +-- ts=toString(time_) readable, row_time Int64 ns for the PxL interval-join. Views +-- are not pixie socket_tracer tables → absent from PixieTables(). + +-- dx_anomaly_orders: one order per anomaly (event_time ± 300s window), order_id +-- content-addressed on (pod, lo, hi). From dx_order_seeds so EVERY uniqueID gets an +-- order (no coalescing loss). rule = the seed's rule_id. +CREATE VIEW IF NOT EXISTS forensic_db.dx_anomaly_orders AS +SELECT unique_id AS uniqueID, rule_id AS rule, pod, + toInt64(event_time) - 300000000000 AS lo, + toInt64(event_time) + 300000000000 AS hi, + lower(substring(hex(SHA256(concat('v1|', pod, '|', toString(toInt64(event_time) - 300000000000), '|', toString(toInt64(event_time) + 300000000000)))), 1, 32)) AS order_id, + hostname, event_time +FROM forensic_db.dx_order_seeds +LIMIT 1 BY unique_id; + +-- dx_kubescape_anomalies: L1 kill-chain graph (subject_pod -> target), deduped by uniqueID. +CREATE VIEW IF NOT EXISTS forensic_db.dx_kubescape_anomalies AS +SELECT JSONExtractString(BaseRuntimeMetadata, 'uniqueID') AS uniqueID, + concat(JSONExtractString(RuntimeK8sDetails, 'podNamespace'), '/', JSONExtractString(RuntimeK8sDetails, 'podName')) AS subject_pod, + RuleID AS rule, + JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'process'), 'name') AS process, + multiIf(JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'dns'), 'domain') != '', JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'dns'), 'domain'), JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'network'), 'dstIP') != '', JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'network'), 'dstIP'), JSONExtractString(JSONExtractRaw(BaseRuntimeMetadata, 'arguments'), 'path') != '', JSONExtractString(JSONExtractRaw(BaseRuntimeMetadata, 'arguments'), 'path'), JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'file'), 'name') != '', concat(JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'file'), 'directory'), '/', JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'file'), 'name')), 'unknown') AS target, + multiIf(JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'dns'), 'domain') != '', 'domain', JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'network'), 'dstIP') != '', 'endpoint', (JSONExtractString(JSONExtractRaw(BaseRuntimeMetadata, 'arguments'), 'path') != '') OR (JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'file'), 'name') != ''), 'file', 'other') AS target_kind, + toInt8OrZero(JSONExtractString(BaseRuntimeMetadata, 'severity')) AS severity, + message AS alert, hostname, event_time +FROM forensic_db.kubescape_logs +WHERE RuleID != '' AND JSONExtractString(BaseRuntimeMetadata, 'uniqueID') != '' +LIMIT 1 BY uniqueID; + +-- dx_src__kubescape_logs: anomaly detail (process tree comm/cmdline/pcomm) per panel. +CREATE VIEW IF NOT EXISTS forensic_db.dx_src__kubescape_logs AS +SELECT toString(fromUnixTimestamp64Nano(toInt64(event_time))) AS ts, toInt64(event_time) AS row_time, event_time, + RuleID, JSONExtractString(BaseRuntimeMetadata, 'uniqueID') AS uniqueID, + JSONExtractString(JSONExtractRaw(RuntimeProcessDetails, 'processTree'), 'comm') AS comm, + JSONExtractString(JSONExtractRaw(RuntimeProcessDetails, 'processTree'), 'pcomm') AS parent, + JSONExtractString(JSONExtractRaw(RuntimeProcessDetails, 'processTree'), 'cmdline') AS cmdline, + message AS alert, + concat(JSONExtractString(RuntimeK8sDetails, 'podNamespace'), '/', JSONExtractString(RuntimeK8sDetails, 'podName')) AS pod, hostname +FROM forensic_db.kubescape_logs WHERE RuleID != ''; + +-- dx_src__: original protocol schema + ts/row_time/event_time, encrypted/ssl dropped. +CREATE VIEW IF NOT EXISTS forensic_db.dx_src__redis_events AS +SELECT toString(time_) AS ts, toInt64(toUnixTimestamp64Nano(time_)) AS row_time, toUInt64(toUnixTimestamp64Nano(event_time)) AS event_time, + upid, namespace, pod, remote_addr, remote_port, trace_role, req_cmd, req_args, resp, latency, hostname +FROM forensic_db.redis_events; + +CREATE VIEW IF NOT EXISTS forensic_db.dx_src__conn_stats AS +SELECT toString(time_) AS ts, toInt64(toUnixTimestamp64Nano(time_)) AS row_time, toUInt64(toUnixTimestamp64Nano(event_time)) AS event_time, + upid, namespace, pod, remote_addr, remote_port, protocol, conn_open, conn_close, conn_active, bytes_sent, bytes_recv, hostname +FROM forensic_db.conn_stats; + +CREATE VIEW IF NOT EXISTS forensic_db.dx_src__http_events AS +SELECT toString(time_) AS ts, toInt64(toUnixTimestamp64Nano(time_)) AS row_time, toUInt64(toUnixTimestamp64Nano(event_time)) AS event_time, + upid, namespace, pod, remote_addr, remote_port, req_method, req_path, req_body, resp_status, resp_body, latency, hostname +FROM forensic_db.http_events; + +CREATE VIEW IF NOT EXISTS forensic_db.dx_src__dns_events AS +SELECT toString(time_) AS ts, toInt64(toUnixTimestamp64Nano(time_)) AS row_time, toUInt64(toUnixTimestamp64Nano(event_time)) AS event_time, + upid, namespace, pod, remote_addr, remote_port, req_body, resp_body, latency, hostname +FROM forensic_db.dns_events; + +CREATE VIEW IF NOT EXISTS forensic_db.dx_src__pgsql_events AS +SELECT toString(time_) AS ts, toInt64(toUnixTimestamp64Nano(time_)) AS row_time, toUInt64(toUnixTimestamp64Nano(event_time)) AS event_time, + upid, namespace, pod, remote_addr, remote_port, req, resp, latency, hostname +FROM forensic_db.pgsql_events; + +CREATE VIEW IF NOT EXISTS forensic_db.dx_src__mysql_events AS +SELECT toString(time_) AS ts, toInt64(toUnixTimestamp64Nano(time_)) AS row_time, toUInt64(toUnixTimestamp64Nano(event_time)) AS event_time, + upid, namespace, pod, remote_addr, remote_port, req_cmd, req_body, resp_status, resp_body, latency, hostname +FROM forensic_db.mysql_events; + +CREATE VIEW IF NOT EXISTS forensic_db.dx_src__dc_snoop AS +SELECT toString(time_) AS ts, toInt64(toUnixTimestamp64Nano(time_)) AS row_time, toUInt64(toUnixTimestamp64Nano(event_time)) AS event_time, + pid, comm, t, file, namespace, pod, container, hostname +FROM forensic_db.dc_snoop; From 51652bca5a388f9ad548b37fe4b9394783208571 Mon Sep 17 00:00:00 2001 From: entlein Date: Wed, 19 Aug 2026 12:08:55 +0200 Subject: [PATCH 34/66] =?UTF-8?q?ae:=20keep=20soc/AE=20ownership=20boundar?= =?UTF-8?q?y=20=E2=80=94=20views=20tolerate=20a=20not-ready=20base=20(#136?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit build-agent flagged two ownership guards red on aeprod68: adding kubescape_logs to OperatorOwnedTables crosses the soc-installer boundary (AE must never DDL a kubescape table). Correct call — keep the boundary. Drop the kubescape_logs ensure. The two views over it (dx_kubescape_anomalies, dx_src__kubescape_logs) rely on the standard deploy order (soc-stack creates kubescape_logs before AE boots). To remove the fragility, Apply now tolerates a VIEW create failure (log + continue) instead of fatal — a derived view whose base isn't ready yet must not crashloop the operator; the next boot retries once the base exists. Tables stay boot-critical (fatal). Validated live: aeprod68 clean deploy already auto-created all 10 views + dx_order_seeds (soc's kubescape_logs present), 22 anomalies -> 22 orders, dc_snoop hostname 6152 rows. --- .../adaptive_export/internal/clickhouse/apply.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/src/vizier/services/adaptive_export/internal/clickhouse/apply.go b/src/vizier/services/adaptive_export/internal/clickhouse/apply.go index d6ff32d8b33..78b205008b5 100644 --- a/src/vizier/services/adaptive_export/internal/clickhouse/apply.go +++ b/src/vizier/services/adaptive_export/internal/clickhouse/apply.go @@ -21,6 +21,7 @@ import ( "context" "encoding/json" "fmt" + "log" "strings" "px.dev/pixie/src/vizier/services/adaptive_export/internal/chhttp" @@ -33,11 +34,6 @@ import ( // reference any pixie table during creation (it does not, but the // invariant is cheap to keep). var OperatorOwnedTables = []string{ - // kubescape_logs — normally soc-owned, but ENSURED here (CREATE TABLE IF NOT - // EXISTS is idempotent; whoever boots first wins, both use the canonical schema) - // so the two order-UUID views over it (dx_kubescape_anomalies, dx_src__kubescape_ - // logs, appended last) can be created at boot without a fatal missing-base error. - "kubescape_logs", // 12 pixie socket_tracer tables — created BEFORE Pixie's retention // plugin gets a chance to auto-DDL them (which would omit our // namespace + pod columns and break analyst JOINs). @@ -136,6 +132,14 @@ func (a *Applier) Apply(ctx context.Context) error { return fmt.Errorf("apply: get DDL for %s: %w", table, err) } if err := a.execute(ctx, ddl); err != nil { + // Views are DERIVED + best-effort: a not-yet-ready base (e.g. the + // soc-owned kubescape_logs before its installer has run) must NOT fatal + // the boot. Log + continue; the next boot retries once the base exists. + // Tables stay boot-critical (fatal) — a missing operator table is real. + if strings.Contains(ddl, "CREATE VIEW") { + log.Printf("[ae] deferred view %s (base not ready?): %v", table, err) + continue + } return fmt.Errorf("apply: create %s: %w", table, err) } } From 53d5e3cdb87ecc4d2cd2dcac21de372a1168c4fb Mon Sep 17 00:00:00 2001 From: constanze Date: Wed, 19 Aug 2026 10:11:53 +0000 Subject: [PATCH 35/66] test(clickhouse): the kubescape-ownership guard matches CREATE TABLE, not a mention MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The guard scanned every DDL body for 'forensic_db.kubescape_logs', so the order-UUID views that legitimately SELECT FROM that table tripped it — reading a soc-owned table read as creating it. Ownership is about who issues the CREATE TABLE, so match that form. AE still never creates alerts or kubescape_logs; with 51652bca5 they are not in OperatorOwnedTables at all. Also gofumpt on the tail-guard want literal. arc lint is clean over src/vizier/services/adaptive_export/; the 163 remaining repo errors are the yamllint findings in k8s/ and skaffold/, untouched. --- .../internal/clickhouse/apply_test.go | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/src/vizier/services/adaptive_export/internal/clickhouse/apply_test.go b/src/vizier/services/adaptive_export/internal/clickhouse/apply_test.go index 4dcc56b55ff..a8124dc066a 100644 --- a/src/vizier/services/adaptive_export/internal/clickhouse/apply_test.go +++ b/src/vizier/services/adaptive_export/internal/clickhouse/apply_test.go @@ -67,10 +67,16 @@ func TestApply_ExecutesEveryOperatorOwnedTable(t *testing.T) { if !strings.Contains(bodies[len(bodies)-1], "forensic_db."+lastTable) { t.Fatalf("last DDL not for %s; got: %s", lastTable, bodies[len(bodies)-1]) } - // And ensure no kubescape DDL leaked through. + // And ensure no kubescape DDL leaked through. Match the CREATE TABLE form, + // not a bare mention: the order-UUID views (dx_kubescape_anomalies, + // dx_src__kubescape_logs) legitimately SELECT FROM forensic_db.kubescape_logs, + // so a substring check on the table name alone flags reading it as creating it. + // Ownership is about who issues the CREATE TABLE, which is still never AE. for _, b := range bodies { - if strings.Contains(b, "forensic_db.alerts") || strings.Contains(b, "forensic_db.kubescape_logs") { - t.Fatalf("operator's Apply must not create kubescape tables; got:\n%s", b) + for _, ks := range []string{"alerts", "kubescape_logs"} { + if strings.Contains(b, "CREATE TABLE IF NOT EXISTS forensic_db."+ks) { + t.Fatalf("operator's Apply must not create kubescape tables; got:\n%s", b) + } } } } @@ -228,8 +234,10 @@ func TestOperatorOwnedTables_DoesNotIncludeKubescape(t *testing.T) { // plugin can auto-DDL them with the wrong schema), then the operator's // own write targets in declared order. func TestOperatorOwnedTables_TrailingOperatorTables(t *testing.T) { - want := []string{"adaptive_attribution", "trigger_watermark", "ae_reconcile", "dx_evidence_graph", "dx_evidence_graph_malignant", "dx_evidence_manifest", "dx_order_seeds", - "dx_anomaly_orders", "dx_kubescape_anomalies", "dx_src__kubescape_logs", "dx_src__redis_events", "dx_src__conn_stats", "dx_src__http_events", "dx_src__dns_events", "dx_src__pgsql_events", "dx_src__mysql_events", "dx_src__dc_snoop"} + want := []string{ + "adaptive_attribution", "trigger_watermark", "ae_reconcile", "dx_evidence_graph", "dx_evidence_graph_malignant", "dx_evidence_manifest", "dx_order_seeds", + "dx_anomaly_orders", "dx_kubescape_anomalies", "dx_src__kubescape_logs", "dx_src__redis_events", "dx_src__conn_stats", "dx_src__http_events", "dx_src__dns_events", "dx_src__pgsql_events", "dx_src__mysql_events", "dx_src__dc_snoop", + } got := OperatorOwnedTables[len(OperatorOwnedTables)-len(want):] for i, w := range want { if got[i] != w { From 9f4c9a83f26695c93428b409723c4599045f9e4c Mon Sep 17 00:00:00 2001 From: entlein Date: Wed, 19 Aug 2026 18:39:18 +0200 Subject: [PATCH 36/66] px/dx_evidence_graph: add dc_snoop + stack_trace panels; rename to Evidence graph Adds file-access (dc_snoop) and native-profiler (stack_trace) panels to the multi-panel dashboard, each interval-joined to the selected order like the other protocols. Renames the top graph "Kill-chain" -> "Evidence graph". --- .../px/dx_evidence_graph/dx_evidence_graph.pxl | 8 ++++++++ src/pxl_scripts/px/dx_evidence_graph/vis.json | 8 ++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl b/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl index 0f026350a0d..f70730191a6 100644 --- a/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl +++ b/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl @@ -67,6 +67,14 @@ def mysql(start_time: str, clickhouse_dsn: str, order_id: str): return _consulted(start_time, clickhouse_dsn, 'dx_src__mysql_events', order_id) +def dc_snoop(start_time: str, clickhouse_dsn: str, order_id: str): + return _consulted(start_time, clickhouse_dsn, 'dx_src__dc_snoop', order_id) + + +def stack_trace(start_time: str, clickhouse_dsn: str, order_id: str): + return _consulted(start_time, clickhouse_dsn, 'dx_src__stack_trace', order_id) + + def orders(start_time: str, clickhouse_dsn: str): df = px.DataFrame('dx_anomaly_orders', clickhouse_dsn=clickhouse_dsn, start_time=start_time) return df[['uniqueID', 'rule', 'pod', 'order_id', 'lo', 'hi']] diff --git a/src/pxl_scripts/px/dx_evidence_graph/vis.json b/src/pxl_scripts/px/dx_evidence_graph/vis.json index 7ef6512953b..50a2b554b7a 100644 --- a/src/pxl_scripts/px/dx_evidence_graph/vis.json +++ b/src/pxl_scripts/px/dx_evidence_graph/vis.json @@ -14,10 +14,12 @@ {"outputName": "g_dns", "func": {"name": "dns", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "order_id", "variable": "order_id"}]}}, {"outputName": "g_pgsql", "func": {"name": "pgsql", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "order_id", "variable": "order_id"}]}}, {"outputName": "g_mysql", "func": {"name": "mysql", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "order_id", "variable": "order_id"}]}}, + {"outputName": "g_dcsnoop", "func": {"name": "dc_snoop", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "order_id", "variable": "order_id"}]}}, + {"outputName": "g_stack", "func": {"name": "stack_trace", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "order_id", "variable": "order_id"}]}}, {"outputName": "g_orders", "func": {"name": "orders", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}]}} ], "widgets": [ - {"name": "Kill-chain graph (subject pod -> target)", "position": {"x": 0, "y": 0, "w": 12, "h": 4}, "globalFuncOutputName": "g_graph", + {"name": "Evidence graph (subject pod -> target)", "position": {"x": 0, "y": 0, "w": 12, "h": 4}, "globalFuncOutputName": "g_graph", "displaySpec": {"@type": "types.px.dev/px.vispb.Graph", "adjacencyList": {"fromColumn": "from_entity", "toColumn": "to_entity"}, "edgeWeightColumn": "severity", "edgeColorColumn": "severity", "edgeLabelColumn": "rule", "edgeThresholds": {"mediumThreshold": 5, "highThreshold": 8}, "edgeHoverInfo": ["uniqueID", "rule", "process", "target", "severity"], "edgeLength": 500}}, {"name": "kubescape_logs (anomalies) for order", "position": {"x": 0, "y": 4, "w": 12, "h": 4}, "globalFuncOutputName": "g_kube", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, {"name": "redis_events", "position": {"x": 0, "y": 8, "w": 6, "h": 4}, "globalFuncOutputName": "g_redis", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, @@ -26,6 +28,8 @@ {"name": "dns_events", "position": {"x": 6, "y": 12, "w": 6, "h": 4}, "globalFuncOutputName": "g_dns", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, {"name": "pgsql_events", "position": {"x": 0, "y": 16, "w": 6, "h": 4}, "globalFuncOutputName": "g_pgsql", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, {"name": "mysql_events", "position": {"x": 6, "y": 16, "w": 6, "h": 4}, "globalFuncOutputName": "g_mysql", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, - {"name": "ORDERS (pick an order_id -> paste into the order_id variable)", "position": {"x": 0, "y": 20, "w": 12, "h": 4}, "globalFuncOutputName": "g_orders", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}} + {"name": "dc_snoop (file access)", "position": {"x": 0, "y": 20, "w": 6, "h": 4}, "globalFuncOutputName": "g_dcsnoop", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, + {"name": "stack_trace (profiler)", "position": {"x": 6, "y": 20, "w": 6, "h": 4}, "globalFuncOutputName": "g_stack", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, + {"name": "ORDERS (pick an order_id -> paste into the order_id variable)", "position": {"x": 0, "y": 24, "w": 12, "h": 4}, "globalFuncOutputName": "g_orders", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}} ] } From 38ac499673af89ce0105215974b32f67840b2478 Mon Sep 17 00:00:00 2001 From: entlein Date: Wed, 19 Aug 2026 18:58:29 +0200 Subject: [PATCH 37/66] px/dx_evidence_graph: hide plumbing columns from protocol panels _consulted drops row_time/event_time/lo/hi/order_id/pod_ord from every protocol panel's output (they remain in the views + the ORDERS panel). upid is dropped at the view level. Panels now show ts + namespace/pod + protocol content + hostname. --- src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl b/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl index f70730191a6..735e4bf56d6 100644 --- a/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl +++ b/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl @@ -28,7 +28,10 @@ def _consulted(start_time: str, clickhouse_dsn: str, view: str, order_id: str): j = src.merge(orders, how='inner', left_on=['pod'], right_on=['pod'], suffixes=['', '_ord']) j = j[j.row_time >= j.lo] j = j[j.row_time <= j.hi] - return j + # Keep these in the views + the ORDERS panel, but hide the join/plumbing columns + # from the protocol panels so they don't pollute the visual (ts stays as the + # human-readable time; the panel is already scoped to one order_id). + return j.drop(['row_time', 'event_time', 'lo', 'hi', 'order_id', 'pod_ord']) def evidence_graph(start_time: str, clickhouse_dsn: str, table: str): From e2be5abba1b5248aaeb114ef55cbdacc80100581 Mon Sep 17 00:00:00 2001 From: entlein Date: Wed, 19 Aug 2026 19:00:32 +0200 Subject: [PATCH 38/66] dx: pin dx-daemon image to rc21 (order-seed writer) --- k8s/vizier/dx/dx-daemon.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/vizier/dx/dx-daemon.yaml b/k8s/vizier/dx/dx-daemon.yaml index fdc0e2c944b..e19a84c6cf8 100644 --- a/k8s/vizier/dx/dx-daemon.yaml +++ b/k8s/vizier/dx/dx-daemon.yaml @@ -23,7 +23,7 @@ spec: # (WITHOUT -tiny — -tiny's pclntab stripping SIGSEGV'd under load). Passes the # obfuscation gate AND survives the kill-chain (restarts=0, 4 rounds). Carries the # evidence-manifest + DX_FOREST_PUSHDOWN code. - image: docker.io/entlein/dx-daemon:0.4.0-ssotforest-rc18 + image: docker.io/entlein/dx-daemon:0.4.0-ssotforest-rc21 ports: - { name: findings, containerPort: 9099, hostPort: 9099 } env: From 59ed1e0e6c9c3d6c818b218cf5f61fa424934a41 Mon Sep 17 00:00:00 2001 From: entlein Date: Wed, 19 Aug 2026 19:23:56 +0200 Subject: [PATCH 39/66] ae: dx_order_records bridge + dx-assigned order_id (#136 stamping) Supports the 1-order-per-kubescape-log stamping model: - dx_order_seeds gains an order_id column (dx now assigns it = hash(uniqueID)). - new dx_order_records table (schema.sql + KnownTables + OperatorOwnedTables): the records dx consulted per primary log, stamped with its order_id. px-readable (event_time UInt64 DEFAULT toUInt64(time_) + hostname, no Bool cols). - dx_anomaly_orders view now exposes the STORED order_id (1:1 with the log) instead of the SHA256(pod|lo|hi) window hash that collided for same-instant anomalies. - apply_test tail guard extended with dx_order_records. --- .../internal/clickhouse/apply.go | 3 ++ .../internal/clickhouse/apply_test.go | 2 +- .../internal/clickhouse/ddl.go | 4 ++ .../internal/clickhouse/schema.sql | 37 +++++++++++++++++-- 4 files changed, 41 insertions(+), 5 deletions(-) diff --git a/src/vizier/services/adaptive_export/internal/clickhouse/apply.go b/src/vizier/services/adaptive_export/internal/clickhouse/apply.go index 78b205008b5..2c3ac37297e 100644 --- a/src/vizier/services/adaptive_export/internal/clickhouse/apply.go +++ b/src/vizier/services/adaptive_export/internal/clickhouse/apply.go @@ -87,6 +87,9 @@ var OperatorOwnedTables = []string{ // dx per-referral order seeds (#136 evidence-loss fix) — created on boot so // dx's direct INSERT has a target. Not a pixie table → not in PixieTables(). "dx_order_seeds", + // consulted-records bridge (#136 stamping) — created on boot so dx's INSERT has + // a target. Not a pixie table → not in PixieTables(). + "dx_order_records", // order-UUID pre-correlation VIEWS (#136) — created LAST, after every base // table above exists (kubescape_logs, the socket_tracer tables, dc_snoop, // dx_order_seeds). Read by the px/dx_evidence_graph dashboard. Not pixie tables. diff --git a/src/vizier/services/adaptive_export/internal/clickhouse/apply_test.go b/src/vizier/services/adaptive_export/internal/clickhouse/apply_test.go index a8124dc066a..8d0ac3c8908 100644 --- a/src/vizier/services/adaptive_export/internal/clickhouse/apply_test.go +++ b/src/vizier/services/adaptive_export/internal/clickhouse/apply_test.go @@ -235,7 +235,7 @@ func TestOperatorOwnedTables_DoesNotIncludeKubescape(t *testing.T) { // own write targets in declared order. func TestOperatorOwnedTables_TrailingOperatorTables(t *testing.T) { want := []string{ - "adaptive_attribution", "trigger_watermark", "ae_reconcile", "dx_evidence_graph", "dx_evidence_graph_malignant", "dx_evidence_manifest", "dx_order_seeds", + "adaptive_attribution", "trigger_watermark", "ae_reconcile", "dx_evidence_graph", "dx_evidence_graph_malignant", "dx_evidence_manifest", "dx_order_seeds", "dx_order_records", "dx_anomaly_orders", "dx_kubescape_anomalies", "dx_src__kubescape_logs", "dx_src__redis_events", "dx_src__conn_stats", "dx_src__http_events", "dx_src__dns_events", "dx_src__pgsql_events", "dx_src__mysql_events", "dx_src__dc_snoop", } got := OperatorOwnedTables[len(OperatorOwnedTables)-len(want):] diff --git a/src/vizier/services/adaptive_export/internal/clickhouse/ddl.go b/src/vizier/services/adaptive_export/internal/clickhouse/ddl.go index 49d3450c332..4852b06f07a 100644 --- a/src/vizier/services/adaptive_export/internal/clickhouse/ddl.go +++ b/src/vizier/services/adaptive_export/internal/clickhouse/ddl.go @@ -88,6 +88,10 @@ var KnownTables = []string{ // INSERTs one row per anomaly so dx_anomaly_orders can window every uniqueID. // NOT a pixie table. "dx_order_seeds", + // order-UUID consulted-records bridge (#136 stamping): the records dx consulted + // per primary kubescape log, stamped with its order_id. dx INSERTs. NOT a pixie + // table. + "dx_order_records", // order-UUID pre-correlation views (#136) read by the px/dx_evidence_graph // dashboard. VIEWS, created after their base tables (kubescape_logs ensured // first). NOT pixie tables. Order matches schema.sql (appended at the end). diff --git a/src/vizier/services/adaptive_export/internal/clickhouse/schema.sql b/src/vizier/services/adaptive_export/internal/clickhouse/schema.sql index 55ce2764560..9c5be21a0ae 100644 --- a/src/vizier/services/adaptive_export/internal/clickhouse/schema.sql +++ b/src/vizier/services/adaptive_export/internal/clickhouse/schema.sql @@ -575,6 +575,7 @@ CREATE TABLE IF NOT EXISTS forensic_db.dx_evidence_manifest ( -- ReplacingMergeTree ORDER BY (unique_id, rule_id) dedups re-fires but keeps -- co-fired rules. NOT a pixie table. CREATE TABLE IF NOT EXISTS forensic_db.dx_order_seeds ( + order_id String, unique_id String, rule_id String, pod String, @@ -587,6 +588,32 @@ CREATE TABLE IF NOT EXISTS forensic_db.dx_order_seeds ( TTL toDateTime(fromUnixTimestamp64Nano(event_time)) + INTERVAL 30 DAY DELETE SETTINGS index_granularity = 8192; +-- dx_order_records — the STAMPED consulted set (entlein/dx#136 stamping model). dx +-- writes one row per (order_id, finding): each record it consulted during the workup +-- for a primary kubescape log, stamped with that log's order_id. The panels read THIS +-- (the exact consulted set) instead of a ±300s time window. event_time is derived from +-- time_ so px can read it (UInt64 + hostname, no Bool cols). AE owns the DDL; dx +-- INSERTs. ReplacingMergeTree collapses re-stamps of the same (order_id,row). +CREATE TABLE IF NOT EXISTS forensic_db.dx_order_records ( + order_id String, + unique_id String, + src_table String, + vector String, + source String, + time_ Int64, + pod String, + remote_addr String, + path String, + comm String, + dns_name String, + hostname String, + event_time UInt64 DEFAULT toUInt64(time_) +) ENGINE = ReplacingMergeTree() + ORDER BY (order_id, src_table, time_, pod, remote_addr, path, comm, dns_name) + PARTITION BY toYYYYMM(fromUnixTimestamp64Nano(event_time)) + TTL toDateTime(fromUnixTimestamp64Nano(event_time)) + INTERVAL 30 DAY DELETE + SETTINGS index_granularity = 8192; + -- ── dx dark-vector tracepoint tables (entlein/dx#126) ──────────────────────── -- Fed by AE-owned bpftrace UpsertTracepoint probes (constantly enabled, no TTL). -- Emit raw kernel pid+comm (NOT upid); namespace/pod enriched at pull time via a @@ -719,14 +746,16 @@ CREATE TABLE IF NOT EXISTS forensic_db.creds_change ( -- ts=toString(time_) readable, row_time Int64 ns for the PxL interval-join. Views -- are not pixie socket_tracer tables → absent from PixieTables(). --- dx_anomaly_orders: one order per anomaly (event_time ± 300s window), order_id --- content-addressed on (pod, lo, hi). From dx_order_seeds so EVERY uniqueID gets an --- order (no coalescing loss). rule = the seed's rule_id. +-- dx_anomaly_orders: ONE order per primary kubescape log (#136 stamping model). +-- order_id is dx-assigned = hash(uniqueID) — 1:1 with the log (stored on the seed), +-- NOT the window hash that collided for same-instant anomalies. lo/hi are kept for +-- reference (the ±300s span); the CONSULTED records for the order live in +-- dx_order_records, stamped with this order_id. CREATE VIEW IF NOT EXISTS forensic_db.dx_anomaly_orders AS SELECT unique_id AS uniqueID, rule_id AS rule, pod, toInt64(event_time) - 300000000000 AS lo, toInt64(event_time) + 300000000000 AS hi, - lower(substring(hex(SHA256(concat('v1|', pod, '|', toString(toInt64(event_time) - 300000000000), '|', toString(toInt64(event_time) + 300000000000)))), 1, 32)) AS order_id, + order_id, hostname, event_time FROM forensic_db.dx_order_seeds LIMIT 1 BY unique_id; From 095a17cf4c3892250462cde88f994e03b070b407 Mon Sep 17 00:00:00 2001 From: entlein Date: Wed, 19 Aug 2026 19:48:36 +0200 Subject: [PATCH 40/66] ae: bake dx_src__stack_trace view + drop upid from the dx_src__ views (#136) Compiles the two hand-applied rig fixes into the image so a fresh deploy reproduces them: (1) new dx_src__stack_trace view (native profiler; registered in KnownTables + OperatorOwnedTables + apply_test tail); (2) drop the raw binary upid column from the six socket_tracer dx_src__ views (it rendered as garbage; namespace/pod carry the identity). Each iteration builds on the previous schema.sql. --- .../internal/clickhouse/apply.go | 1 + .../internal/clickhouse/apply_test.go | 2 +- .../adaptive_export/internal/clickhouse/ddl.go | 1 + .../internal/clickhouse/schema.sql | 17 +++++++++++------ 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/vizier/services/adaptive_export/internal/clickhouse/apply.go b/src/vizier/services/adaptive_export/internal/clickhouse/apply.go index 2c3ac37297e..f65e8b05d22 100644 --- a/src/vizier/services/adaptive_export/internal/clickhouse/apply.go +++ b/src/vizier/services/adaptive_export/internal/clickhouse/apply.go @@ -103,6 +103,7 @@ var OperatorOwnedTables = []string{ "dx_src__pgsql_events", "dx_src__mysql_events", "dx_src__dc_snoop", + "dx_src__stack_trace", } // Applier applies operator-owned DDL to a ClickHouse cluster over the diff --git a/src/vizier/services/adaptive_export/internal/clickhouse/apply_test.go b/src/vizier/services/adaptive_export/internal/clickhouse/apply_test.go index 8d0ac3c8908..4c1291cee70 100644 --- a/src/vizier/services/adaptive_export/internal/clickhouse/apply_test.go +++ b/src/vizier/services/adaptive_export/internal/clickhouse/apply_test.go @@ -236,7 +236,7 @@ func TestOperatorOwnedTables_DoesNotIncludeKubescape(t *testing.T) { func TestOperatorOwnedTables_TrailingOperatorTables(t *testing.T) { want := []string{ "adaptive_attribution", "trigger_watermark", "ae_reconcile", "dx_evidence_graph", "dx_evidence_graph_malignant", "dx_evidence_manifest", "dx_order_seeds", "dx_order_records", - "dx_anomaly_orders", "dx_kubescape_anomalies", "dx_src__kubescape_logs", "dx_src__redis_events", "dx_src__conn_stats", "dx_src__http_events", "dx_src__dns_events", "dx_src__pgsql_events", "dx_src__mysql_events", "dx_src__dc_snoop", + "dx_anomaly_orders", "dx_kubescape_anomalies", "dx_src__kubescape_logs", "dx_src__redis_events", "dx_src__conn_stats", "dx_src__http_events", "dx_src__dns_events", "dx_src__pgsql_events", "dx_src__mysql_events", "dx_src__dc_snoop", "dx_src__stack_trace", } got := OperatorOwnedTables[len(OperatorOwnedTables)-len(want):] for i, w := range want { diff --git a/src/vizier/services/adaptive_export/internal/clickhouse/ddl.go b/src/vizier/services/adaptive_export/internal/clickhouse/ddl.go index 4852b06f07a..999b216c015 100644 --- a/src/vizier/services/adaptive_export/internal/clickhouse/ddl.go +++ b/src/vizier/services/adaptive_export/internal/clickhouse/ddl.go @@ -105,6 +105,7 @@ var KnownTables = []string{ "dx_src__pgsql_events", "dx_src__mysql_events", "dx_src__dc_snoop", + "dx_src__stack_trace", } // ErrUnknownTable is returned by DDL / Columns when asked for a table diff --git a/src/vizier/services/adaptive_export/internal/clickhouse/schema.sql b/src/vizier/services/adaptive_export/internal/clickhouse/schema.sql index 9c5be21a0ae..80ce42ed913 100644 --- a/src/vizier/services/adaptive_export/internal/clickhouse/schema.sql +++ b/src/vizier/services/adaptive_export/internal/clickhouse/schema.sql @@ -788,35 +788,40 @@ FROM forensic_db.kubescape_logs WHERE RuleID != ''; -- dx_src__: original protocol schema + ts/row_time/event_time, encrypted/ssl dropped. CREATE VIEW IF NOT EXISTS forensic_db.dx_src__redis_events AS SELECT toString(time_) AS ts, toInt64(toUnixTimestamp64Nano(time_)) AS row_time, toUInt64(toUnixTimestamp64Nano(event_time)) AS event_time, - upid, namespace, pod, remote_addr, remote_port, trace_role, req_cmd, req_args, resp, latency, hostname + namespace, pod, remote_addr, remote_port, trace_role, req_cmd, req_args, resp, latency, hostname FROM forensic_db.redis_events; CREATE VIEW IF NOT EXISTS forensic_db.dx_src__conn_stats AS SELECT toString(time_) AS ts, toInt64(toUnixTimestamp64Nano(time_)) AS row_time, toUInt64(toUnixTimestamp64Nano(event_time)) AS event_time, - upid, namespace, pod, remote_addr, remote_port, protocol, conn_open, conn_close, conn_active, bytes_sent, bytes_recv, hostname + namespace, pod, remote_addr, remote_port, protocol, conn_open, conn_close, conn_active, bytes_sent, bytes_recv, hostname FROM forensic_db.conn_stats; CREATE VIEW IF NOT EXISTS forensic_db.dx_src__http_events AS SELECT toString(time_) AS ts, toInt64(toUnixTimestamp64Nano(time_)) AS row_time, toUInt64(toUnixTimestamp64Nano(event_time)) AS event_time, - upid, namespace, pod, remote_addr, remote_port, req_method, req_path, req_body, resp_status, resp_body, latency, hostname + namespace, pod, remote_addr, remote_port, req_method, req_path, req_body, resp_status, resp_body, latency, hostname FROM forensic_db.http_events; CREATE VIEW IF NOT EXISTS forensic_db.dx_src__dns_events AS SELECT toString(time_) AS ts, toInt64(toUnixTimestamp64Nano(time_)) AS row_time, toUInt64(toUnixTimestamp64Nano(event_time)) AS event_time, - upid, namespace, pod, remote_addr, remote_port, req_body, resp_body, latency, hostname + namespace, pod, remote_addr, remote_port, req_body, resp_body, latency, hostname FROM forensic_db.dns_events; CREATE VIEW IF NOT EXISTS forensic_db.dx_src__pgsql_events AS SELECT toString(time_) AS ts, toInt64(toUnixTimestamp64Nano(time_)) AS row_time, toUInt64(toUnixTimestamp64Nano(event_time)) AS event_time, - upid, namespace, pod, remote_addr, remote_port, req, resp, latency, hostname + namespace, pod, remote_addr, remote_port, req, resp, latency, hostname FROM forensic_db.pgsql_events; CREATE VIEW IF NOT EXISTS forensic_db.dx_src__mysql_events AS SELECT toString(time_) AS ts, toInt64(toUnixTimestamp64Nano(time_)) AS row_time, toUInt64(toUnixTimestamp64Nano(event_time)) AS event_time, - upid, namespace, pod, remote_addr, remote_port, req_cmd, req_body, resp_status, resp_body, latency, hostname + namespace, pod, remote_addr, remote_port, req_cmd, req_body, resp_status, resp_body, latency, hostname FROM forensic_db.mysql_events; CREATE VIEW IF NOT EXISTS forensic_db.dx_src__dc_snoop AS SELECT toString(time_) AS ts, toInt64(toUnixTimestamp64Nano(time_)) AS row_time, toUInt64(toUnixTimestamp64Nano(event_time)) AS event_time, pid, comm, t, file, namespace, pod, container, hostname FROM forensic_db.dc_snoop; + +CREATE VIEW IF NOT EXISTS forensic_db.dx_src__stack_trace AS +SELECT toString(time_) AS ts, toInt64(toUnixTimestamp64Nano(time_)) AS row_time, toUInt64(toUnixTimestamp64Nano(event_time)) AS event_time, + namespace, pod, container, stack_trace_id, stack_trace, count, hostname +FROM forensic_db.stack_trace; From ddc7363db88ce1b507b4d20a41bdaec4a9fecc0b Mon Sep 17 00:00:00 2001 From: entlein Date: Wed, 19 Aug 2026 20:16:00 +0200 Subject: [PATCH 41/66] px/dx_evidence_graph: graph shows all alerts (process->target); panels read stamped dx_order_records per order Graph edge changed from (subject_pod,target) to (process,target) so co-target alerts (cat, ln both -> /etc/shadow) render as distinct edges: 5 alerts = 5 edges. Protocol panels now read forensic_db.dx_order_records filtered by order_id + src_table (the records dx stamped to the primary kubescape alert) instead of the +/-300s window-join on pod, which over-collected all orders sharing a pod (measured 27 redis rows vs 1 stamped for one order). kubescape panel keyed on the order's uniqueID = the single primary log. Adds dx_panels_test.sh: SQL assertion harness (R1 graph completeness, R2 per-order scoping + no foreign-order leakage) runnable against the forensic_db ClickHouse. --- .../dx_evidence_graph/dx_evidence_graph.pxl | 45 +++++++-------- .../px/dx_evidence_graph/dx_panels_test.sh | 55 +++++++++++++++++++ 2 files changed, 75 insertions(+), 25 deletions(-) create mode 100755 src/pxl_scripts/px/dx_evidence_graph/dx_panels_test.sh diff --git a/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl b/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl index 735e4bf56d6..e8b31901c3f 100644 --- a/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl +++ b/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl @@ -17,65 +17,60 @@ import px -# _consulted: interval-join one protocol's rows to a chosen order — equi-join on -# pod + row_time within the order's [lo,hi] window. This is the pre-correlation -# record set per protocol for the selected anomaly (order_id). -def _consulted(start_time: str, clickhouse_dsn: str, view: str, order_id: str): - orders = px.DataFrame('dx_anomaly_orders', clickhouse_dsn=clickhouse_dsn, start_time=start_time) - orders = orders[orders.order_id == order_id] - orders = orders[['order_id', 'pod', 'lo', 'hi']] - src = px.DataFrame(view, clickhouse_dsn=clickhouse_dsn, start_time=start_time) - j = src.merge(orders, how='inner', left_on=['pod'], right_on=['pod'], suffixes=['', '_ord']) - j = j[j.row_time >= j.lo] - j = j[j.row_time <= j.hi] - # Keep these in the views + the ORDERS panel, but hide the join/plumbing columns - # from the protocol panels so they don't pollute the visual (ts stays as the - # human-readable time; the panel is already scoped to one order_id). - return j.drop(['row_time', 'event_time', 'lo', 'hi', 'order_id', 'pod_ord']) +def _stamped(start_time: str, clickhouse_dsn: str, src_table: str, order_id: str): + df = px.DataFrame('dx_order_records', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + df = df[df.order_id == order_id] + df = df[df.src_table == src_table] + return df[['vector', 'source', 'pod', 'remote_addr', 'path', 'comm', 'dns_name']] def evidence_graph(start_time: str, clickhouse_dsn: str, table: str): df = px.DataFrame(table, clickhouse_dsn=clickhouse_dsn, start_time=start_time) - df.from_entity = px.Pod(df.subject_pod) + df.from_entity = df.process df.to_entity = df.target return df[['from_entity', 'to_entity', 'uniqueID', 'rule', 'process', 'target', 'target_kind', 'severity', 'alert', 'subject_pod']] def kubescape(start_time: str, clickhouse_dsn: str, order_id: str): - return _consulted(start_time, clickhouse_dsn, 'dx_src__kubescape_logs', order_id) + orders = px.DataFrame('dx_anomaly_orders', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + orders = orders[orders.order_id == order_id] + orders = orders[['order_id', 'uniqueID']] + k = px.DataFrame('dx_src__kubescape_logs', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + j = k.merge(orders, how='inner', left_on=['uniqueID'], right_on=['uniqueID'], suffixes=['', '_o']) + return j.drop(['order_id', 'uniqueID_o', 'row_time', 'event_time']) def redis(start_time: str, clickhouse_dsn: str, order_id: str): - return _consulted(start_time, clickhouse_dsn, 'dx_src__redis_events', order_id) + return _stamped(start_time, clickhouse_dsn, 'redis_events', order_id) def conn(start_time: str, clickhouse_dsn: str, order_id: str): - return _consulted(start_time, clickhouse_dsn, 'dx_src__conn_stats', order_id) + return _stamped(start_time, clickhouse_dsn, 'conn_stats', order_id) def http(start_time: str, clickhouse_dsn: str, order_id: str): - return _consulted(start_time, clickhouse_dsn, 'dx_src__http_events', order_id) + return _stamped(start_time, clickhouse_dsn, 'http_events', order_id) def dns(start_time: str, clickhouse_dsn: str, order_id: str): - return _consulted(start_time, clickhouse_dsn, 'dx_src__dns_events', order_id) + return _stamped(start_time, clickhouse_dsn, 'dns_events', order_id) def pgsql(start_time: str, clickhouse_dsn: str, order_id: str): - return _consulted(start_time, clickhouse_dsn, 'dx_src__pgsql_events', order_id) + return _stamped(start_time, clickhouse_dsn, 'pgsql_events', order_id) def mysql(start_time: str, clickhouse_dsn: str, order_id: str): - return _consulted(start_time, clickhouse_dsn, 'dx_src__mysql_events', order_id) + return _stamped(start_time, clickhouse_dsn, 'mysql_events', order_id) def dc_snoop(start_time: str, clickhouse_dsn: str, order_id: str): - return _consulted(start_time, clickhouse_dsn, 'dx_src__dc_snoop', order_id) + return _stamped(start_time, clickhouse_dsn, 'dc_snoop', order_id) def stack_trace(start_time: str, clickhouse_dsn: str, order_id: str): - return _consulted(start_time, clickhouse_dsn, 'dx_src__stack_trace', order_id) + return _stamped(start_time, clickhouse_dsn, 'stack_trace', order_id) def orders(start_time: str, clickhouse_dsn: str): diff --git a/src/pxl_scripts/px/dx_evidence_graph/dx_panels_test.sh b/src/pxl_scripts/px/dx_evidence_graph/dx_panels_test.sh new file mode 100755 index 00000000000..4bc20536739 --- /dev/null +++ b/src/pxl_scripts/px/dx_evidence_graph/dx_panels_test.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +set -euo pipefail + +CHPOD="${CHPOD:-chi-forensic-soc-db-soc-cluster-0-0-0}" +CHNS="${CHNS:-clickhouse}" +DB="${DB:-forensic_db}" + +q() { kubectl -n "$CHNS" exec -i "$CHPOD" -- clickhouse-client -q "$1" 2>/dev/null; } + +fail=0 +check() { + local name="$1" got="$2" want="$3" + if [ "$got" = "$want" ]; then + printf 'PASS %-52s got=%s\n' "$name" "$got" + else + printf 'FAIL %-52s got=%s want=%s\n' "$name" "$got" "$want" + fail=1 + fi +} + +alerts=$(q "SELECT uniqExact(uniqueID) FROM $DB.dx_kubescape_anomalies") +edges=$(q "SELECT uniqExact((process,target)) FROM $DB.dx_kubescape_anomalies") +check "R1 graph edges == distinct alerts" "$edges" "$alerts" + +seen=$(q "SELECT count() FROM $DB.dx_kubescape_anomalies g LEFT JOIN (SELECT DISTINCT uniqueID FROM $DB.dx_kubescape_anomalies) u USING (uniqueID) WHERE u.uniqueID = ''") +check "R1 every alert has an edge (no null uniqueID)" "$seen" "0" + +mapfile -t ORDERS < <(q "SELECT order_id FROM $DB.dx_anomaly_orders ORDER BY order_id") + +for oid in "${ORDERS[@]}"; do + uid=$(q "SELECT uniqueID FROM $DB.dx_anomaly_orders WHERE order_id='$oid' LIMIT 1") + + ks=$(q "SELECT count() FROM $DB.dx_src__kubescape_logs k INNER JOIN (SELECT uniqueID FROM $DB.dx_anomaly_orders WHERE order_id='$oid') o USING (uniqueID)") + ksbad=$(q "SELECT count() FROM $DB.dx_src__kubescape_logs k INNER JOIN (SELECT uniqueID FROM $DB.dx_anomaly_orders WHERE order_id='$oid') o USING (uniqueID) WHERE k.uniqueID != '$uid'") + check "R2 kubescape($oid) is the primary log only" "$ksbad" "0" + [ "$ks" -ge 1 ] && check "R2 kubescape($oid) present (>=1)" "1" "1" || check "R2 kubescape($oid) present (>=1)" "0" "1" + + for tbl in redis_events conn_stats dc_snoop http_events dns_events stack_trace; do + stamped=$(q "SELECT count() FROM $DB.dx_order_records WHERE order_id='$oid' AND src_table='$tbl'") + foreign=$(q "SELECT count() FROM $DB.dx_order_records WHERE order_id='$oid' AND src_table='$tbl' AND order_id != '$oid'") + check "R2 $tbl($oid) no foreign-order rows" "$foreign" "0" + done +done + +echo "--- leakage proof: window-join vs stamped for one order ---" +oid="${ORDERS[0]}" +win=$(q "SELECT count() FROM $DB.dx_src__redis_events s INNER JOIN (SELECT pod, lo, hi FROM $DB.dx_anomaly_orders WHERE order_id='$oid') o USING (pod) WHERE s.row_time BETWEEN o.lo AND o.hi") +stamped=$(q "SELECT count() FROM $DB.dx_order_records WHERE order_id='$oid' AND src_table='redis_events'") +printf 'window-join redis rows for %s = %s stamped rows = %s\n' "$oid" "$win" "$stamped" +if [ "$win" -gt "$stamped" ]; then + echo " -> confirms window over-collects (leaks other orders' pod rows); stamped is scoped." +fi + +echo +[ "$fail" -eq 0 ] && echo "ALL PASS" || { echo "FAILURES PRESENT"; exit 1; } From 407e196d6608ee4fd9415cc701d5a77a5401ae21 Mon Sep 17 00:00:00 2001 From: entlein Date: Wed, 19 Aug 2026 21:04:38 +0200 Subject: [PATCH 42/66] Revert ddc7363db: restore dx_evidence_graph.pxl to the agreed version ddc7363db rewrote the pxl (evidence_graph process->target edge, panels swapped from _consulted window-join to _stamped dx_order_records) and added a test file, and a cloud image was cut from it. That was not the requested change. This restores the pxl to commit 38ac49967 verbatim (the v0.0.25 cloud version) and removes the added test. Cloud tag release/cloud/v0.0.26-pre-v0.0 deleted from origin. --- .../dx_evidence_graph/dx_evidence_graph.pxl | 45 ++++++++------- .../px/dx_evidence_graph/dx_panels_test.sh | 55 ------------------- 2 files changed, 25 insertions(+), 75 deletions(-) delete mode 100755 src/pxl_scripts/px/dx_evidence_graph/dx_panels_test.sh diff --git a/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl b/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl index e8b31901c3f..735e4bf56d6 100644 --- a/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl +++ b/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl @@ -17,60 +17,65 @@ import px -def _stamped(start_time: str, clickhouse_dsn: str, src_table: str, order_id: str): - df = px.DataFrame('dx_order_records', clickhouse_dsn=clickhouse_dsn, start_time=start_time) - df = df[df.order_id == order_id] - df = df[df.src_table == src_table] - return df[['vector', 'source', 'pod', 'remote_addr', 'path', 'comm', 'dns_name']] +# _consulted: interval-join one protocol's rows to a chosen order — equi-join on +# pod + row_time within the order's [lo,hi] window. This is the pre-correlation +# record set per protocol for the selected anomaly (order_id). +def _consulted(start_time: str, clickhouse_dsn: str, view: str, order_id: str): + orders = px.DataFrame('dx_anomaly_orders', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + orders = orders[orders.order_id == order_id] + orders = orders[['order_id', 'pod', 'lo', 'hi']] + src = px.DataFrame(view, clickhouse_dsn=clickhouse_dsn, start_time=start_time) + j = src.merge(orders, how='inner', left_on=['pod'], right_on=['pod'], suffixes=['', '_ord']) + j = j[j.row_time >= j.lo] + j = j[j.row_time <= j.hi] + # Keep these in the views + the ORDERS panel, but hide the join/plumbing columns + # from the protocol panels so they don't pollute the visual (ts stays as the + # human-readable time; the panel is already scoped to one order_id). + return j.drop(['row_time', 'event_time', 'lo', 'hi', 'order_id', 'pod_ord']) def evidence_graph(start_time: str, clickhouse_dsn: str, table: str): df = px.DataFrame(table, clickhouse_dsn=clickhouse_dsn, start_time=start_time) - df.from_entity = df.process + df.from_entity = px.Pod(df.subject_pod) df.to_entity = df.target return df[['from_entity', 'to_entity', 'uniqueID', 'rule', 'process', 'target', 'target_kind', 'severity', 'alert', 'subject_pod']] def kubescape(start_time: str, clickhouse_dsn: str, order_id: str): - orders = px.DataFrame('dx_anomaly_orders', clickhouse_dsn=clickhouse_dsn, start_time=start_time) - orders = orders[orders.order_id == order_id] - orders = orders[['order_id', 'uniqueID']] - k = px.DataFrame('dx_src__kubescape_logs', clickhouse_dsn=clickhouse_dsn, start_time=start_time) - j = k.merge(orders, how='inner', left_on=['uniqueID'], right_on=['uniqueID'], suffixes=['', '_o']) - return j.drop(['order_id', 'uniqueID_o', 'row_time', 'event_time']) + return _consulted(start_time, clickhouse_dsn, 'dx_src__kubescape_logs', order_id) def redis(start_time: str, clickhouse_dsn: str, order_id: str): - return _stamped(start_time, clickhouse_dsn, 'redis_events', order_id) + return _consulted(start_time, clickhouse_dsn, 'dx_src__redis_events', order_id) def conn(start_time: str, clickhouse_dsn: str, order_id: str): - return _stamped(start_time, clickhouse_dsn, 'conn_stats', order_id) + return _consulted(start_time, clickhouse_dsn, 'dx_src__conn_stats', order_id) def http(start_time: str, clickhouse_dsn: str, order_id: str): - return _stamped(start_time, clickhouse_dsn, 'http_events', order_id) + return _consulted(start_time, clickhouse_dsn, 'dx_src__http_events', order_id) def dns(start_time: str, clickhouse_dsn: str, order_id: str): - return _stamped(start_time, clickhouse_dsn, 'dns_events', order_id) + return _consulted(start_time, clickhouse_dsn, 'dx_src__dns_events', order_id) def pgsql(start_time: str, clickhouse_dsn: str, order_id: str): - return _stamped(start_time, clickhouse_dsn, 'pgsql_events', order_id) + return _consulted(start_time, clickhouse_dsn, 'dx_src__pgsql_events', order_id) def mysql(start_time: str, clickhouse_dsn: str, order_id: str): - return _stamped(start_time, clickhouse_dsn, 'mysql_events', order_id) + return _consulted(start_time, clickhouse_dsn, 'dx_src__mysql_events', order_id) def dc_snoop(start_time: str, clickhouse_dsn: str, order_id: str): - return _stamped(start_time, clickhouse_dsn, 'dc_snoop', order_id) + return _consulted(start_time, clickhouse_dsn, 'dx_src__dc_snoop', order_id) def stack_trace(start_time: str, clickhouse_dsn: str, order_id: str): - return _stamped(start_time, clickhouse_dsn, 'stack_trace', order_id) + return _consulted(start_time, clickhouse_dsn, 'dx_src__stack_trace', order_id) def orders(start_time: str, clickhouse_dsn: str): diff --git a/src/pxl_scripts/px/dx_evidence_graph/dx_panels_test.sh b/src/pxl_scripts/px/dx_evidence_graph/dx_panels_test.sh deleted file mode 100755 index 4bc20536739..00000000000 --- a/src/pxl_scripts/px/dx_evidence_graph/dx_panels_test.sh +++ /dev/null @@ -1,55 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -CHPOD="${CHPOD:-chi-forensic-soc-db-soc-cluster-0-0-0}" -CHNS="${CHNS:-clickhouse}" -DB="${DB:-forensic_db}" - -q() { kubectl -n "$CHNS" exec -i "$CHPOD" -- clickhouse-client -q "$1" 2>/dev/null; } - -fail=0 -check() { - local name="$1" got="$2" want="$3" - if [ "$got" = "$want" ]; then - printf 'PASS %-52s got=%s\n' "$name" "$got" - else - printf 'FAIL %-52s got=%s want=%s\n' "$name" "$got" "$want" - fail=1 - fi -} - -alerts=$(q "SELECT uniqExact(uniqueID) FROM $DB.dx_kubescape_anomalies") -edges=$(q "SELECT uniqExact((process,target)) FROM $DB.dx_kubescape_anomalies") -check "R1 graph edges == distinct alerts" "$edges" "$alerts" - -seen=$(q "SELECT count() FROM $DB.dx_kubescape_anomalies g LEFT JOIN (SELECT DISTINCT uniqueID FROM $DB.dx_kubescape_anomalies) u USING (uniqueID) WHERE u.uniqueID = ''") -check "R1 every alert has an edge (no null uniqueID)" "$seen" "0" - -mapfile -t ORDERS < <(q "SELECT order_id FROM $DB.dx_anomaly_orders ORDER BY order_id") - -for oid in "${ORDERS[@]}"; do - uid=$(q "SELECT uniqueID FROM $DB.dx_anomaly_orders WHERE order_id='$oid' LIMIT 1") - - ks=$(q "SELECT count() FROM $DB.dx_src__kubescape_logs k INNER JOIN (SELECT uniqueID FROM $DB.dx_anomaly_orders WHERE order_id='$oid') o USING (uniqueID)") - ksbad=$(q "SELECT count() FROM $DB.dx_src__kubescape_logs k INNER JOIN (SELECT uniqueID FROM $DB.dx_anomaly_orders WHERE order_id='$oid') o USING (uniqueID) WHERE k.uniqueID != '$uid'") - check "R2 kubescape($oid) is the primary log only" "$ksbad" "0" - [ "$ks" -ge 1 ] && check "R2 kubescape($oid) present (>=1)" "1" "1" || check "R2 kubescape($oid) present (>=1)" "0" "1" - - for tbl in redis_events conn_stats dc_snoop http_events dns_events stack_trace; do - stamped=$(q "SELECT count() FROM $DB.dx_order_records WHERE order_id='$oid' AND src_table='$tbl'") - foreign=$(q "SELECT count() FROM $DB.dx_order_records WHERE order_id='$oid' AND src_table='$tbl' AND order_id != '$oid'") - check "R2 $tbl($oid) no foreign-order rows" "$foreign" "0" - done -done - -echo "--- leakage proof: window-join vs stamped for one order ---" -oid="${ORDERS[0]}" -win=$(q "SELECT count() FROM $DB.dx_src__redis_events s INNER JOIN (SELECT pod, lo, hi FROM $DB.dx_anomaly_orders WHERE order_id='$oid') o USING (pod) WHERE s.row_time BETWEEN o.lo AND o.hi") -stamped=$(q "SELECT count() FROM $DB.dx_order_records WHERE order_id='$oid' AND src_table='redis_events'") -printf 'window-join redis rows for %s = %s stamped rows = %s\n' "$oid" "$win" "$stamped" -if [ "$win" -gt "$stamped" ]; then - echo " -> confirms window over-collects (leaks other orders' pod rows); stamped is scoped." -fi - -echo -[ "$fail" -eq 0 ] && echo "ALL PASS" || { echo "FAILURES PRESENT"; exit 1; } From da0c7a4d74c7b10823b58fe78a4618b8a19b6429 Mon Sep 17 00:00:00 2001 From: entlein Date: Wed, 19 Aug 2026 21:12:09 +0200 Subject: [PATCH 43/66] px/dx_evidence_graph: _consulted drops join/plumbing columns; vis.json per spec --- src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl | 6 ------ src/pxl_scripts/px/dx_evidence_graph/vis.json | 3 +-- 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl b/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl index 735e4bf56d6..8b6c9a09c62 100644 --- a/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl +++ b/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl @@ -17,9 +17,6 @@ import px -# _consulted: interval-join one protocol's rows to a chosen order — equi-join on -# pod + row_time within the order's [lo,hi] window. This is the pre-correlation -# record set per protocol for the selected anomaly (order_id). def _consulted(start_time: str, clickhouse_dsn: str, view: str, order_id: str): orders = px.DataFrame('dx_anomaly_orders', clickhouse_dsn=clickhouse_dsn, start_time=start_time) orders = orders[orders.order_id == order_id] @@ -28,9 +25,6 @@ def _consulted(start_time: str, clickhouse_dsn: str, view: str, order_id: str): j = src.merge(orders, how='inner', left_on=['pod'], right_on=['pod'], suffixes=['', '_ord']) j = j[j.row_time >= j.lo] j = j[j.row_time <= j.hi] - # Keep these in the views + the ORDERS panel, but hide the join/plumbing columns - # from the protocol panels so they don't pollute the visual (ts stays as the - # human-readable time; the panel is already scoped to one order_id). return j.drop(['row_time', 'event_time', 'lo', 'hi', 'order_id', 'pod_ord']) diff --git a/src/pxl_scripts/px/dx_evidence_graph/vis.json b/src/pxl_scripts/px/dx_evidence_graph/vis.json index 50a2b554b7a..dd652236bd1 100644 --- a/src/pxl_scripts/px/dx_evidence_graph/vis.json +++ b/src/pxl_scripts/px/dx_evidence_graph/vis.json @@ -19,8 +19,7 @@ {"outputName": "g_orders", "func": {"name": "orders", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}]}} ], "widgets": [ - {"name": "Evidence graph (subject pod -> target)", "position": {"x": 0, "y": 0, "w": 12, "h": 4}, "globalFuncOutputName": "g_graph", - "displaySpec": {"@type": "types.px.dev/px.vispb.Graph", "adjacencyList": {"fromColumn": "from_entity", "toColumn": "to_entity"}, "edgeWeightColumn": "severity", "edgeColorColumn": "severity", "edgeLabelColumn": "rule", "edgeThresholds": {"mediumThreshold": 5, "highThreshold": 8}, "edgeHoverInfo": ["uniqueID", "rule", "process", "target", "severity"], "edgeLength": 500}}, + {"name": "Evidence graph (subject pod -> target)", "position": {"x": 0, "y": 0, "w": 12, "h": 4}, "globalFuncOutputName": "g_graph", "displaySpec": {"@type": "types.px.dev/px.vispb.Graph", "adjacencyList": {"fromColumn": "from_entity", "toColumn": "to_entity"}, "edgeWeightColumn": "severity", "edgeColorColumn": "severity", "edgeLabelColumn": "rule", "edgeThresholds": {"mediumThreshold": 5, "highThreshold": 8}, "edgeHoverInfo": ["uniqueID", "rule", "process", "target", "severity"], "edgeLength": 500}}, {"name": "kubescape_logs (anomalies) for order", "position": {"x": 0, "y": 4, "w": 12, "h": 4}, "globalFuncOutputName": "g_kube", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, {"name": "redis_events", "position": {"x": 0, "y": 8, "w": 6, "h": 4}, "globalFuncOutputName": "g_redis", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, {"name": "conn_stats", "position": {"x": 6, "y": 8, "w": 6, "h": 4}, "globalFuncOutputName": "g_conn", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, From c6fe7076898fb5fed20b917bcc8bc1b72c578e6e Mon Sep 17 00:00:00 2001 From: entlein Date: Wed, 19 Aug 2026 22:12:41 +0200 Subject: [PATCH 44/66] skaffold: pin AE 0.14.19-aeprod71 + dx rc22 (latest order-UUID stamping images) aeprod71 (from 095a17cf4) creates dx_order_seeds/dx_order_records/dx_anomaly_orders/ dx_src__* schema; rc22 stamps consulted records into dx_order_records. Prior pins (aeprod65 / rc21) predate the stamping and leave the order tables uncreated. --- k8s/vizier/adaptive_export/kustomization.yaml | 2 +- k8s/vizier/dx/dx-daemon.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/k8s/vizier/adaptive_export/kustomization.yaml b/k8s/vizier/adaptive_export/kustomization.yaml index d5c054b445a..79c2048c5a7 100644 --- a/k8s/vizier/adaptive_export/kustomization.yaml +++ b/k8s/vizier/adaptive_export/kustomization.yaml @@ -7,4 +7,4 @@ resources: images: - name: vizier-adaptive_export_image newName: ghcr.io/k8sstormcenter/vizier-adaptive_export_image - newTag: 0.14.19-aeprod65 + newTag: 0.14.19-aeprod71 diff --git a/k8s/vizier/dx/dx-daemon.yaml b/k8s/vizier/dx/dx-daemon.yaml index e19a84c6cf8..18261102a63 100644 --- a/k8s/vizier/dx/dx-daemon.yaml +++ b/k8s/vizier/dx/dx-daemon.yaml @@ -23,7 +23,7 @@ spec: # (WITHOUT -tiny — -tiny's pclntab stripping SIGSEGV'd under load). Passes the # obfuscation gate AND survives the kill-chain (restarts=0, 4 rounds). Carries the # evidence-manifest + DX_FOREST_PUSHDOWN code. - image: docker.io/entlein/dx-daemon:0.4.0-ssotforest-rc21 + image: docker.io/entlein/dx-daemon:0.4.0-ssotforest-rc22 ports: - { name: findings, containerPort: 9099, hostPort: 9099 } env: From e426be3f5d12ffd14ae46d85042f7c02dba30a19 Mon Sep 17 00:00:00 2001 From: entlein Date: Wed, 19 Aug 2026 23:13:29 +0200 Subject: [PATCH 45/66] ae: re-key dx_order_seeds + dx_anomaly_orders on order_id (dx owns dedup logic) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit AE encoded the 'one order per uniqueID' assumption in two places: - dx_order_seeds ReplacingMergeTree ORDER BY (unique_id, rule_id) - dx_anomaly_orders LIMIT 1 BY unique_id so any finer dx dedup (per rule/target/event) would be collapsed in the seeds table or hidden by the view. Re-key both on order_id — the identity dx computes and owns — so dx alone controls dedup granularity and AE never changes again. No-op with current dx (order_id is 1:1 with uniqueID today); future-proofs finer dedup. AE stays purely writing + DDL; dx owns all order logic. --- .../internal/clickhouse/schema.sql | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/src/vizier/services/adaptive_export/internal/clickhouse/schema.sql b/src/vizier/services/adaptive_export/internal/clickhouse/schema.sql index 80ce42ed913..2f9621da93c 100644 --- a/src/vizier/services/adaptive_export/internal/clickhouse/schema.sql +++ b/src/vizier/services/adaptive_export/internal/clickhouse/schema.sql @@ -567,13 +567,13 @@ CREATE TABLE IF NOT EXISTS forensic_db.dx_evidence_manifest ( TTL toDateTime(fromUnixTimestamp64Nano(event_time)) + INTERVAL 30 DAY DELETE SETTINGS index_granularity = 8192; --- dx_order_seeds — one row per kubescape referral dx sees (entlein/dx#136 --- evidence-loss fix). dx coalesces same-pod anomalies into one investigation, so --- most anomalies write no manifest; this table records EVERY anomaly's identity so --- the dx_anomaly_orders view can give each uniqueID its own consulted window --- (event_time ± 300s). dx INSERTs (POST-less, direct CH); AE owns the DDL. --- ReplacingMergeTree ORDER BY (unique_id, rule_id) dedups re-fires but keeps --- co-fired rules. NOT a pixie table. +-- dx_order_seeds — one row per ORDER dx opens (entlein/dx#136 evidence-loss fix). +-- dx owns the order identity: it computes order_id and decides the dedup +-- granularity (1:1 with uniqueID today; finer — per rule/target/event — later). +-- AE only stores and surfaces exactly what dx emits, so the key is order_id and +-- NOTHING here assumes how many orders map to a uniqueID. dx INSERTs (POST-less, +-- direct CH); AE owns the DDL. ReplacingMergeTree ORDER BY (order_id) dedups +-- re-fires of the same order. NOT a pixie table. CREATE TABLE IF NOT EXISTS forensic_db.dx_order_seeds ( order_id String, unique_id String, @@ -583,7 +583,7 @@ CREATE TABLE IF NOT EXISTS forensic_db.dx_order_seeds ( hostname String, case_key String ) ENGINE = ReplacingMergeTree() - ORDER BY (unique_id, rule_id) + ORDER BY (order_id) PARTITION BY toYYYYMM(fromUnixTimestamp64Nano(event_time)) TTL toDateTime(fromUnixTimestamp64Nano(event_time)) + INTERVAL 30 DAY DELETE SETTINGS index_granularity = 8192; @@ -746,11 +746,11 @@ CREATE TABLE IF NOT EXISTS forensic_db.creds_change ( -- ts=toString(time_) readable, row_time Int64 ns for the PxL interval-join. Views -- are not pixie socket_tracer tables → absent from PixieTables(). --- dx_anomaly_orders: ONE order per primary kubescape log (#136 stamping model). --- order_id is dx-assigned = hash(uniqueID) — 1:1 with the log (stored on the seed), --- NOT the window hash that collided for same-instant anomalies. lo/hi are kept for --- reference (the ±300s span); the CONSULTED records for the order live in --- dx_order_records, stamped with this order_id. +-- dx_anomaly_orders: ONE row per order dx opened. order_id is dx-assigned and +-- dx owns its granularity (hash(uniqueID) = 1:1 with the log today; finer later), +-- so the view dedups on order_id and makes NO assumption about orders-per-uniqueID. +-- lo/hi are kept for reference (the ±300s span); the CONSULTED records for the +-- order live in dx_order_records, stamped with this order_id. CREATE VIEW IF NOT EXISTS forensic_db.dx_anomaly_orders AS SELECT unique_id AS uniqueID, rule_id AS rule, pod, toInt64(event_time) - 300000000000 AS lo, @@ -758,7 +758,7 @@ SELECT unique_id AS uniqueID, rule_id AS rule, pod, order_id, hostname, event_time FROM forensic_db.dx_order_seeds -LIMIT 1 BY unique_id; +LIMIT 1 BY order_id; -- dx_kubescape_anomalies: L1 kill-chain graph (subject_pod -> target), deduped by uniqueID. CREATE VIEW IF NOT EXISTS forensic_db.dx_kubescape_anomalies AS From 1b9ca3652b82d152ad003b31d41a8a0cf2d6b4e3 Mon Sep 17 00:00:00 2001 From: entlein Date: Wed, 19 Aug 2026 23:48:16 +0200 Subject: [PATCH 46/66] skaffold: pin dx rc23 (finer order_id = uniqueID+RuleID+Disc) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Pairs with aeprod72 (dx_order_seeds/dx_anomaly_orders keyed on order_id). Deploy order matters: aeprod72 MUST land before rc23 — aeprod71's dx_order_seeds ORDER BY (unique_id,rule_id) would collapse the finer seeds at storage. --- k8s/vizier/dx/dx-daemon.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/vizier/dx/dx-daemon.yaml b/k8s/vizier/dx/dx-daemon.yaml index 18261102a63..98256fb38a7 100644 --- a/k8s/vizier/dx/dx-daemon.yaml +++ b/k8s/vizier/dx/dx-daemon.yaml @@ -23,7 +23,7 @@ spec: # (WITHOUT -tiny — -tiny's pclntab stripping SIGSEGV'd under load). Passes the # obfuscation gate AND survives the kill-chain (restarts=0, 4 rounds). Carries the # evidence-manifest + DX_FOREST_PUSHDOWN code. - image: docker.io/entlein/dx-daemon:0.4.0-ssotforest-rc22 + image: docker.io/entlein/dx-daemon:0.4.0-ssotforest-rc23 ports: - { name: findings, containerPort: 9099, hostPort: 9099 } env: From a057cb6fa25ceb0e5119d4ba114c9ed8287b85c9 Mon Sep 17 00:00:00 2001 From: entlein Date: Thu, 20 Aug 2026 08:40:34 +0200 Subject: [PATCH 47/66] skaffold: pin AE aeprod72 (order_id re-key; pairs with dx rc23) --- k8s/vizier/adaptive_export/kustomization.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/vizier/adaptive_export/kustomization.yaml b/k8s/vizier/adaptive_export/kustomization.yaml index 79c2048c5a7..2818e8f62bb 100644 --- a/k8s/vizier/adaptive_export/kustomization.yaml +++ b/k8s/vizier/adaptive_export/kustomization.yaml @@ -7,4 +7,4 @@ resources: images: - name: vizier-adaptive_export_image newName: ghcr.io/k8sstormcenter/vizier-adaptive_export_image - newTag: 0.14.19-aeprod71 + newTag: 0.14.19-aeprod72 From e168d19df77447d1e9595e959d3ea88d6468bf4c Mon Sep 17 00:00:00 2001 From: entlein Date: Thu, 20 Aug 2026 10:35:35 +0200 Subject: [PATCH 48/66] px/dx_evidence_graph: order_id deep-links + kubescape detail in graph popup - evidence_graph joins dx_anomaly_orders (uniqueID,rule) and emits an order_link px.script_reference; clicking it re-runs the script with order_id set so every panel snaps to that order. - orders() renders order_id as a clickable deep-link and drops raw lo/hi; ORDERS panel moves directly under the graph. - vis.json edgeHoverInfo restores 'alert' (kubescape message header) and adds order_link. - graph.tsx: ST_SCRIPT_REFERENCE columns in edgeHoverInfo render as a ScriptReference link at the top of the pinned edge popup instead of a tooltip text line; popup drag skips anchor clicks. --- .../dx_evidence_graph/dx_evidence_graph.pxl | 22 +++++++++++-- src/pxl_scripts/px/dx_evidence_graph/vis.json | 26 +++++++-------- .../containers/live-widgets/graph/graph.tsx | 33 +++++++++++++++++-- 3 files changed, 62 insertions(+), 19 deletions(-) diff --git a/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl b/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl index 8b6c9a09c62..ffc605817b8 100644 --- a/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl +++ b/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl @@ -30,9 +30,19 @@ def _consulted(start_time: str, clickhouse_dsn: str, view: str, order_id: str): def evidence_graph(start_time: str, clickhouse_dsn: str, table: str): df = px.DataFrame(table, clickhouse_dsn=clickhouse_dsn, start_time=start_time) + orders = px.DataFrame('dx_anomaly_orders', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + orders = orders[['uniqueID', 'rule', 'order_id']] + df = df.merge(orders, how='left', left_on=['uniqueID', 'rule'], + right_on=['uniqueID', 'rule'], suffixes=['', '_ord']) + df.order_link = px.script_reference(df.order_id, 'px/dx_evidence_graph', { + 'start_time': start_time, + 'clickhouse_dsn': clickhouse_dsn, + 'graph_table': table, + 'order_id': df.order_id, + }) df.from_entity = px.Pod(df.subject_pod) df.to_entity = df.target - return df[['from_entity', 'to_entity', 'uniqueID', 'rule', 'process', + return df[['from_entity', 'to_entity', 'order_link', 'uniqueID', 'rule', 'process', 'target', 'target_kind', 'severity', 'alert', 'subject_pod']] @@ -72,6 +82,12 @@ def stack_trace(start_time: str, clickhouse_dsn: str, order_id: str): return _consulted(start_time, clickhouse_dsn, 'dx_src__stack_trace', order_id) -def orders(start_time: str, clickhouse_dsn: str): +def orders(start_time: str, clickhouse_dsn: str, graph_table: str): df = px.DataFrame('dx_anomaly_orders', clickhouse_dsn=clickhouse_dsn, start_time=start_time) - return df[['uniqueID', 'rule', 'pod', 'order_id', 'lo', 'hi']] + df.order = px.script_reference(df.order_id, 'px/dx_evidence_graph', { + 'start_time': start_time, + 'clickhouse_dsn': clickhouse_dsn, + 'graph_table': graph_table, + 'order_id': df.order_id, + }) + return df[['order', 'uniqueID', 'rule', 'pod']] diff --git a/src/pxl_scripts/px/dx_evidence_graph/vis.json b/src/pxl_scripts/px/dx_evidence_graph/vis.json index dd652236bd1..3dd33601598 100644 --- a/src/pxl_scripts/px/dx_evidence_graph/vis.json +++ b/src/pxl_scripts/px/dx_evidence_graph/vis.json @@ -3,7 +3,7 @@ {"name": "start_time", "type": "PX_STRING", "description": "Window start.", "defaultValue": "-6h"}, {"name": "clickhouse_dsn", "type": "PX_STRING", "description": "forensic_db DSN: user:pass@host:port/db.", "defaultValue": "forensic_analyst:changeme-analyst@clickhouse-forensic-soc-db.clickhouse.svc.cluster.local:9000/forensic_db"}, {"name": "graph_table", "type": "PX_STRING", "description": "L1 kill-chain graph source.", "defaultValue": "dx_kubescape_anomalies"}, - {"name": "order_id", "type": "PX_STRING", "description": "Pick an order_id from the Orders panel at the bottom; every protocol panel snaps to that order's consulted records.", "defaultValue": ""} + {"name": "order_id", "type": "PX_STRING", "description": "Set by clicking an order link in the graph popup or the ORDERS panel; every protocol panel snaps to that order's consulted records.", "defaultValue": ""} ], "globalFuncs": [ {"outputName": "g_graph", "func": {"name": "evidence_graph", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "table", "variable": "graph_table"}]}}, @@ -16,19 +16,19 @@ {"outputName": "g_mysql", "func": {"name": "mysql", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "order_id", "variable": "order_id"}]}}, {"outputName": "g_dcsnoop", "func": {"name": "dc_snoop", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "order_id", "variable": "order_id"}]}}, {"outputName": "g_stack", "func": {"name": "stack_trace", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "order_id", "variable": "order_id"}]}}, - {"outputName": "g_orders", "func": {"name": "orders", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}]}} + {"outputName": "g_orders", "func": {"name": "orders", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "graph_table", "variable": "graph_table"}]}} ], "widgets": [ - {"name": "Evidence graph (subject pod -> target)", "position": {"x": 0, "y": 0, "w": 12, "h": 4}, "globalFuncOutputName": "g_graph", "displaySpec": {"@type": "types.px.dev/px.vispb.Graph", "adjacencyList": {"fromColumn": "from_entity", "toColumn": "to_entity"}, "edgeWeightColumn": "severity", "edgeColorColumn": "severity", "edgeLabelColumn": "rule", "edgeThresholds": {"mediumThreshold": 5, "highThreshold": 8}, "edgeHoverInfo": ["uniqueID", "rule", "process", "target", "severity"], "edgeLength": 500}}, - {"name": "kubescape_logs (anomalies) for order", "position": {"x": 0, "y": 4, "w": 12, "h": 4}, "globalFuncOutputName": "g_kube", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, - {"name": "redis_events", "position": {"x": 0, "y": 8, "w": 6, "h": 4}, "globalFuncOutputName": "g_redis", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, - {"name": "conn_stats", "position": {"x": 6, "y": 8, "w": 6, "h": 4}, "globalFuncOutputName": "g_conn", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, - {"name": "http_events", "position": {"x": 0, "y": 12, "w": 6, "h": 4}, "globalFuncOutputName": "g_http", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, - {"name": "dns_events", "position": {"x": 6, "y": 12, "w": 6, "h": 4}, "globalFuncOutputName": "g_dns", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, - {"name": "pgsql_events", "position": {"x": 0, "y": 16, "w": 6, "h": 4}, "globalFuncOutputName": "g_pgsql", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, - {"name": "mysql_events", "position": {"x": 6, "y": 16, "w": 6, "h": 4}, "globalFuncOutputName": "g_mysql", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, - {"name": "dc_snoop (file access)", "position": {"x": 0, "y": 20, "w": 6, "h": 4}, "globalFuncOutputName": "g_dcsnoop", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, - {"name": "stack_trace (profiler)", "position": {"x": 6, "y": 20, "w": 6, "h": 4}, "globalFuncOutputName": "g_stack", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, - {"name": "ORDERS (pick an order_id -> paste into the order_id variable)", "position": {"x": 0, "y": 24, "w": 12, "h": 4}, "globalFuncOutputName": "g_orders", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}} + {"name": "Evidence graph (subject pod -> target; click an edge for details + order link)", "position": {"x": 0, "y": 0, "w": 12, "h": 4}, "globalFuncOutputName": "g_graph", "displaySpec": {"@type": "types.px.dev/px.vispb.Graph", "adjacencyList": {"fromColumn": "from_entity", "toColumn": "to_entity"}, "edgeWeightColumn": "severity", "edgeColorColumn": "severity", "edgeLabelColumn": "rule", "edgeThresholds": {"mediumThreshold": 5, "highThreshold": 8}, "edgeHoverInfo": ["order_link", "rule", "alert", "process", "target", "severity", "uniqueID"], "edgeLength": 500}}, + {"name": "ORDERS (click an order to filter all panels)", "position": {"x": 0, "y": 4, "w": 12, "h": 3}, "globalFuncOutputName": "g_orders", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, + {"name": "kubescape_logs (anomalies) for order", "position": {"x": 0, "y": 7, "w": 12, "h": 4}, "globalFuncOutputName": "g_kube", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, + {"name": "redis_events", "position": {"x": 0, "y": 11, "w": 6, "h": 4}, "globalFuncOutputName": "g_redis", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, + {"name": "conn_stats", "position": {"x": 6, "y": 11, "w": 6, "h": 4}, "globalFuncOutputName": "g_conn", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, + {"name": "http_events", "position": {"x": 0, "y": 15, "w": 6, "h": 4}, "globalFuncOutputName": "g_http", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, + {"name": "dns_events", "position": {"x": 6, "y": 15, "w": 6, "h": 4}, "globalFuncOutputName": "g_dns", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, + {"name": "pgsql_events", "position": {"x": 0, "y": 19, "w": 6, "h": 4}, "globalFuncOutputName": "g_pgsql", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, + {"name": "mysql_events", "position": {"x": 6, "y": 19, "w": 6, "h": 4}, "globalFuncOutputName": "g_mysql", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, + {"name": "dc_snoop (file access)", "position": {"x": 0, "y": 23, "w": 6, "h": 4}, "globalFuncOutputName": "g_dcsnoop", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, + {"name": "stack_trace (profiler)", "position": {"x": 6, "y": 23, "w": 6, "h": 4}, "globalFuncOutputName": "g_stack", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}} ] } diff --git a/src/ui/src/containers/live-widgets/graph/graph.tsx b/src/ui/src/containers/live-widgets/graph/graph.tsx index a8a642eec84..685931bf571 100644 --- a/src/ui/src/containers/live-widgets/graph/graph.tsx +++ b/src/ui/src/containers/live-widgets/graph/graph.tsx @@ -46,6 +46,7 @@ import { } from './graph-utils'; import { formatByDataType, formatBySemType } from '../../format-data/format-data'; import { deepLinkURLFromSemanticType } from '../utils/live-view-params'; +import { ScriptReference } from '../utils/script-reference'; interface AdjacencyList { toColumn: string; @@ -157,8 +158,11 @@ export const Graph = React.memo(({ const [graph, setGraph] = React.useState(null); const [pinned, setPinned] = React.useState>([]); const pinSeq = React.useRef(0); + const [edgeScriptRefs, setEdgeScriptRefs] = React.useState< + Map>(() => new Map()); const [edgeLabels, setEdgeLabels] = React.useState>(() => new Map()); const [labelOffsets, setLabelOffsets] = React.useState>(() => new Map()); @@ -198,6 +202,7 @@ export const Graph = React.memo(({ const nodes = new visData.DataSet(); const idToSemType = {}; const labelMap = new Map(); + const scriptRefMap = new Map(); const selfLoopCounts = new Map(); const selfLoopRank = new Map(); @@ -257,8 +262,17 @@ export const Graph = React.memo(({ if (edgeHoverInfo && edgeHoverInfo.length > 0) { let edgeInfo = ''; - edgeHoverInfo.forEach((info, i) => { + edgeHoverInfo.forEach((info) => { if (info != null) { + // Script-reference columns become the deep link in the pinned popup, + // not a line in the hover text (a hover tooltip can't be clicked). + if (info.semType === SemanticType.ST_SCRIPT_REFERENCE) { + const ref = d[info.name]; + if (ref && ref.script) { + scriptRefMap.set(edgeId, { label: ref.label, script: ref.script, args: ref.args }); + } + return; + } let val: string; if (info.semType === SemanticType.ST_NONE || info.semType === SemanticType.ST_UNSPECIFIED) { val = formatByDataType(info.type, d[info.name]); @@ -266,7 +280,7 @@ export const Graph = React.memo(({ const valWithUnits = formatBySemType(info.semType, d[info.name]); val = `${valWithUnits.val} ${valWithUnits.units}`; } - edgeInfo = `${edgeInfo}${i === 0 ? '' : '
'} ${info.name}: ${val}`; + edgeInfo = `${edgeInfo}${edgeInfo === '' ? '' : '
'} ${info.name}: ${val}`; } }); edge.title = edgeInfo; @@ -279,6 +293,7 @@ export const Graph = React.memo(({ nodes, edges, idToSemType, }); setEdgeLabels(labelMap); + setEdgeScriptRefs(scriptRefMap); setLabelOffsets((prev) => { const next = new Map(); selfLoopRank.forEach((rank, edgeId) => { @@ -331,6 +346,7 @@ export const Graph = React.memo(({ title: String(edgeData?.title ?? ''), x: rect.left + params.pointer.DOM.x, y: rect.top + params.pointer.DOM.y, + scriptRef: edgeScriptRefs.get(edgeId), }]); } }); @@ -401,7 +417,7 @@ export const Graph = React.memo(({ const onPinPointerDown = React.useCallback((key: number, initialX: number, initialY: number) => (e: React.PointerEvent) => { - if ((e.target as HTMLElement).closest('[data-pin-close]')) return; + if ((e.target as HTMLElement).closest('a, [data-pin-close]')) return; e.stopPropagation(); const startX = e.clientX; const startY = e.clientY; @@ -490,6 +506,17 @@ export const Graph = React.memo(({ touchAction: 'none', }} > + {p.scriptRef && ( +
+ +
+ )}
Date: Thu, 20 Aug 2026 10:55:34 +0200 Subject: [PATCH 49/66] px/dx_evidence_graph: order deep-link opens the EXACT consulted set Panels behind the order_id deep-link now read the stamped dx_order_records rows for that order (order_id + src_table filter, identity columns vector/source/pod/remote_addr/path/comm/dns_name) instead of reconstructing an approximate pod +/-300s window over the dx_src__* views. This is the dx#136 stamping model the dx_order_records DDL documents as the intended panel read path. kubescape panel joins dx_src__kubescape_logs on (uniqueID, RuleID) of the selected order, keeping the full anomaly rows (alert, cmdline, process tree) for the seed. evidence_graph/orders deep-links and vis.json unchanged from e168d19df. --- .../dx_evidence_graph/dx_evidence_graph.pxl | 38 ++++++++++--------- 1 file changed, 20 insertions(+), 18 deletions(-) diff --git a/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl b/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl index ffc605817b8..850154427af 100644 --- a/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl +++ b/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl @@ -17,15 +17,11 @@ import px -def _consulted(start_time: str, clickhouse_dsn: str, view: str, order_id: str): - orders = px.DataFrame('dx_anomaly_orders', clickhouse_dsn=clickhouse_dsn, start_time=start_time) - orders = orders[orders.order_id == order_id] - orders = orders[['order_id', 'pod', 'lo', 'hi']] - src = px.DataFrame(view, clickhouse_dsn=clickhouse_dsn, start_time=start_time) - j = src.merge(orders, how='inner', left_on=['pod'], right_on=['pod'], suffixes=['', '_ord']) - j = j[j.row_time >= j.lo] - j = j[j.row_time <= j.hi] - return j.drop(['row_time', 'event_time', 'lo', 'hi', 'order_id', 'pod_ord']) +def _stamped(start_time: str, clickhouse_dsn: str, src_table: str, order_id: str): + df = px.DataFrame('dx_order_records', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + df = df[df.order_id == order_id] + df = df[df.src_table == src_table] + return df[['vector', 'source', 'pod', 'remote_addr', 'path', 'comm', 'dns_name']] def evidence_graph(start_time: str, clickhouse_dsn: str, table: str): @@ -47,39 +43,45 @@ def evidence_graph(start_time: str, clickhouse_dsn: str, table: str): def kubescape(start_time: str, clickhouse_dsn: str, order_id: str): - return _consulted(start_time, clickhouse_dsn, 'dx_src__kubescape_logs', order_id) + orders = px.DataFrame('dx_anomaly_orders', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + orders = orders[orders.order_id == order_id] + orders = orders[['uniqueID', 'rule']] + k = px.DataFrame('dx_src__kubescape_logs', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + j = k.merge(orders, how='inner', left_on=['uniqueID', 'RuleID'], + right_on=['uniqueID', 'rule'], suffixes=['', '_ord']) + return j.drop(['row_time', 'event_time', 'uniqueID_ord', 'rule']) def redis(start_time: str, clickhouse_dsn: str, order_id: str): - return _consulted(start_time, clickhouse_dsn, 'dx_src__redis_events', order_id) + return _stamped(start_time, clickhouse_dsn, 'redis_events', order_id) def conn(start_time: str, clickhouse_dsn: str, order_id: str): - return _consulted(start_time, clickhouse_dsn, 'dx_src__conn_stats', order_id) + return _stamped(start_time, clickhouse_dsn, 'conn_stats', order_id) def http(start_time: str, clickhouse_dsn: str, order_id: str): - return _consulted(start_time, clickhouse_dsn, 'dx_src__http_events', order_id) + return _stamped(start_time, clickhouse_dsn, 'http_events', order_id) def dns(start_time: str, clickhouse_dsn: str, order_id: str): - return _consulted(start_time, clickhouse_dsn, 'dx_src__dns_events', order_id) + return _stamped(start_time, clickhouse_dsn, 'dns_events', order_id) def pgsql(start_time: str, clickhouse_dsn: str, order_id: str): - return _consulted(start_time, clickhouse_dsn, 'dx_src__pgsql_events', order_id) + return _stamped(start_time, clickhouse_dsn, 'pgsql_events', order_id) def mysql(start_time: str, clickhouse_dsn: str, order_id: str): - return _consulted(start_time, clickhouse_dsn, 'dx_src__mysql_events', order_id) + return _stamped(start_time, clickhouse_dsn, 'mysql_events', order_id) def dc_snoop(start_time: str, clickhouse_dsn: str, order_id: str): - return _consulted(start_time, clickhouse_dsn, 'dx_src__dc_snoop', order_id) + return _stamped(start_time, clickhouse_dsn, 'dc_snoop', order_id) def stack_trace(start_time: str, clickhouse_dsn: str, order_id: str): - return _consulted(start_time, clickhouse_dsn, 'dx_src__stack_trace', order_id) + return _stamped(start_time, clickhouse_dsn, 'stack_trace', order_id) def orders(start_time: str, clickhouse_dsn: str, graph_table: str): From d72d9045c6970f36044d6d20e9eaf84fd321eed6 Mon Sep 17 00:00:00 2001 From: entlein Date: Thu, 20 Aug 2026 11:42:57 +0200 Subject: [PATCH 50/66] px/dx_evidence_graph: ORDERS shows the kubescape alert text, not uniqueID MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit orders() joins the graph source (dx_kubescape_anomalies) on (uniqueID, rule) to carry the human-readable alert message (e.g. 'Unexpected process launched: cat ...') into the ORDERS panel — the same description the graph popup shows. uniqueID dropped from the display (no analyst value; it remains the join key internally). Columns: order (deep-link) / rule / alert / pod. --- src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl b/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl index 850154427af..9b30e09ebe6 100644 --- a/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl +++ b/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl @@ -86,10 +86,14 @@ def stack_trace(start_time: str, clickhouse_dsn: str, order_id: str): def orders(start_time: str, clickhouse_dsn: str, graph_table: str): df = px.DataFrame('dx_anomaly_orders', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + alerts = px.DataFrame(graph_table, clickhouse_dsn=clickhouse_dsn, start_time=start_time) + alerts = alerts[['uniqueID', 'rule', 'alert']] + df = df.merge(alerts, how='left', left_on=['uniqueID', 'rule'], + right_on=['uniqueID', 'rule'], suffixes=['', '_k']) df.order = px.script_reference(df.order_id, 'px/dx_evidence_graph', { 'start_time': start_time, 'clickhouse_dsn': clickhouse_dsn, 'graph_table': graph_table, 'order_id': df.order_id, }) - return df[['order', 'uniqueID', 'rule', 'pod']] + return df[['order', 'rule', 'alert', 'pod']] From bbb8a5c8dc79994f5fe3a82e548adcc47b340b46 Mon Sep 17 00:00:00 2001 From: entlein Date: Thu, 20 Aug 2026 13:02:32 +0200 Subject: [PATCH 51/66] px/dx_evidence_graph: restore dx_src__ protocol-view panels (fix v0.0.29 regression) v0.0.29 swapped every protocol panel (redis/conn/http/dns/pgsql/mysql/ dc_snoop/stack_trace) from reading its dx_src__
view to reading dx_order_records filtered by a bare src_table name. That dropped the panels to 7 fixed identity columns AND left them empty wherever dx_order_records isn't stamped with a matching order_id/src_table - breaking all protocol tables in the panel. Restore _consulted (reads the dx_src__ views, order lo/hi window) for all 8 protocol panels. Keeps the v0.0.29 graph order_link deep-link, the ORDERS alert-text column, and the kubescape uniqueID+RuleID join (all unaffected - they already read the correct tables). Tradeoff: back to the +/-300s window scoping (not the stamped-exact set). Exact-records WITH full protocol columns needs a hybrid join (stamped identities back to the dx_src__ views) - separate follow-up, not another broken release. --- .../dx_evidence_graph/dx_evidence_graph.pxl | 30 +++++++++++-------- 1 file changed, 17 insertions(+), 13 deletions(-) diff --git a/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl b/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl index 9b30e09ebe6..b8e4990dd5b 100644 --- a/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl +++ b/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl @@ -17,11 +17,15 @@ import px -def _stamped(start_time: str, clickhouse_dsn: str, src_table: str, order_id: str): - df = px.DataFrame('dx_order_records', clickhouse_dsn=clickhouse_dsn, start_time=start_time) - df = df[df.order_id == order_id] - df = df[df.src_table == src_table] - return df[['vector', 'source', 'pod', 'remote_addr', 'path', 'comm', 'dns_name']] +def _consulted(start_time: str, clickhouse_dsn: str, view: str, order_id: str): + orders = px.DataFrame('dx_anomaly_orders', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + orders = orders[orders.order_id == order_id] + orders = orders[['order_id', 'pod', 'lo', 'hi']] + src = px.DataFrame(view, clickhouse_dsn=clickhouse_dsn, start_time=start_time) + j = src.merge(orders, how='inner', left_on=['pod'], right_on=['pod'], suffixes=['', '_ord']) + j = j[j.row_time >= j.lo] + j = j[j.row_time <= j.hi] + return j.drop(['row_time', 'event_time', 'lo', 'hi', 'order_id', 'pod_ord']) def evidence_graph(start_time: str, clickhouse_dsn: str, table: str): @@ -53,35 +57,35 @@ def kubescape(start_time: str, clickhouse_dsn: str, order_id: str): def redis(start_time: str, clickhouse_dsn: str, order_id: str): - return _stamped(start_time, clickhouse_dsn, 'redis_events', order_id) + return _consulted(start_time, clickhouse_dsn, 'dx_src__redis_events', order_id) def conn(start_time: str, clickhouse_dsn: str, order_id: str): - return _stamped(start_time, clickhouse_dsn, 'conn_stats', order_id) + return _consulted(start_time, clickhouse_dsn, 'dx_src__conn_stats', order_id) def http(start_time: str, clickhouse_dsn: str, order_id: str): - return _stamped(start_time, clickhouse_dsn, 'http_events', order_id) + return _consulted(start_time, clickhouse_dsn, 'dx_src__http_events', order_id) def dns(start_time: str, clickhouse_dsn: str, order_id: str): - return _stamped(start_time, clickhouse_dsn, 'dns_events', order_id) + return _consulted(start_time, clickhouse_dsn, 'dx_src__dns_events', order_id) def pgsql(start_time: str, clickhouse_dsn: str, order_id: str): - return _stamped(start_time, clickhouse_dsn, 'pgsql_events', order_id) + return _consulted(start_time, clickhouse_dsn, 'dx_src__pgsql_events', order_id) def mysql(start_time: str, clickhouse_dsn: str, order_id: str): - return _stamped(start_time, clickhouse_dsn, 'mysql_events', order_id) + return _consulted(start_time, clickhouse_dsn, 'dx_src__mysql_events', order_id) def dc_snoop(start_time: str, clickhouse_dsn: str, order_id: str): - return _stamped(start_time, clickhouse_dsn, 'dc_snoop', order_id) + return _consulted(start_time, clickhouse_dsn, 'dx_src__dc_snoop', order_id) def stack_trace(start_time: str, clickhouse_dsn: str, order_id: str): - return _stamped(start_time, clickhouse_dsn, 'stack_trace', order_id) + return _consulted(start_time, clickhouse_dsn, 'dx_src__stack_trace', order_id) def orders(start_time: str, clickhouse_dsn: str, graph_table: str): From 1d24ba5927a3538d43168c708f650da36826ebe4 Mon Sep 17 00:00:00 2001 From: entlein Date: Thu, 20 Aug 2026 17:56:29 +0200 Subject: [PATCH 52/66] px/dx_evidence_graph: differential stack-trace flamegraph panel (attack vs baseline) Adds stack_diff() + a full-width StackTraceFlameGraph panel below the existing panels. For the selected order it diffs the pod's stacks DURING the attack window [lo,hi] against its BASELINE stacks before the attack (row_time < lo), keyed off dx_anomaly_orders. delta = attack - baseline drives the differential colouring (red = frames that spiked during the attack, e.g. mal.sh / kustomize --enable-exec / head -> /etc/shadow). Reuses Pixie's StackTraceFlameGraph widget (semicolon-folded stacks, differenceColumn). Groupby over the ClickHouse-DSN source compiles and runs. Validated live via paste on the user's cluster. --- .../dx_evidence_graph/dx_evidence_graph.pxl | 31 +++++++++++++++++++ src/pxl_scripts/px/dx_evidence_graph/vis.json | 4 ++- 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl b/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl index b8e4990dd5b..e30a5edae1d 100644 --- a/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl +++ b/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl @@ -88,6 +88,37 @@ def stack_trace(start_time: str, clickhouse_dsn: str, order_id: str): return _consulted(start_time, clickhouse_dsn, 'dx_src__stack_trace', order_id) +def stack_diff(start_time: str, clickhouse_dsn: str, order_id: str): + # Differential flame graph for ONE order's pod: the pod's stacks DURING the + # attack window [lo,hi] vs its BASELINE stacks before the attack (row_time < lo). + # delta = attack_count - baseline_count → the StackTraceFlameGraph widget colours + # frames that spiked during the attack red, quiet-baseline frames blue. + orders = px.DataFrame('dx_anomaly_orders', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + orders = orders[orders.order_id == order_id] + orders = orders[['pod', 'lo', 'hi']] + st = px.DataFrame('dx_src__stack_trace', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + st = st.merge(orders, how='inner', left_on=['pod'], right_on=['pod'], suffixes=['', '_o']) + + base = st[st.row_time < st.lo] + base = base.groupby(['pod', 'stack_trace']).agg(count=('count', px.sum)) + + atk = st[st.row_time >= st.lo] + atk = atk[atk.row_time <= atk.hi] + atk = atk.groupby(['pod', 'stack_trace']).agg(count=('count', px.sum)) + + diff = base.merge(atk, how='right', left_on=['stack_trace'], right_on=['stack_trace'], + suffixes=['_base', '_atk']) + diff.pod = diff.pod_atk + diff.stack_trace = px.replace(' ', diff.stack_trace_atk, '') + diff.count = diff.count_atk + diff.delta = diff.count_atk - diff.count_base + + total = atk.groupby(['pod']).agg(total=('count', px.sum)) + merged = diff.merge(total, how='inner', left_on=['pod'], right_on=['pod'], suffixes=['', '_t']) + merged.percent = 100 * merged.count / merged.total + return merged[['stack_trace', 'count', 'delta', 'percent', 'pod']] + + def orders(start_time: str, clickhouse_dsn: str, graph_table: str): df = px.DataFrame('dx_anomaly_orders', clickhouse_dsn=clickhouse_dsn, start_time=start_time) alerts = px.DataFrame(graph_table, clickhouse_dsn=clickhouse_dsn, start_time=start_time) diff --git a/src/pxl_scripts/px/dx_evidence_graph/vis.json b/src/pxl_scripts/px/dx_evidence_graph/vis.json index 3dd33601598..2ad4bafc5bb 100644 --- a/src/pxl_scripts/px/dx_evidence_graph/vis.json +++ b/src/pxl_scripts/px/dx_evidence_graph/vis.json @@ -16,6 +16,7 @@ {"outputName": "g_mysql", "func": {"name": "mysql", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "order_id", "variable": "order_id"}]}}, {"outputName": "g_dcsnoop", "func": {"name": "dc_snoop", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "order_id", "variable": "order_id"}]}}, {"outputName": "g_stack", "func": {"name": "stack_trace", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "order_id", "variable": "order_id"}]}}, + {"outputName": "g_stackdiff", "func": {"name": "stack_diff", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "order_id", "variable": "order_id"}]}}, {"outputName": "g_orders", "func": {"name": "orders", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "graph_table", "variable": "graph_table"}]}} ], "widgets": [ @@ -29,6 +30,7 @@ {"name": "pgsql_events", "position": {"x": 0, "y": 19, "w": 6, "h": 4}, "globalFuncOutputName": "g_pgsql", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, {"name": "mysql_events", "position": {"x": 6, "y": 19, "w": 6, "h": 4}, "globalFuncOutputName": "g_mysql", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, {"name": "dc_snoop (file access)", "position": {"x": 0, "y": 23, "w": 6, "h": 4}, "globalFuncOutputName": "g_dcsnoop", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, - {"name": "stack_trace (profiler)", "position": {"x": 6, "y": 23, "w": 6, "h": 4}, "globalFuncOutputName": "g_stack", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}} + {"name": "stack_trace (profiler)", "position": {"x": 6, "y": 23, "w": 6, "h": 4}, "globalFuncOutputName": "g_stack", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, + {"name": "Differential stack trace — attack window vs baseline (red = spiked during attack)", "position": {"x": 0, "y": 27, "w": 12, "h": 7}, "globalFuncOutputName": "g_stackdiff", "displaySpec": {"@type": "types.px.dev/px.vispb.StackTraceFlameGraph", "stacktraceColumn": "stack_trace", "countColumn": "count", "percentageColumn": "percent", "podColumn": "pod", "differenceColumn": "delta"}} ] } From 065f1dcce863cf0c57330819e5aad2b3012339cd Mon Sep 17 00:00:00 2001 From: entlein Date: Fri, 21 Aug 2026 12:24:19 +0200 Subject: [PATCH 53/66] adaptive_export: bridge all consulted protocol tables to orders via unique_id Extends the loop-1 conn_stats order-bridge to every consulted protocol table (redis/http/dns/pgsql/mysql/dc_snoop/stack_trace): - schema.sql: add unique_id String to the 8 base tables; add 8 dx_ord__
join views (order_id + ts/row_time/event_time nanos convention + e.hostname from the edge, so the px connector's node shard resolves) over dx_order_edges INNER JOIN ON unique_id. - ddl.go/apply.go: register the 8 views in KnownTables + OperatorOwnedTables (created on boot after their base table + dx_order_edges). - control/server.go: dxRowsAllowedTables += the 8 bridged tables. - cmd/main.go: bridgedPushSkip excludes them from ADAPTIVE_PUSH_PIXIE_ROWS (dx hands them via /dx/rows with a pre-stamped unique_id; a push write without unique_id would collapse the join under ReplacingMergeTree). - pxl/vis.json: 8 protocol panels reading dx_ord__
, filtered by order_id. --- .../dx_evidence_graph/dx_evidence_graph.pxl | 108 +++------ src/pxl_scripts/px/dx_evidence_graph/vis.json | 227 +++++++++++++++--- .../services/adaptive_export/cmd/main.go | 25 ++ .../internal/clickhouse/apply.go | 12 + .../internal/clickhouse/apply_test.go | 4 +- .../internal/clickhouse/columns_test.go | 4 +- .../internal/clickhouse/ddl.go | 12 + .../internal/clickhouse/schema.sql | 213 +++++++++++++++- .../internal/control/server.go | 75 ++++++ 9 files changed, 562 insertions(+), 118 deletions(-) diff --git a/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl b/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl index e30a5edae1d..b8c136a6244 100644 --- a/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl +++ b/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl @@ -17,23 +17,18 @@ import px -def _consulted(start_time: str, clickhouse_dsn: str, view: str, order_id: str): - orders = px.DataFrame('dx_anomaly_orders', clickhouse_dsn=clickhouse_dsn, start_time=start_time) - orders = orders[orders.order_id == order_id] - orders = orders[['order_id', 'pod', 'lo', 'hi']] - src = px.DataFrame(view, clickhouse_dsn=clickhouse_dsn, start_time=start_time) - j = src.merge(orders, how='inner', left_on=['pod'], right_on=['pod'], suffixes=['', '_ord']) - j = j[j.row_time >= j.lo] - j = j[j.row_time <= j.hi] - return j.drop(['row_time', 'event_time', 'lo', 'hi', 'order_id', 'pod_ord']) +def _ord(start_time: str, clickhouse_dsn: str, view: str, order_id: str): + df = px.DataFrame(view, clickhouse_dsn=clickhouse_dsn, start_time=start_time) + df = df[df.order_id == order_id] + return df.drop(['order_id', 'row_time', 'event_time']) def evidence_graph(start_time: str, clickhouse_dsn: str, table: str): df = px.DataFrame(table, clickhouse_dsn=clickhouse_dsn, start_time=start_time) - orders = px.DataFrame('dx_anomaly_orders', clickhouse_dsn=clickhouse_dsn, start_time=start_time) - orders = orders[['uniqueID', 'rule', 'order_id']] + orders = px.DataFrame('dx_orders', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + orders = orders[['kubescape_uid', 'rule_id', 'order_id']] df = df.merge(orders, how='left', left_on=['uniqueID', 'rule'], - right_on=['uniqueID', 'rule'], suffixes=['', '_ord']) + right_on=['kubescape_uid', 'rule_id'], suffixes=['', '_ord']) df.order_link = px.script_reference(df.order_id, 'px/dx_evidence_graph', { 'start_time': start_time, 'clickhouse_dsn': clickhouse_dsn, @@ -46,89 +41,58 @@ def evidence_graph(start_time: str, clickhouse_dsn: str, table: str): 'target', 'target_kind', 'severity', 'alert', 'subject_pod']] +def orders(start_time: str, clickhouse_dsn: str, graph_table: str): + df = px.DataFrame('dx_orders', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + alerts = px.DataFrame(graph_table, clickhouse_dsn=clickhouse_dsn, start_time=start_time) + alerts = alerts[['uniqueID', 'rule', 'alert']] + df = df.merge(alerts, how='left', left_on=['kubescape_uid', 'rule_id'], + right_on=['uniqueID', 'rule'], suffixes=['', '_k']) + df.order = px.script_reference(df.order_id, 'px/dx_evidence_graph', { + 'start_time': start_time, + 'clickhouse_dsn': clickhouse_dsn, + 'graph_table': graph_table, + 'order_id': df.order_id, + }) + return df[['order', 'rule_id', 'disc', 'alert', 'pod']] + + def kubescape(start_time: str, clickhouse_dsn: str, order_id: str): - orders = px.DataFrame('dx_anomaly_orders', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + orders = px.DataFrame('dx_orders', clickhouse_dsn=clickhouse_dsn, start_time=start_time) orders = orders[orders.order_id == order_id] - orders = orders[['uniqueID', 'rule']] + orders = orders[['kubescape_uid', 'rule_id']] k = px.DataFrame('dx_src__kubescape_logs', clickhouse_dsn=clickhouse_dsn, start_time=start_time) j = k.merge(orders, how='inner', left_on=['uniqueID', 'RuleID'], - right_on=['uniqueID', 'rule'], suffixes=['', '_ord']) - return j.drop(['row_time', 'event_time', 'uniqueID_ord', 'rule']) + right_on=['kubescape_uid', 'rule_id'], suffixes=['', '_ord']) + return j.drop(['row_time', 'event_time', 'kubescape_uid', 'rule_id']) -def redis(start_time: str, clickhouse_dsn: str, order_id: str): - return _consulted(start_time, clickhouse_dsn, 'dx_src__redis_events', order_id) +def conn(start_time: str, clickhouse_dsn: str, order_id: str): + return _ord(start_time, clickhouse_dsn, 'dx_ord__conn_stats', order_id) -def conn(start_time: str, clickhouse_dsn: str, order_id: str): - return _consulted(start_time, clickhouse_dsn, 'dx_src__conn_stats', order_id) +def redis(start_time: str, clickhouse_dsn: str, order_id: str): + return _ord(start_time, clickhouse_dsn, 'dx_ord__redis_events', order_id) def http(start_time: str, clickhouse_dsn: str, order_id: str): - return _consulted(start_time, clickhouse_dsn, 'dx_src__http_events', order_id) + return _ord(start_time, clickhouse_dsn, 'dx_ord__http_events', order_id) def dns(start_time: str, clickhouse_dsn: str, order_id: str): - return _consulted(start_time, clickhouse_dsn, 'dx_src__dns_events', order_id) + return _ord(start_time, clickhouse_dsn, 'dx_ord__dns_events', order_id) def pgsql(start_time: str, clickhouse_dsn: str, order_id: str): - return _consulted(start_time, clickhouse_dsn, 'dx_src__pgsql_events', order_id) + return _ord(start_time, clickhouse_dsn, 'dx_ord__pgsql_events', order_id) def mysql(start_time: str, clickhouse_dsn: str, order_id: str): - return _consulted(start_time, clickhouse_dsn, 'dx_src__mysql_events', order_id) + return _ord(start_time, clickhouse_dsn, 'dx_ord__mysql_events', order_id) def dc_snoop(start_time: str, clickhouse_dsn: str, order_id: str): - return _consulted(start_time, clickhouse_dsn, 'dx_src__dc_snoop', order_id) + return _ord(start_time, clickhouse_dsn, 'dx_ord__dc_snoop', order_id) def stack_trace(start_time: str, clickhouse_dsn: str, order_id: str): - return _consulted(start_time, clickhouse_dsn, 'dx_src__stack_trace', order_id) - - -def stack_diff(start_time: str, clickhouse_dsn: str, order_id: str): - # Differential flame graph for ONE order's pod: the pod's stacks DURING the - # attack window [lo,hi] vs its BASELINE stacks before the attack (row_time < lo). - # delta = attack_count - baseline_count → the StackTraceFlameGraph widget colours - # frames that spiked during the attack red, quiet-baseline frames blue. - orders = px.DataFrame('dx_anomaly_orders', clickhouse_dsn=clickhouse_dsn, start_time=start_time) - orders = orders[orders.order_id == order_id] - orders = orders[['pod', 'lo', 'hi']] - st = px.DataFrame('dx_src__stack_trace', clickhouse_dsn=clickhouse_dsn, start_time=start_time) - st = st.merge(orders, how='inner', left_on=['pod'], right_on=['pod'], suffixes=['', '_o']) - - base = st[st.row_time < st.lo] - base = base.groupby(['pod', 'stack_trace']).agg(count=('count', px.sum)) - - atk = st[st.row_time >= st.lo] - atk = atk[atk.row_time <= atk.hi] - atk = atk.groupby(['pod', 'stack_trace']).agg(count=('count', px.sum)) - - diff = base.merge(atk, how='right', left_on=['stack_trace'], right_on=['stack_trace'], - suffixes=['_base', '_atk']) - diff.pod = diff.pod_atk - diff.stack_trace = px.replace(' ', diff.stack_trace_atk, '') - diff.count = diff.count_atk - diff.delta = diff.count_atk - diff.count_base - - total = atk.groupby(['pod']).agg(total=('count', px.sum)) - merged = diff.merge(total, how='inner', left_on=['pod'], right_on=['pod'], suffixes=['', '_t']) - merged.percent = 100 * merged.count / merged.total - return merged[['stack_trace', 'count', 'delta', 'percent', 'pod']] - - -def orders(start_time: str, clickhouse_dsn: str, graph_table: str): - df = px.DataFrame('dx_anomaly_orders', clickhouse_dsn=clickhouse_dsn, start_time=start_time) - alerts = px.DataFrame(graph_table, clickhouse_dsn=clickhouse_dsn, start_time=start_time) - alerts = alerts[['uniqueID', 'rule', 'alert']] - df = df.merge(alerts, how='left', left_on=['uniqueID', 'rule'], - right_on=['uniqueID', 'rule'], suffixes=['', '_k']) - df.order = px.script_reference(df.order_id, 'px/dx_evidence_graph', { - 'start_time': start_time, - 'clickhouse_dsn': clickhouse_dsn, - 'graph_table': graph_table, - 'order_id': df.order_id, - }) - return df[['order', 'rule', 'alert', 'pod']] + return _ord(start_time, clickhouse_dsn, 'dx_ord__stack_trace', order_id) diff --git a/src/pxl_scripts/px/dx_evidence_graph/vis.json b/src/pxl_scripts/px/dx_evidence_graph/vis.json index 2ad4bafc5bb..dba495a3f9e 100644 --- a/src/pxl_scripts/px/dx_evidence_graph/vis.json +++ b/src/pxl_scripts/px/dx_evidence_graph/vis.json @@ -1,36 +1,195 @@ { - "variables": [ - {"name": "start_time", "type": "PX_STRING", "description": "Window start.", "defaultValue": "-6h"}, - {"name": "clickhouse_dsn", "type": "PX_STRING", "description": "forensic_db DSN: user:pass@host:port/db.", "defaultValue": "forensic_analyst:changeme-analyst@clickhouse-forensic-soc-db.clickhouse.svc.cluster.local:9000/forensic_db"}, - {"name": "graph_table", "type": "PX_STRING", "description": "L1 kill-chain graph source.", "defaultValue": "dx_kubescape_anomalies"}, - {"name": "order_id", "type": "PX_STRING", "description": "Set by clicking an order link in the graph popup or the ORDERS panel; every protocol panel snaps to that order's consulted records.", "defaultValue": ""} - ], - "globalFuncs": [ - {"outputName": "g_graph", "func": {"name": "evidence_graph", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "table", "variable": "graph_table"}]}}, - {"outputName": "g_kube", "func": {"name": "kubescape", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "order_id", "variable": "order_id"}]}}, - {"outputName": "g_redis", "func": {"name": "redis", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "order_id", "variable": "order_id"}]}}, - {"outputName": "g_conn", "func": {"name": "conn", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "order_id", "variable": "order_id"}]}}, - {"outputName": "g_http", "func": {"name": "http", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "order_id", "variable": "order_id"}]}}, - {"outputName": "g_dns", "func": {"name": "dns", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "order_id", "variable": "order_id"}]}}, - {"outputName": "g_pgsql", "func": {"name": "pgsql", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "order_id", "variable": "order_id"}]}}, - {"outputName": "g_mysql", "func": {"name": "mysql", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "order_id", "variable": "order_id"}]}}, - {"outputName": "g_dcsnoop", "func": {"name": "dc_snoop", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "order_id", "variable": "order_id"}]}}, - {"outputName": "g_stack", "func": {"name": "stack_trace", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "order_id", "variable": "order_id"}]}}, - {"outputName": "g_stackdiff", "func": {"name": "stack_diff", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "order_id", "variable": "order_id"}]}}, - {"outputName": "g_orders", "func": {"name": "orders", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "graph_table", "variable": "graph_table"}]}} - ], - "widgets": [ - {"name": "Evidence graph (subject pod -> target; click an edge for details + order link)", "position": {"x": 0, "y": 0, "w": 12, "h": 4}, "globalFuncOutputName": "g_graph", "displaySpec": {"@type": "types.px.dev/px.vispb.Graph", "adjacencyList": {"fromColumn": "from_entity", "toColumn": "to_entity"}, "edgeWeightColumn": "severity", "edgeColorColumn": "severity", "edgeLabelColumn": "rule", "edgeThresholds": {"mediumThreshold": 5, "highThreshold": 8}, "edgeHoverInfo": ["order_link", "rule", "alert", "process", "target", "severity", "uniqueID"], "edgeLength": 500}}, - {"name": "ORDERS (click an order to filter all panels)", "position": {"x": 0, "y": 4, "w": 12, "h": 3}, "globalFuncOutputName": "g_orders", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, - {"name": "kubescape_logs (anomalies) for order", "position": {"x": 0, "y": 7, "w": 12, "h": 4}, "globalFuncOutputName": "g_kube", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, - {"name": "redis_events", "position": {"x": 0, "y": 11, "w": 6, "h": 4}, "globalFuncOutputName": "g_redis", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, - {"name": "conn_stats", "position": {"x": 6, "y": 11, "w": 6, "h": 4}, "globalFuncOutputName": "g_conn", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, - {"name": "http_events", "position": {"x": 0, "y": 15, "w": 6, "h": 4}, "globalFuncOutputName": "g_http", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, - {"name": "dns_events", "position": {"x": 6, "y": 15, "w": 6, "h": 4}, "globalFuncOutputName": "g_dns", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, - {"name": "pgsql_events", "position": {"x": 0, "y": 19, "w": 6, "h": 4}, "globalFuncOutputName": "g_pgsql", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, - {"name": "mysql_events", "position": {"x": 6, "y": 19, "w": 6, "h": 4}, "globalFuncOutputName": "g_mysql", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, - {"name": "dc_snoop (file access)", "position": {"x": 0, "y": 23, "w": 6, "h": 4}, "globalFuncOutputName": "g_dcsnoop", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, - {"name": "stack_trace (profiler)", "position": {"x": 6, "y": 23, "w": 6, "h": 4}, "globalFuncOutputName": "g_stack", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, - {"name": "Differential stack trace — attack window vs baseline (red = spiked during attack)", "position": {"x": 0, "y": 27, "w": 12, "h": 7}, "globalFuncOutputName": "g_stackdiff", "displaySpec": {"@type": "types.px.dev/px.vispb.StackTraceFlameGraph", "stacktraceColumn": "stack_trace", "countColumn": "count", "percentageColumn": "percent", "podColumn": "pod", "differenceColumn": "delta"}} - ] + "variables": [ + { + "name": "start_time", + "type": "PX_STRING", + "description": "Window start.", + "defaultValue": "-6h" + }, + { + "name": "clickhouse_dsn", + "type": "PX_STRING", + "description": "forensic_db DSN: user:pass@host:port/db.", + "defaultValue": "forensic_analyst:changeme-analyst@clickhouse-forensic-soc-db.clickhouse.svc.cluster.local:9000/forensic_db" + }, + { + "name": "graph_table", + "type": "PX_STRING", + "description": "L1 kill-chain graph source.", + "defaultValue": "dx_kubescape_anomalies" + }, + { + "name": "order_id", + "type": "PX_STRING", + "description": "Set by clicking an order link in the graph popup or the ORDERS panel; every protocol panel snaps to that order's consulted records.", + "defaultValue": "" + } + ], + "globalFuncs": [ + { + "outputName": "g_graph", + "func": {"name": "evidence_graph", "args": [ + {"name": "start_time", "variable": "start_time"}, + {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, + {"name": "table", "variable": "graph_table"} + ]} + }, + { + "outputName": "g_orders", + "func": {"name": "orders", "args": [ + {"name": "start_time", "variable": "start_time"}, + {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, + {"name": "graph_table", "variable": "graph_table"} + ]} + }, + { + "outputName": "g_kube", + "func": {"name": "kubescape", "args": [ + {"name": "start_time", "variable": "start_time"}, + {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, + {"name": "order_id", "variable": "order_id"} + ]} + }, + { + "outputName": "g_conn", + "func": {"name": "conn", "args": [ + {"name": "start_time", "variable": "start_time"}, + {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, + {"name": "order_id", "variable": "order_id"} + ]} + }, + { + "outputName": "g_redis", + "func": {"name": "redis", "args": [ + {"name": "start_time", "variable": "start_time"}, + {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, + {"name": "order_id", "variable": "order_id"} + ]} + }, + { + "outputName": "g_http", + "func": {"name": "http", "args": [ + {"name": "start_time", "variable": "start_time"}, + {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, + {"name": "order_id", "variable": "order_id"} + ]} + }, + { + "outputName": "g_dns", + "func": {"name": "dns", "args": [ + {"name": "start_time", "variable": "start_time"}, + {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, + {"name": "order_id", "variable": "order_id"} + ]} + }, + { + "outputName": "g_pgsql", + "func": {"name": "pgsql", "args": [ + {"name": "start_time", "variable": "start_time"}, + {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, + {"name": "order_id", "variable": "order_id"} + ]} + }, + { + "outputName": "g_mysql", + "func": {"name": "mysql", "args": [ + {"name": "start_time", "variable": "start_time"}, + {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, + {"name": "order_id", "variable": "order_id"} + ]} + }, + { + "outputName": "g_dcsnoop", + "func": {"name": "dc_snoop", "args": [ + {"name": "start_time", "variable": "start_time"}, + {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, + {"name": "order_id", "variable": "order_id"} + ]} + }, + { + "outputName": "g_stack", + "func": {"name": "stack_trace", "args": [ + {"name": "start_time", "variable": "start_time"}, + {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, + {"name": "order_id", "variable": "order_id"} + ]} + } + ], + "widgets": [ + { + "name": "Evidence graph (subject pod -> target; click an edge for details + order link)", + "position": {"x": 0, "y": 0, "w": 12, "h": 4}, + "globalFuncOutputName": "g_graph", + "displaySpec": { + "@type": "types.px.dev/px.vispb.Graph", + "adjacencyList": {"fromColumn": "from_entity", "toColumn": "to_entity"}, + "edgeWeightColumn": "severity", + "edgeColorColumn": "severity", + "edgeLabelColumn": "rule", + "edgeThresholds": {"mediumThreshold": 5, "highThreshold": 8}, + "edgeHoverInfo": ["order_link", "rule", "alert", "process", "target", "severity", "uniqueID"], + "edgeLength": 500 + } + }, + { + "name": "ORDERS (click an order to filter all panels)", + "position": {"x": 0, "y": 4, "w": 12, "h": 3}, + "globalFuncOutputName": "g_orders", + "displaySpec": {"@type": "types.px.dev/px.vispb.Table"} + }, + { + "name": "kubescape_logs (anomalies) for order", + "position": {"x": 0, "y": 7, "w": 12, "h": 4}, + "globalFuncOutputName": "g_kube", + "displaySpec": {"@type": "types.px.dev/px.vispb.Table"} + }, + { + "name": "conn_stats (consulted records for order)", + "position": {"x": 0, "y": 11, "w": 6, "h": 4}, + "globalFuncOutputName": "g_conn", + "displaySpec": {"@type": "types.px.dev/px.vispb.Table"} + }, + { + "name": "redis_events (consulted records for order)", + "position": {"x": 6, "y": 11, "w": 6, "h": 4}, + "globalFuncOutputName": "g_redis", + "displaySpec": {"@type": "types.px.dev/px.vispb.Table"} + }, + { + "name": "http_events (consulted records for order)", + "position": {"x": 0, "y": 15, "w": 6, "h": 4}, + "globalFuncOutputName": "g_http", + "displaySpec": {"@type": "types.px.dev/px.vispb.Table"} + }, + { + "name": "dns_events (consulted records for order)", + "position": {"x": 6, "y": 15, "w": 6, "h": 4}, + "globalFuncOutputName": "g_dns", + "displaySpec": {"@type": "types.px.dev/px.vispb.Table"} + }, + { + "name": "pgsql_events (consulted records for order)", + "position": {"x": 0, "y": 19, "w": 6, "h": 4}, + "globalFuncOutputName": "g_pgsql", + "displaySpec": {"@type": "types.px.dev/px.vispb.Table"} + }, + { + "name": "mysql_events (consulted records for order)", + "position": {"x": 6, "y": 19, "w": 6, "h": 4}, + "globalFuncOutputName": "g_mysql", + "displaySpec": {"@type": "types.px.dev/px.vispb.Table"} + }, + { + "name": "dc_snoop (file access) for order", + "position": {"x": 0, "y": 23, "w": 6, "h": 4}, + "globalFuncOutputName": "g_dcsnoop", + "displaySpec": {"@type": "types.px.dev/px.vispb.Table"} + }, + { + "name": "stack_trace (profiler) for order", + "position": {"x": 6, "y": 23, "w": 6, "h": 4}, + "globalFuncOutputName": "g_stack", + "displaySpec": {"@type": "types.px.dev/px.vispb.Table"} + } + ] } diff --git a/src/vizier/services/adaptive_export/cmd/main.go b/src/vizier/services/adaptive_export/cmd/main.go index 38e46e3260e..5338ca3e190 100644 --- a/src/vizier/services/adaptive_export/cmd/main.go +++ b/src/vizier/services/adaptive_export/cmd/main.go @@ -201,6 +201,21 @@ const ( envReconcile = "ADAPTIVE_RECONCILE" ) +// bridgedPushSkip lists the tables dx hands directly via POST /dx/rows (pre-stamped +// unique_id + a dx_ord__ view). They must be excluded from the ADAPTIVE_PUSH_PIXIE_ROWS +// push path or the un-stamped push write collapses the dx-handed rows in +// ReplacingMergeTree. Mirrors the /dx/rows allowlist and dx evidencegraph.UIDColsByTable. +var bridgedPushSkip = map[string]bool{ + "conn_stats": true, + "redis_events": true, + "http_events": true, + "dns_events": true, + "pgsql_events": true, + "mysql_events": true, + "dc_snoop": true, + "stack_trace": true, +} + func main() { // Wire AE into the shared pixie service scaffold: // - SetupService registers --version + ports. @@ -486,6 +501,15 @@ func main() { log.WithField("table", t).Info("skipping dotted-name table from push list — PxL DataFrame rejects it") continue } + // Bridged tables are dx-handed only (POST /dx/rows with a pre-stamped + // unique_id). A push-path write here would land rows WITHOUT unique_id, + // which ReplacingMergeTree collapses against the dx-handed rows + // (unique_id is not in the ORDER BY) — destroying the order↔row join. + // Skip them (mirrors dx evidencegraph.UIDColsByTable / the /dx/rows allowlist). + if bridgedPushSkip[t] { + log.WithField("table", t).Info("skipping bridged table from push list — dx-handed via /dx/rows") + continue + } tables = append(tables, t) } ctlCfg.PushPixieTables = tables @@ -758,6 +782,7 @@ func main() { ctrlSrv := control.New(activeSet, ctl) ctrlSrv.SetGraphWriter(applier) // dx_evidence_graph ingest → ClickHouse ctrlSrv.SetManifestWriter(applier) // dx_evidence_manifest ingest → ClickHouse + ctrlSrv.SetRowsWriter(snk) // /dx/rows (loop-1 dx-handed base rows) → ClickHouse // Bearer-JWT auth default-ON whenever a signing key is present. Same // shared lib + signing key the broker/PEM use — dx attaches the service // JWT it already mints. No key is only reachable with CONTROL_INSECURE. diff --git a/src/vizier/services/adaptive_export/internal/clickhouse/apply.go b/src/vizier/services/adaptive_export/internal/clickhouse/apply.go index f65e8b05d22..285a8e9c14e 100644 --- a/src/vizier/services/adaptive_export/internal/clickhouse/apply.go +++ b/src/vizier/services/adaptive_export/internal/clickhouse/apply.go @@ -90,6 +90,9 @@ var OperatorOwnedTables = []string{ // consulted-records bridge (#136 stamping) — created on boot so dx's INSERT has // a target. Not a pixie table → not in PixieTables(). "dx_order_records", + // NEW identity-model tables (added alongside; dx INSERTs). Not pixie tables. + "dx_orders", + "dx_order_edges", // order-UUID pre-correlation VIEWS (#136) — created LAST, after every base // table above exists (kubescape_logs, the socket_tracer tables, dc_snoop, // dx_order_seeds). Read by the px/dx_evidence_graph dashboard. Not pixie tables. @@ -104,6 +107,15 @@ var OperatorOwnedTables = []string{ "dx_src__mysql_events", "dx_src__dc_snoop", "dx_src__stack_trace", + // NEW identity-model join views (after their base table + dx_order_edges exist). + "dx_ord__conn_stats", + "dx_ord__redis_events", + "dx_ord__http_events", + "dx_ord__dns_events", + "dx_ord__pgsql_events", + "dx_ord__mysql_events", + "dx_ord__dc_snoop", + "dx_ord__stack_trace", } // Applier applies operator-owned DDL to a ClickHouse cluster over the diff --git a/src/vizier/services/adaptive_export/internal/clickhouse/apply_test.go b/src/vizier/services/adaptive_export/internal/clickhouse/apply_test.go index 4c1291cee70..d7243b2850f 100644 --- a/src/vizier/services/adaptive_export/internal/clickhouse/apply_test.go +++ b/src/vizier/services/adaptive_export/internal/clickhouse/apply_test.go @@ -235,8 +235,8 @@ func TestOperatorOwnedTables_DoesNotIncludeKubescape(t *testing.T) { // own write targets in declared order. func TestOperatorOwnedTables_TrailingOperatorTables(t *testing.T) { want := []string{ - "adaptive_attribution", "trigger_watermark", "ae_reconcile", "dx_evidence_graph", "dx_evidence_graph_malignant", "dx_evidence_manifest", "dx_order_seeds", "dx_order_records", - "dx_anomaly_orders", "dx_kubescape_anomalies", "dx_src__kubescape_logs", "dx_src__redis_events", "dx_src__conn_stats", "dx_src__http_events", "dx_src__dns_events", "dx_src__pgsql_events", "dx_src__mysql_events", "dx_src__dc_snoop", "dx_src__stack_trace", + "adaptive_attribution", "trigger_watermark", "ae_reconcile", "dx_evidence_graph", "dx_evidence_graph_malignant", "dx_evidence_manifest", "dx_order_seeds", "dx_order_records", "dx_orders", "dx_order_edges", + "dx_anomaly_orders", "dx_kubescape_anomalies", "dx_src__kubescape_logs", "dx_src__redis_events", "dx_src__conn_stats", "dx_src__http_events", "dx_src__dns_events", "dx_src__pgsql_events", "dx_src__mysql_events", "dx_src__dc_snoop", "dx_src__stack_trace", "dx_ord__conn_stats", "dx_ord__redis_events", "dx_ord__http_events", "dx_ord__dns_events", "dx_ord__pgsql_events", "dx_ord__mysql_events", "dx_ord__dc_snoop", "dx_ord__stack_trace", } got := OperatorOwnedTables[len(OperatorOwnedTables)-len(want):] for i, w := range want { diff --git a/src/vizier/services/adaptive_export/internal/clickhouse/columns_test.go b/src/vizier/services/adaptive_export/internal/clickhouse/columns_test.go index 2e3a94bfb73..6a26fb611dc 100644 --- a/src/vizier/services/adaptive_export/internal/clickhouse/columns_test.go +++ b/src/vizier/services/adaptive_export/internal/clickhouse/columns_test.go @@ -37,7 +37,7 @@ func TestColumns_http_events_ExactList(t *testing.T) { "content_type", "req_headers", "req_method", "req_path", "req_body", "req_body_size", "resp_headers", "resp_status", "resp_message", "resp_body", "resp_body_size", "latency", - "hostname", "event_time", + "hostname", "event_time", "unique_id", } if !reflect.DeepEqual(got, want) { t.Fatalf("Columns(http_events) mismatch:\n got=%v\nwant=%v", got, want) @@ -56,7 +56,7 @@ func TestColumns_conn_stats_ExactList(t *testing.T) { "time_", "upid", "namespace", "pod", "remote_addr", "remote_port", "trace_role", "addr_family", "protocol", "ssl", "conn_open", "conn_close", "conn_active", - "bytes_sent", "bytes_recv", "hostname", "event_time", + "bytes_sent", "bytes_recv", "hostname", "event_time", "unique_id", } if !reflect.DeepEqual(got, want) { t.Fatalf("Columns(conn_stats) mismatch:\n got=%v\nwant=%v", got, want) diff --git a/src/vizier/services/adaptive_export/internal/clickhouse/ddl.go b/src/vizier/services/adaptive_export/internal/clickhouse/ddl.go index 999b216c015..be131be8984 100644 --- a/src/vizier/services/adaptive_export/internal/clickhouse/ddl.go +++ b/src/vizier/services/adaptive_export/internal/clickhouse/ddl.go @@ -92,6 +92,9 @@ var KnownTables = []string{ // per primary kubescape log, stamped with its order_id. dx INSERTs. NOT a pixie // table. "dx_order_records", + // NEW identity-model tables (added alongside dx_order_seeds/records). NOT pixie tables. + "dx_orders", + "dx_order_edges", // order-UUID pre-correlation views (#136) read by the px/dx_evidence_graph // dashboard. VIEWS, created after their base tables (kubescape_logs ensured // first). NOT pixie tables. Order matches schema.sql (appended at the end). @@ -106,6 +109,15 @@ var KnownTables = []string{ "dx_src__mysql_events", "dx_src__dc_snoop", "dx_src__stack_trace", + // NEW identity-model join view. + "dx_ord__conn_stats", + "dx_ord__redis_events", + "dx_ord__http_events", + "dx_ord__dns_events", + "dx_ord__pgsql_events", + "dx_ord__mysql_events", + "dx_ord__dc_snoop", + "dx_ord__stack_trace", } // ErrUnknownTable is returned by DDL / Columns when asked for a table diff --git a/src/vizier/services/adaptive_export/internal/clickhouse/schema.sql b/src/vizier/services/adaptive_export/internal/clickhouse/schema.sql index 2f9621da93c..939f067c772 100644 --- a/src/vizier/services/adaptive_export/internal/clickhouse/schema.sql +++ b/src/vizier/services/adaptive_export/internal/clickhouse/schema.sql @@ -107,7 +107,8 @@ CREATE TABLE IF NOT EXISTS forensic_db.http_events ( resp_body_size Int64, latency Int64, hostname String, - event_time DateTime64(9, 'UTC') DEFAULT toDateTime64(time_, 9) + event_time DateTime64(9, 'UTC') DEFAULT toDateTime64(time_, 9), + unique_id String DEFAULT '' ) ENGINE = ReplacingMergeTree() PARTITION BY toYYYYMM(event_time) ORDER BY (hostname, event_time, time_, upid, trace_role, remote_port, local_port, latency, req_method, req_path); @@ -152,7 +153,8 @@ CREATE TABLE IF NOT EXISTS forensic_db.dns_events ( resp_body String, latency Int64, hostname String, - event_time DateTime64(9, 'UTC') DEFAULT toDateTime64(time_, 9) + event_time DateTime64(9, 'UTC') DEFAULT toDateTime64(time_, 9), + unique_id String DEFAULT '' ) ENGINE = ReplacingMergeTree() PARTITION BY toYYYYMM(event_time) ORDER BY (hostname, event_time, time_, upid, trace_role, remote_port, local_port, latency, req_body); @@ -174,7 +176,8 @@ CREATE TABLE IF NOT EXISTS forensic_db.redis_events ( resp String, latency Int64, hostname String, - event_time DateTime64(9, 'UTC') DEFAULT toDateTime64(time_, 9) + event_time DateTime64(9, 'UTC') DEFAULT toDateTime64(time_, 9), + unique_id String DEFAULT '' ) ENGINE = ReplacingMergeTree() PARTITION BY toYYYYMM(event_time) ORDER BY (hostname, event_time, time_, upid, trace_role, remote_port, local_port, latency, req_cmd); @@ -197,7 +200,8 @@ CREATE TABLE IF NOT EXISTS forensic_db.mysql_events ( resp_body String, latency Int64, hostname String, - event_time DateTime64(9, 'UTC') DEFAULT toDateTime64(time_, 9) + event_time DateTime64(9, 'UTC') DEFAULT toDateTime64(time_, 9), + unique_id String DEFAULT '' ) ENGINE = MergeTree() PARTITION BY toYYYYMM(event_time) ORDER BY (hostname, event_time); @@ -218,7 +222,8 @@ CREATE TABLE IF NOT EXISTS forensic_db.pgsql_events ( resp String, latency Int64, hostname String, - event_time DateTime64(9, 'UTC') DEFAULT toDateTime64(time_, 9) + event_time DateTime64(9, 'UTC') DEFAULT toDateTime64(time_, 9), + unique_id String DEFAULT '' ) ENGINE = MergeTree() PARTITION BY toYYYYMM(event_time) ORDER BY (hostname, event_time); @@ -383,7 +388,8 @@ CREATE TABLE IF NOT EXISTS forensic_db.conn_stats ( bytes_sent Int64, bytes_recv Int64, hostname String, - event_time DateTime64(9, 'UTC') DEFAULT toDateTime64(time_, 9) + event_time DateTime64(9, 'UTC') DEFAULT toDateTime64(time_, 9), + unique_id String DEFAULT '' ) ENGINE = ReplacingMergeTree() PARTITION BY toYYYYMM(event_time) ORDER BY (hostname, event_time, time_, upid, remote_addr, remote_port, trace_role); @@ -614,6 +620,63 @@ CREATE TABLE IF NOT EXISTS forensic_db.dx_order_records ( TTL toDateTime(fromUnixTimestamp64Nano(event_time)) + INTERVAL 30 DAY DELETE SETTINGS index_granularity = 8192; +-- ── NEW identity model (added ALONGSIDE dx_order_seeds/records, which stay) ─── +-- dx_orders — one row per kubescape detection INSTANT. order_id is TRULY unique = +-- hash(uniqueID|Disc|event_time_ns). kubescape_uid/disc are provenance only, NEVER +-- keys. dx INSERTs; AE owns the DDL. +CREATE TABLE IF NOT EXISTS forensic_db.dx_orders ( + order_id String, + kubescape_uid String, + rule_id String, + disc String, + pod String, + event_time UInt64, + hostname String +) ENGINE = ReplacingMergeTree() + ORDER BY (order_id) + PARTITION BY toYYYYMM(fromUnixTimestamp64Nano(event_time)) + TTL toDateTime(fromUnixTimestamp64Nano(event_time)) + INTERVAL 30 DAY DELETE + SETTINGS index_granularity = 8192; + +-- dx_order_edges — the identity bridge. One row per (order, consulted pixie row): +-- links order_id to a base-table row via unique_id = the dx-computed content hash +-- of the row's fields (FNV-1a 64, lowercase hex String), the SAME value dx stamps +-- onto that base row's unique_id column — so they match by construction, no +-- CH-side hashing. String (not UInt64): a 64-bit integer does not survive a JSON +-- decode through float64. Many-to-many: a row consulted by N orders → N edges; +-- re-stamps collapse. dx INSERTs. +CREATE TABLE IF NOT EXISTS forensic_db.dx_order_edges ( + order_id String, + src_table String, + unique_id String, + hostname String, + event_time UInt64 DEFAULT 0 +) ENGINE = ReplacingMergeTree() + ORDER BY (order_id, src_table, unique_id) + SETTINGS index_granularity = 8192; + +-- dx_ord__conn_stats — join view: conn_stats rows consulted for an order, via the +-- bridge (edge.unique_id = conn_stats.unique_id). Panel filters by order_id. +CREATE VIEW IF NOT EXISTS forensic_db.dx_ord__conn_stats AS +SELECT + e.order_id AS order_id, + toString(c.time_) AS ts, + toInt64(toUnixTimestamp64Nano(c.time_)) AS row_time, + toUInt64(toUnixTimestamp64Nano(c.event_time)) AS event_time, + c.namespace AS namespace, + c.pod AS pod, + c.remote_addr AS remote_addr, + c.remote_port AS remote_port, + c.protocol AS protocol, + c.conn_open AS conn_open, + c.conn_close AS conn_close, + c.conn_active AS conn_active, + c.bytes_sent AS bytes_sent, + e.hostname AS hostname +FROM forensic_db.dx_order_edges AS e +INNER JOIN forensic_db.conn_stats AS c ON c.unique_id = e.unique_id +WHERE e.src_table = 'conn_stats'; + -- ── dx dark-vector tracepoint tables (entlein/dx#126) ──────────────────────── -- Fed by AE-owned bpftrace UpsertTracepoint probes (constantly enabled, no TTL). -- Emit raw kernel pid+comm (NOT upid); namespace/pod enriched at pull time via a @@ -707,7 +770,8 @@ CREATE TABLE IF NOT EXISTS forensic_db.dc_snoop ( pod String, container String, hostname String, - event_time DateTime64(9, 'UTC') + event_time DateTime64(9, 'UTC'), + unique_id String DEFAULT '' ) ENGINE = ReplacingMergeTree ORDER BY (time_, pid, comm, t, file, pod); -- stack_trace (native continuous profiler stack_traces.beta, V9) — OTel export. @@ -721,7 +785,8 @@ CREATE TABLE IF NOT EXISTS forensic_db.stack_trace ( stack_trace_id Int64, stack_trace String, count Int64, - event_time DateTime64(9, 'UTC') + event_time DateTime64(9, 'UTC'), + unique_id String DEFAULT '' ) ENGINE = ReplacingMergeTree ORDER BY (time_, upid, stack_trace_id, pod); -- creds_change (commit_creds privilege-escalation to root, V7) — OTel export. @@ -825,3 +890,135 @@ CREATE VIEW IF NOT EXISTS forensic_db.dx_src__stack_trace AS SELECT toString(time_) AS ts, toInt64(toUnixTimestamp64Nano(time_)) AS row_time, toUInt64(toUnixTimestamp64Nano(event_time)) AS event_time, namespace, pod, container, stack_trace_id, stack_trace, count, hostname FROM forensic_db.stack_trace; + +CREATE VIEW IF NOT EXISTS forensic_db.dx_ord__redis_events AS +SELECT + e.order_id AS order_id, + toString(c.time_) AS ts, + toInt64(toUnixTimestamp64Nano(c.time_)) AS row_time, + toUInt64(toUnixTimestamp64Nano(c.event_time)) AS event_time, + c.namespace AS namespace, + c.pod AS pod, + c.remote_addr AS remote_addr, + c.remote_port AS remote_port, + c.trace_role AS trace_role, + c.req_cmd AS req_cmd, + c.req_args AS req_args, + c.resp AS resp, + c.latency AS latency, + e.hostname AS hostname +FROM forensic_db.dx_order_edges AS e +INNER JOIN forensic_db.redis_events AS c ON c.unique_id = e.unique_id +WHERE e.src_table = 'redis_events'; + +CREATE VIEW IF NOT EXISTS forensic_db.dx_ord__http_events AS +SELECT + e.order_id AS order_id, + toString(c.time_) AS ts, + toInt64(toUnixTimestamp64Nano(c.time_)) AS row_time, + toUInt64(toUnixTimestamp64Nano(c.event_time)) AS event_time, + c.namespace AS namespace, + c.pod AS pod, + c.remote_addr AS remote_addr, + c.remote_port AS remote_port, + c.req_method AS req_method, + c.req_path AS req_path, + c.req_body AS req_body, + c.resp_status AS resp_status, + c.resp_body AS resp_body, + c.latency AS latency, + e.hostname AS hostname +FROM forensic_db.dx_order_edges AS e +INNER JOIN forensic_db.http_events AS c ON c.unique_id = e.unique_id +WHERE e.src_table = 'http_events'; + +CREATE VIEW IF NOT EXISTS forensic_db.dx_ord__dns_events AS +SELECT + e.order_id AS order_id, + toString(c.time_) AS ts, + toInt64(toUnixTimestamp64Nano(c.time_)) AS row_time, + toUInt64(toUnixTimestamp64Nano(c.event_time)) AS event_time, + c.namespace AS namespace, + c.pod AS pod, + c.remote_addr AS remote_addr, + c.remote_port AS remote_port, + c.req_body AS req_body, + c.resp_body AS resp_body, + c.latency AS latency, + e.hostname AS hostname +FROM forensic_db.dx_order_edges AS e +INNER JOIN forensic_db.dns_events AS c ON c.unique_id = e.unique_id +WHERE e.src_table = 'dns_events'; + +CREATE VIEW IF NOT EXISTS forensic_db.dx_ord__pgsql_events AS +SELECT + e.order_id AS order_id, + toString(c.time_) AS ts, + toInt64(toUnixTimestamp64Nano(c.time_)) AS row_time, + toUInt64(toUnixTimestamp64Nano(c.event_time)) AS event_time, + c.namespace AS namespace, + c.pod AS pod, + c.remote_addr AS remote_addr, + c.remote_port AS remote_port, + c.req AS req, + c.resp AS resp, + c.latency AS latency, + e.hostname AS hostname +FROM forensic_db.dx_order_edges AS e +INNER JOIN forensic_db.pgsql_events AS c ON c.unique_id = e.unique_id +WHERE e.src_table = 'pgsql_events'; + +CREATE VIEW IF NOT EXISTS forensic_db.dx_ord__mysql_events AS +SELECT + e.order_id AS order_id, + toString(c.time_) AS ts, + toInt64(toUnixTimestamp64Nano(c.time_)) AS row_time, + toUInt64(toUnixTimestamp64Nano(c.event_time)) AS event_time, + c.namespace AS namespace, + c.pod AS pod, + c.remote_addr AS remote_addr, + c.remote_port AS remote_port, + c.req_cmd AS req_cmd, + c.req_body AS req_body, + c.resp_status AS resp_status, + c.resp_body AS resp_body, + c.latency AS latency, + e.hostname AS hostname +FROM forensic_db.dx_order_edges AS e +INNER JOIN forensic_db.mysql_events AS c ON c.unique_id = e.unique_id +WHERE e.src_table = 'mysql_events'; + +CREATE VIEW IF NOT EXISTS forensic_db.dx_ord__dc_snoop AS +SELECT + e.order_id AS order_id, + toString(c.time_) AS ts, + toInt64(toUnixTimestamp64Nano(c.time_)) AS row_time, + toUInt64(toUnixTimestamp64Nano(c.event_time)) AS event_time, + c.pid AS pid, + c.comm AS comm, + c.t AS t, + c.file AS file, + c.namespace AS namespace, + c.pod AS pod, + c.container AS container, + e.hostname AS hostname +FROM forensic_db.dx_order_edges AS e +INNER JOIN forensic_db.dc_snoop AS c ON c.unique_id = e.unique_id +WHERE e.src_table = 'dc_snoop'; + +CREATE VIEW IF NOT EXISTS forensic_db.dx_ord__stack_trace AS +SELECT + e.order_id AS order_id, + toString(c.time_) AS ts, + toInt64(toUnixTimestamp64Nano(c.time_)) AS row_time, + toUInt64(toUnixTimestamp64Nano(c.event_time)) AS event_time, + c.namespace AS namespace, + c.pod AS pod, + c.container AS container, + c.stack_trace_id AS stack_trace_id, + c.stack_trace AS stack_trace, + c.count AS count, + e.hostname AS hostname +FROM forensic_db.dx_order_edges AS e +INNER JOIN forensic_db.stack_trace AS c ON c.unique_id = e.unique_id +WHERE e.src_table = 'stack_trace'; diff --git a/src/vizier/services/adaptive_export/internal/control/server.go b/src/vizier/services/adaptive_export/internal/control/server.go index 1f049af5f28..d18c61a194c 100644 --- a/src/vizier/services/adaptive_export/internal/control/server.go +++ b/src/vizier/services/adaptive_export/internal/control/server.go @@ -34,6 +34,7 @@ import ( "strings" "time" + log "github.com/sirupsen/logrus" jwtutils "px.dev/pixie/src/shared/services/utils" "px.dev/pixie/src/vizier/services/adaptive_export/internal/activeset" "px.dev/pixie/src/vizier/services/adaptive_export/internal/anomaly" @@ -90,12 +91,21 @@ type manifestWriter interface { WriteEvidenceManifest(ctx context.Context, jsonEachRow []byte) error } +// rowsWriter persists dx-handed pixie base rows (loop 1: conn_stats with a +// pre-stamped unique_id) into forensic_db.
through the SAME sink the +// controller capture path uses (sink.ClickHouseHTTP.WritePixieRows). +// nil → /dx/rows 501s. +type rowsWriter interface { + WritePixieRows(ctx context.Context, table string, rows []map[string]any) error +} + // Server is the control HTTP surface. type Server struct { set exporter runner queryRunner // may be nil; /query then returns 501 graph graphWriter // may be nil; /dx/evidence_graph then returns 501 manifest manifestWriter // may be nil; /dx/evidence_manifest then returns 501 + rows rowsWriter // may be nil; /dx/rows then returns 501 mux *http.ServeMux verify func(bearer string) error // nil → auth disabled; set via SetAuth } @@ -110,6 +120,7 @@ func New(set exporter, runner queryRunner) *Server { s.mux.HandleFunc("/query", s.handleQuery) s.mux.HandleFunc("/dx/evidence_graph", s.handleDXEvidenceGraph) s.mux.HandleFunc("/dx/evidence_manifest", s.handleDXEvidenceManifest) + s.mux.HandleFunc("/dx/rows", s.handleDXRows) return s } @@ -119,6 +130,9 @@ func (s *Server) SetGraphWriter(g graphWriter) { s.graph = g } // SetManifestWriter wires the dx_evidence_manifest sink. func (s *Server) SetManifestWriter(m manifestWriter) { s.manifest = m } +// SetRowsWriter wires the /dx/rows base-row sink (loop 1). +func (s *Server) SetRowsWriter(rw rowsWriter) { s.rows = rw } + // SetAuth turns on bearer-JWT auth for the control surface, verified with the // SAME shared lib + signing key the vizier broker/PEM use (px.dev/pixie/src/ // shared/services/utils). dx already mints a service JWT (GenerateJWTForService, @@ -183,6 +197,59 @@ func (s *Server) handleDXEvidenceGraph(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusAccepted) } +// dxRowsAllowedTables guards /dx/rows against arbitrary-table writes: only the +// bridged tables dx hands base rows for (each carries a pre-stamped unique_id and +// a dx_ord__ view) are accepted. Mirrors evidencegraph.UIDColsByTable on the dx side. +var dxRowsAllowedTables = map[string]bool{ + "conn_stats": true, + "redis_events": true, + "http_events": true, + "dns_events": true, + "pgsql_events": true, + "mysql_events": true, + "dc_snoop": true, + "stack_trace": true, +} + +// dxRowsReq is the /dx/rows wire body: dx-handed base rows for one table. +type dxRowsReq struct { + Table string `json:"table"` + Rows []map[string]any `json:"rows"` +} + +// handleDXRows ingests dx-handed base rows (loop 1: conn_stats carrying a +// pre-stamped content-hash unique_id, a hex String) and writes them to +// forensic_db.
via the same sink path the controller capture uses. +// decodeNumber (UseNumber) keeps large integer columns as json.Number so the +// fast encoder emits exact decimal text; the shared decode() would cast them to +// float64, and the sink's appendFloat renders large values in scientific +// notation, which ClickHouse rejects for Int64/UInt64 columns (whole batch 502). +func (s *Server) handleDXRows(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + if s.rows == nil { + w.WriteHeader(http.StatusNotImplemented) + return + } + var req dxRowsReq + if !decodeNumber(w, r, &req) || !dxRowsAllowedTables[req.Table] { + w.WriteHeader(http.StatusBadRequest) + return + } + if len(req.Rows) == 0 { + w.WriteHeader(http.StatusAccepted) + return + } + if err := s.rows.WritePixieRows(r.Context(), req.Table, req.Rows); err != nil { + log.WithField("table", req.Table).WithField("rows", len(req.Rows)).WithError(err).Error("dx/rows: WritePixieRows failed") + w.WriteHeader(http.StatusBadGateway) + return + } + w.WriteHeader(http.StatusAccepted) +} + // dxManifest mirrors the wire shape of dx's manifest.Manifest (internal/manifest). // Scalars map to typed forensic_db.dx_evidence_manifest columns; the nested // collections are held as raw JSON and persisted as JSON text in String columns @@ -299,6 +366,14 @@ func decode(w http.ResponseWriter, r *http.Request, v any) bool { return json.NewDecoder(r.Body).Decode(v) == nil } +func decodeNumber(w http.ResponseWriter, r *http.Request, v any) bool { + defer r.Body.Close() + r.Body = http.MaxBytesReader(w, r.Body, maxControlBodyBytes) + dec := json.NewDecoder(r.Body) + dec.UseNumber() + return dec.Decode(v) == nil +} + // ── handlers ────────────────────────────────────────────────────────── func (s *Server) handleHealth(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) From 77de8fa90dd876cb65ddfdafdc9034d110c80fdb Mon Sep 17 00:00:00 2001 From: constanze Date: Fri, 21 Aug 2026 11:01:49 +0000 Subject: [PATCH 54/66] build(ae): add the logrus dep to internal/control (strict deps) server.go picked up a logrus import with the /dx/rows allowlist, but control/BUILD.bazel was not regenerated, so bazel strict-deps failed //k8s/vizier:vizier_images_push on 065f1dcce and aeprod73 never published. Plain go build was unaffected. Also splits the import block the way gci wants (third-party and px.dev in separate groups), which CI's golangci-lint flags otherwise. arc lint is clean over src/vizier/services/adaptive_export/; the remaining repo findings are the pre-existing yamllint ones in k8s/ and skaffold/. --- src/vizier/services/adaptive_export/internal/control/BUILD.bazel | 1 + src/vizier/services/adaptive_export/internal/control/server.go | 1 + 2 files changed, 2 insertions(+) diff --git a/src/vizier/services/adaptive_export/internal/control/BUILD.bazel b/src/vizier/services/adaptive_export/internal/control/BUILD.bazel index 3b39c27bef6..6253c6a54d5 100644 --- a/src/vizier/services/adaptive_export/internal/control/BUILD.bazel +++ b/src/vizier/services/adaptive_export/internal/control/BUILD.bazel @@ -29,6 +29,7 @@ go_library( "//src/shared/services/utils", "//src/vizier/services/adaptive_export/internal/activeset", "//src/vizier/services/adaptive_export/internal/anomaly", + "@com_github_sirupsen_logrus//:logrus", ], ) diff --git a/src/vizier/services/adaptive_export/internal/control/server.go b/src/vizier/services/adaptive_export/internal/control/server.go index d18c61a194c..99831bd339b 100644 --- a/src/vizier/services/adaptive_export/internal/control/server.go +++ b/src/vizier/services/adaptive_export/internal/control/server.go @@ -35,6 +35,7 @@ import ( "time" log "github.com/sirupsen/logrus" + jwtutils "px.dev/pixie/src/shared/services/utils" "px.dev/pixie/src/vizier/services/adaptive_export/internal/activeset" "px.dev/pixie/src/vizier/services/adaptive_export/internal/anomaly" From 4e079a5d89848b96a5d0ce6eb3f8981ea3579332 Mon Sep 17 00:00:00 2001 From: entlein Date: Fri, 21 Aug 2026 17:16:23 +0200 Subject: [PATCH 55/66] pxl_scripts: add dx/evidence_graph (SOC, MITRE, deep-links, differential flamegraph) New script dx/evidence_graph (namespace dx added to the bundle dir list). Description: 'SOC pixie, works only with clickhouse enabled.' - MITRE ATT&CK prominent: edge label = ruleID + technique (rule_mitre), tactic+technique in the edge popup and as leading columns in the kubescape panel. Sourced from BaseRuntimeMetadata.mitreTactic/Technique via the dx_kubescape_mitre / dx_src__kubescape_mitre views. - Order/evidence data model = dx_orders + dx_ord__* (kubescape_uid+rule_id). - Order deep-links (script_reference -> dx/evidence_graph) on graph edges and the ORDERS panel. - Differential stack-trace flamegraph (attack window vs baseline via dx_orders_win lo/hi). px/dx_evidence_graph is untouched. MITRE views are currently rig-only on 6a87e018 (not yet in AE schema). --- src/pxl_scripts/Makefile | 2 +- .../dx/evidence_graph/evidence_graph.pxl | 134 ++++++++++++++++++ .../dx/evidence_graph/manifest.yaml | 4 + src/pxl_scripts/dx/evidence_graph/vis.json | 36 +++++ 4 files changed, 175 insertions(+), 1 deletion(-) create mode 100644 src/pxl_scripts/dx/evidence_graph/evidence_graph.pxl create mode 100644 src/pxl_scripts/dx/evidence_graph/manifest.yaml create mode 100644 src/pxl_scripts/dx/evidence_graph/vis.json diff --git a/src/pxl_scripts/Makefile b/src/pxl_scripts/Makefile index 1cca03f4dc5..4e8a3562658 100644 --- a/src/pxl_scripts/Makefile +++ b/src/pxl_scripts/Makefile @@ -15,7 +15,7 @@ # SPDX-License-Identifier: Apache-2.0 # Update dir name here if you want to add a new directory. -dirs := bpftrace px pxbeta sotw +dirs := bpftrace dx px pxbeta sotw script_files := $(foreach dir,$(dirs),$(wildcard $(dir)/**/*)) EXECUTABLES ?= px diff --git a/src/pxl_scripts/dx/evidence_graph/evidence_graph.pxl b/src/pxl_scripts/dx/evidence_graph/evidence_graph.pxl new file mode 100644 index 00000000000..b31b3236985 --- /dev/null +++ b/src/pxl_scripts/dx/evidence_graph/evidence_graph.pxl @@ -0,0 +1,134 @@ +# Copyright 2018- The Pixie Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# +# SOC pixie evidence graph — works only with ClickHouse enabled. +# MITRE ATT&CK (mitreTactic / mitreTechnique) is surfaced on the graph edges +# and in the kubescape panel. Order/evidence data model = dx_orders + dx_ord__*. + +import px + + +def _ord(start_time: str, clickhouse_dsn: str, view: str, order_id: str): + df = px.DataFrame(view, clickhouse_dsn=clickhouse_dsn, start_time=start_time) + df = df[df.order_id == order_id] + return df.drop(['order_id', 'row_time', 'event_time']) + + +def evidence_graph(start_time: str, clickhouse_dsn: str, table: str): + df = px.DataFrame(table, clickhouse_dsn=clickhouse_dsn, start_time=start_time) + orders = px.DataFrame('dx_orders', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + orders = orders[['kubescape_uid', 'rule_id', 'order_id']] + df = df.merge(orders, how='left', left_on=['uniqueID', 'rule'], + right_on=['kubescape_uid', 'rule_id'], suffixes=['', '_ord']) + df.order_link = px.script_reference(df.order_id, 'dx/evidence_graph', { + 'start_time': start_time, + 'clickhouse_dsn': clickhouse_dsn, + 'graph_table': table, + 'order_id': df.order_id, + }) + df.from_entity = px.Pod(df.subject_pod) + df.to_entity = df.target + return df[['from_entity', 'to_entity', 'order_link', 'rule_mitre', 'mitre_tactic', + 'mitre_technique', 'uniqueID', 'rule', 'process', 'target', 'target_kind', + 'severity', 'alert', 'subject_pod']] + + +def orders(start_time: str, clickhouse_dsn: str, graph_table: str): + df = px.DataFrame('dx_orders', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + alerts = px.DataFrame(graph_table, clickhouse_dsn=clickhouse_dsn, start_time=start_time) + alerts = alerts[['uniqueID', 'rule', 'mitre_tactic', 'mitre_technique', 'alert']] + df = df.merge(alerts, how='left', left_on=['kubescape_uid', 'rule_id'], + right_on=['uniqueID', 'rule'], suffixes=['', '_k']) + df.order = px.script_reference(df.order_id, 'dx/evidence_graph', { + 'start_time': start_time, + 'clickhouse_dsn': clickhouse_dsn, + 'graph_table': graph_table, + 'order_id': df.order_id, + }) + return df[['order', 'rule_id', 'disc', 'mitre_tactic', 'mitre_technique', 'alert', 'pod']] + + +def kubescape(start_time: str, clickhouse_dsn: str, order_id: str): + orders = px.DataFrame('dx_orders', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + orders = orders[orders.order_id == order_id] + orders = orders[['kubescape_uid', 'rule_id']] + k = px.DataFrame('dx_src__kubescape_mitre', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + j = k.merge(orders, how='inner', left_on=['uniqueID', 'RuleID'], + right_on=['kubescape_uid', 'rule_id'], suffixes=['', '_ord']) + return j.drop(['row_time', 'event_time', 'kubescape_uid', 'rule_id']) + + +def conn(start_time: str, clickhouse_dsn: str, order_id: str): + return _ord(start_time, clickhouse_dsn, 'dx_ord__conn_stats', order_id) + + +def redis(start_time: str, clickhouse_dsn: str, order_id: str): + return _ord(start_time, clickhouse_dsn, 'dx_ord__redis_events', order_id) + + +def http(start_time: str, clickhouse_dsn: str, order_id: str): + return _ord(start_time, clickhouse_dsn, 'dx_ord__http_events', order_id) + + +def dns(start_time: str, clickhouse_dsn: str, order_id: str): + return _ord(start_time, clickhouse_dsn, 'dx_ord__dns_events', order_id) + + +def pgsql(start_time: str, clickhouse_dsn: str, order_id: str): + return _ord(start_time, clickhouse_dsn, 'dx_ord__pgsql_events', order_id) + + +def mysql(start_time: str, clickhouse_dsn: str, order_id: str): + return _ord(start_time, clickhouse_dsn, 'dx_ord__mysql_events', order_id) + + +def dc_snoop(start_time: str, clickhouse_dsn: str, order_id: str): + return _ord(start_time, clickhouse_dsn, 'dx_ord__dc_snoop', order_id) + + +def stack_trace(start_time: str, clickhouse_dsn: str, order_id: str): + return _ord(start_time, clickhouse_dsn, 'dx_ord__stack_trace', order_id) + + +def stack_diff(start_time: str, clickhouse_dsn: str, order_id: str): + # Differential flame graph for ONE order's pod: stacks DURING the attack window + # [lo,hi] vs BASELINE stacks before the attack (row_time < lo). lo/hi come from + # dx_orders_win (event_time +/- 300s, Int64). delta = attack - baseline drives the + # StackTraceFlameGraph colouring (red = spiked during the attack). + orders = px.DataFrame('dx_orders_win', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + orders = orders[orders.order_id == order_id] + orders = orders[['pod', 'lo', 'hi']] + st = px.DataFrame('dx_src__stack_trace', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + st = st.merge(orders, how='inner', left_on=['pod'], right_on=['pod'], suffixes=['', '_o']) + + base = st[st.row_time < st.lo] + base = base.groupby(['pod', 'stack_trace']).agg(count=('count', px.sum)) + + atk = st[st.row_time >= st.lo] + atk = atk[atk.row_time <= atk.hi] + atk = atk.groupby(['pod', 'stack_trace']).agg(count=('count', px.sum)) + + diff = base.merge(atk, how='right', left_on=['stack_trace'], right_on=['stack_trace'], + suffixes=['_base', '_atk']) + diff.pod = diff.pod_atk + diff.stack_trace = px.replace(' ', diff.stack_trace_atk, '') + diff.count = diff.count_atk + diff.delta = diff.count_atk - diff.count_base + + total = atk.groupby(['pod']).agg(total=('count', px.sum)) + merged = diff.merge(total, how='inner', left_on=['pod'], right_on=['pod'], suffixes=['', '_t']) + merged.percent = 100 * merged.count / merged.total + return merged[['stack_trace', 'count', 'delta', 'percent', 'pod']] diff --git a/src/pxl_scripts/dx/evidence_graph/manifest.yaml b/src/pxl_scripts/dx/evidence_graph/manifest.yaml new file mode 100644 index 00000000000..59572ec64e3 --- /dev/null +++ b/src/pxl_scripts/dx/evidence_graph/manifest.yaml @@ -0,0 +1,4 @@ +--- +short: SOC Evidence Graph +long: > + SOC pixie, works only with clickhouse enabled. diff --git a/src/pxl_scripts/dx/evidence_graph/vis.json b/src/pxl_scripts/dx/evidence_graph/vis.json new file mode 100644 index 00000000000..f3b883ca085 --- /dev/null +++ b/src/pxl_scripts/dx/evidence_graph/vis.json @@ -0,0 +1,36 @@ +{ + "variables": [ + {"name": "start_time", "type": "PX_STRING", "description": "Window start.", "defaultValue": "-6h"}, + {"name": "clickhouse_dsn", "type": "PX_STRING", "description": "forensic_db DSN: user:pass@host:port/db.", "defaultValue": "forensic_analyst:changeme-analyst@clickhouse-forensic-soc-db.clickhouse.svc.cluster.local:9000/forensic_db"}, + {"name": "graph_table", "type": "PX_STRING", "description": "L1 kill-chain graph source (MITRE-enriched).", "defaultValue": "dx_kubescape_mitre"}, + {"name": "order_id", "type": "PX_STRING", "description": "Set by clicking an order link in the graph popup or the ORDERS panel; every panel snaps to that order.", "defaultValue": ""} + ], + "globalFuncs": [ + {"outputName": "g_graph", "func": {"name": "evidence_graph", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "table", "variable": "graph_table"}]}}, + {"outputName": "g_orders", "func": {"name": "orders", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "graph_table", "variable": "graph_table"}]}}, + {"outputName": "g_kube", "func": {"name": "kubescape", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "order_id", "variable": "order_id"}]}}, + {"outputName": "g_conn", "func": {"name": "conn", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "order_id", "variable": "order_id"}]}}, + {"outputName": "g_redis", "func": {"name": "redis", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "order_id", "variable": "order_id"}]}}, + {"outputName": "g_http", "func": {"name": "http", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "order_id", "variable": "order_id"}]}}, + {"outputName": "g_dns", "func": {"name": "dns", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "order_id", "variable": "order_id"}]}}, + {"outputName": "g_pgsql", "func": {"name": "pgsql", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "order_id", "variable": "order_id"}]}}, + {"outputName": "g_mysql", "func": {"name": "mysql", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "order_id", "variable": "order_id"}]}}, + {"outputName": "g_dcsnoop", "func": {"name": "dc_snoop", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "order_id", "variable": "order_id"}]}}, + {"outputName": "g_stack", "func": {"name": "stack_trace", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "order_id", "variable": "order_id"}]}}, + {"outputName": "g_stackdiff", "func": {"name": "stack_diff", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "order_id", "variable": "order_id"}]}} + ], + "widgets": [ + {"name": "Evidence graph (subject pod -> target; edge = ruleID + MITRE technique; click for details + order link)", "position": {"x": 0, "y": 0, "w": 12, "h": 4}, "globalFuncOutputName": "g_graph", "displaySpec": {"@type": "types.px.dev/px.vispb.Graph", "adjacencyList": {"fromColumn": "from_entity", "toColumn": "to_entity"}, "edgeWeightColumn": "severity", "edgeColorColumn": "severity", "edgeLabelColumn": "rule_mitre", "edgeThresholds": {"mediumThreshold": 5, "highThreshold": 8}, "edgeHoverInfo": ["order_link", "rule", "mitre_tactic", "mitre_technique", "alert", "process", "target", "severity", "uniqueID"], "edgeLength": 500}}, + {"name": "ORDERS (click an order to filter all panels)", "position": {"x": 0, "y": 4, "w": 12, "h": 3}, "globalFuncOutputName": "g_orders", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, + {"name": "kubescape_logs for order (MITRE tactic + technique)", "position": {"x": 0, "y": 7, "w": 12, "h": 4}, "globalFuncOutputName": "g_kube", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, + {"name": "conn_stats", "position": {"x": 0, "y": 11, "w": 6, "h": 4}, "globalFuncOutputName": "g_conn", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, + {"name": "redis_events", "position": {"x": 6, "y": 11, "w": 6, "h": 4}, "globalFuncOutputName": "g_redis", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, + {"name": "http_events", "position": {"x": 0, "y": 15, "w": 6, "h": 4}, "globalFuncOutputName": "g_http", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, + {"name": "dns_events", "position": {"x": 6, "y": 15, "w": 6, "h": 4}, "globalFuncOutputName": "g_dns", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, + {"name": "pgsql_events", "position": {"x": 0, "y": 19, "w": 6, "h": 4}, "globalFuncOutputName": "g_pgsql", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, + {"name": "mysql_events", "position": {"x": 6, "y": 19, "w": 6, "h": 4}, "globalFuncOutputName": "g_mysql", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, + {"name": "dc_snoop (file access)", "position": {"x": 0, "y": 23, "w": 6, "h": 4}, "globalFuncOutputName": "g_dcsnoop", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, + {"name": "stack_trace (profiler)", "position": {"x": 6, "y": 23, "w": 6, "h": 4}, "globalFuncOutputName": "g_stack", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, + {"name": "Differential stack trace (attack window vs baseline; red = spiked during attack)", "position": {"x": 0, "y": 27, "w": 12, "h": 7}, "globalFuncOutputName": "g_stackdiff", "displaySpec": {"@type": "types.px.dev/px.vispb.StackTraceFlameGraph", "stacktraceColumn": "stack_trace", "countColumn": "count", "percentageColumn": "percent", "podColumn": "pod", "differenceColumn": "delta"}} + ] +} From 88977a3802c362c0ad915f7d11aecc347c7403e2 Mon Sep 17 00:00:00 2001 From: entlein Date: Fri, 21 Aug 2026 18:35:20 +0200 Subject: [PATCH 56/66] adaptive_export: MITRE ATT&CK + per-order window views (operator-owned, boot-created) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Makes the px/dx_evidence_graph MITRE views permanent — AE creates them on boot via apply.go, so no manual DDL per PG: - dx_kubescape_mitre: L1 graph source, one row/anomaly (LIMIT 1 BY uniqueID), MITRE tactic/technique + resolved target/target_kind from BaseRuntimeMetadata. - dx_src__kubescape_mitre: kubescape detail panel (MITRE cols + process tree). - dx_orders_win: per-order ±300s window for the differential flamegraph; carries hostname so the px connector node-shard resolves. Registered in ddl.go KnownTables + apply.go OperatorOwnedTables (VIEWS, created after kubescape_logs / dx_orders); apply_test coverage tail updated. --- .../internal/clickhouse/apply.go | 5 ++ .../internal/clickhouse/apply_test.go | 2 +- .../internal/clickhouse/ddl.go | 4 ++ .../internal/clickhouse/schema.sql | 53 +++++++++++++++++++ 4 files changed, 63 insertions(+), 1 deletion(-) diff --git a/src/vizier/services/adaptive_export/internal/clickhouse/apply.go b/src/vizier/services/adaptive_export/internal/clickhouse/apply.go index 285a8e9c14e..9b5331047d6 100644 --- a/src/vizier/services/adaptive_export/internal/clickhouse/apply.go +++ b/src/vizier/services/adaptive_export/internal/clickhouse/apply.go @@ -116,6 +116,11 @@ var OperatorOwnedTables = []string{ "dx_ord__mysql_events", "dx_ord__dc_snoop", "dx_ord__stack_trace", + // MITRE ATT&CK enrichment (over kubescape_logs) + per-order window (over + // dx_orders) — VIEWS, created after their base tables. px/dx_evidence_graph reads them. + "dx_kubescape_mitre", + "dx_src__kubescape_mitre", + "dx_orders_win", } // Applier applies operator-owned DDL to a ClickHouse cluster over the diff --git a/src/vizier/services/adaptive_export/internal/clickhouse/apply_test.go b/src/vizier/services/adaptive_export/internal/clickhouse/apply_test.go index d7243b2850f..1abe1061e6c 100644 --- a/src/vizier/services/adaptive_export/internal/clickhouse/apply_test.go +++ b/src/vizier/services/adaptive_export/internal/clickhouse/apply_test.go @@ -236,7 +236,7 @@ func TestOperatorOwnedTables_DoesNotIncludeKubescape(t *testing.T) { func TestOperatorOwnedTables_TrailingOperatorTables(t *testing.T) { want := []string{ "adaptive_attribution", "trigger_watermark", "ae_reconcile", "dx_evidence_graph", "dx_evidence_graph_malignant", "dx_evidence_manifest", "dx_order_seeds", "dx_order_records", "dx_orders", "dx_order_edges", - "dx_anomaly_orders", "dx_kubescape_anomalies", "dx_src__kubescape_logs", "dx_src__redis_events", "dx_src__conn_stats", "dx_src__http_events", "dx_src__dns_events", "dx_src__pgsql_events", "dx_src__mysql_events", "dx_src__dc_snoop", "dx_src__stack_trace", "dx_ord__conn_stats", "dx_ord__redis_events", "dx_ord__http_events", "dx_ord__dns_events", "dx_ord__pgsql_events", "dx_ord__mysql_events", "dx_ord__dc_snoop", "dx_ord__stack_trace", + "dx_anomaly_orders", "dx_kubescape_anomalies", "dx_src__kubescape_logs", "dx_src__redis_events", "dx_src__conn_stats", "dx_src__http_events", "dx_src__dns_events", "dx_src__pgsql_events", "dx_src__mysql_events", "dx_src__dc_snoop", "dx_src__stack_trace", "dx_ord__conn_stats", "dx_ord__redis_events", "dx_ord__http_events", "dx_ord__dns_events", "dx_ord__pgsql_events", "dx_ord__mysql_events", "dx_ord__dc_snoop", "dx_ord__stack_trace", "dx_kubescape_mitre", "dx_src__kubescape_mitre", "dx_orders_win", } got := OperatorOwnedTables[len(OperatorOwnedTables)-len(want):] for i, w := range want { diff --git a/src/vizier/services/adaptive_export/internal/clickhouse/ddl.go b/src/vizier/services/adaptive_export/internal/clickhouse/ddl.go index be131be8984..84728fdfe62 100644 --- a/src/vizier/services/adaptive_export/internal/clickhouse/ddl.go +++ b/src/vizier/services/adaptive_export/internal/clickhouse/ddl.go @@ -118,6 +118,10 @@ var KnownTables = []string{ "dx_ord__mysql_events", "dx_ord__dc_snoop", "dx_ord__stack_trace", + // MITRE ATT&CK enrichment + per-order window views (px/dx_evidence_graph). + "dx_kubescape_mitre", + "dx_src__kubescape_mitre", + "dx_orders_win", } // ErrUnknownTable is returned by DDL / Columns when asked for a table diff --git a/src/vizier/services/adaptive_export/internal/clickhouse/schema.sql b/src/vizier/services/adaptive_export/internal/clickhouse/schema.sql index 939f067c772..dd168a42d18 100644 --- a/src/vizier/services/adaptive_export/internal/clickhouse/schema.sql +++ b/src/vizier/services/adaptive_export/internal/clickhouse/schema.sql @@ -1022,3 +1022,56 @@ SELECT FROM forensic_db.dx_order_edges AS e INNER JOIN forensic_db.stack_trace AS c ON c.unique_id = e.unique_id WHERE e.src_table = 'stack_trace'; + +-- ── MITRE ATT&CK enrichment over kubescape_logs (px/dx_evidence_graph) ──────── +-- dx_kubescape_mitre: L1 graph source — one row per anomaly (LIMIT 1 BY uniqueID), +-- MITRE tactic/technique + resolved target/target_kind from BaseRuntimeMetadata. +CREATE VIEW IF NOT EXISTS forensic_db.dx_kubescape_mitre AS +SELECT JSONExtractString(BaseRuntimeMetadata, 'uniqueID') AS uniqueID, + concat(JSONExtractString(RuntimeK8sDetails, 'podNamespace'), '/', JSONExtractString(RuntimeK8sDetails, 'podName')) AS subject_pod, + RuleID AS rule, + JSONExtractString(BaseRuntimeMetadata, 'mitreTactic') AS mitre_tactic, + JSONExtractString(BaseRuntimeMetadata, 'mitreTechnique') AS mitre_technique, + concat(RuleID, ' · ', JSONExtractString(BaseRuntimeMetadata, 'mitreTechnique')) AS rule_mitre, + JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'process'), 'name') AS process, + multiIf(JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'dns'), 'domain') != '', +JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'dns'), 'domain'), +JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'network'), 'dstIP') != '', +JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'network'), 'dstIP'), JSONExtractString(JSONExtractRaw(BaseRuntimeMetadata, +'arguments'), 'path') != '', JSONExtractString(JSONExtractRaw(BaseRuntimeMetadata, 'arguments'), 'path'), +JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'file'), 'name') != '', +concat(JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'file'), 'directory'), '/', +JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'file'), 'name')), 'unknown') AS target, + multiIf(JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'dns'), 'domain') != '', 'domain', +JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'network'), 'dstIP') != '', 'endpoint', +(JSONExtractString(JSONExtractRaw(BaseRuntimeMetadata, 'arguments'), 'path') != '') OR (JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, +'identifiers'), 'file'), 'name') != ''), 'file', 'other') AS target_kind, + toInt8OrZero(JSONExtractString(BaseRuntimeMetadata, 'severity')) AS severity, + message AS alert, hostname, event_time +FROM forensic_db.kubescape_logs +WHERE RuleID != '' AND JSONExtractString(BaseRuntimeMetadata, 'uniqueID') != '' +LIMIT 1 BY uniqueID; + +-- dx_src__kubescape_mitre: kubescape detail panel — MITRE cols after RuleID, plus +-- process tree (comm/pcomm/cmdline). ts/row_time/event_time px-connector convention. +CREATE VIEW IF NOT EXISTS forensic_db.dx_src__kubescape_mitre AS +SELECT toString(fromUnixTimestamp64Nano(toInt64(event_time))) AS ts, toInt64(event_time) AS row_time, event_time, + RuleID, + JSONExtractString(BaseRuntimeMetadata, 'mitreTactic') AS mitre_tactic, + JSONExtractString(BaseRuntimeMetadata, 'mitreTechnique') AS mitre_technique, + JSONExtractString(BaseRuntimeMetadata, 'uniqueID') AS uniqueID, + JSONExtractString(JSONExtractRaw(RuntimeProcessDetails, 'processTree'), 'comm') AS comm, + JSONExtractString(JSONExtractRaw(RuntimeProcessDetails, 'processTree'), 'pcomm') AS parent, + JSONExtractString(JSONExtractRaw(RuntimeProcessDetails, 'processTree'), 'cmdline') AS cmdline, + message AS alert, + concat(JSONExtractString(RuntimeK8sDetails, 'podNamespace'), '/', JSONExtractString(RuntimeK8sDetails, 'podName')) AS pod, hostname +FROM forensic_db.kubescape_logs WHERE RuleID != ''; + +-- dx_orders_win: per-order ±300s baseline/attack window for the differential +-- flamegraph (stack_diff). hostname carried so the px node-shard resolves. +CREATE VIEW IF NOT EXISTS forensic_db.dx_orders_win AS +SELECT order_id, pod, + toInt64(event_time) - 300000000000 AS lo, + toInt64(event_time) + 300000000000 AS hi, + event_time, hostname +FROM forensic_db.dx_orders; From 83bab98f76686ba9916c882e264cf31e84d74784 Mon Sep 17 00:00:00 2001 From: entlein Date: Fri, 21 Aug 2026 19:59:30 +0200 Subject: [PATCH 57/66] dx/evidence_graph: ORDERS disc->Alert, kubescape +pid/ppid -uniqueID, dc_snoop fast path - ORDERS: rename the disc column to 'Alert', drop the old alert column. - kubescape panel: add pid + ppid (from processTree via dx_src__kubescape_mitre), drop uniqueID. - PERF: dc_snoop panel switched from _ord (pre-joined dx_ord__dc_snoop view) to _bridge. The dx_ord__ bridge fans a base row out by every order that consulted it (dc_snoop 6.3k -> 160k rows / 28MB), and the Pixie CH source can't push the order_id filter down (clickhouse_source_ir.cc emits SELECT cols FROM table only) so _ord pulls the whole fanned view. _bridge joins the small base table (0.9MB) to just this order's narrow edge set (9.7MB) = ~10.5MB, ~2.7x less. Measured on rig 6a87e018. Other panels keep _ord (their fan-out is small; _bridge's edge pull would cost more). --- .../dx/evidence_graph/evidence_graph.pxl | 24 +++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/src/pxl_scripts/dx/evidence_graph/evidence_graph.pxl b/src/pxl_scripts/dx/evidence_graph/evidence_graph.pxl index b31b3236985..2abdf3c19a0 100644 --- a/src/pxl_scripts/dx/evidence_graph/evidence_graph.pxl +++ b/src/pxl_scripts/dx/evidence_graph/evidence_graph.pxl @@ -27,6 +27,21 @@ def _ord(start_time: str, clickhouse_dsn: str, view: str, order_id: str): return df.drop(['order_id', 'row_time', 'event_time']) +def _bridge(start_time: str, clickhouse_dsn: str, base_table: str, order_id: str): + # Fast path for high-fan-out tables (dc_snoop): the pre-joined dx_ord__ view + # multiplies a base row by every order that consulted it (dc_snoop 6.3k -> 160k), + # and the Pixie CH connector can't push the order_id filter down, so _ord pulls + # the full fanned-out view (~28MB). Instead join the SMALL base table to just + # this order's narrow edge set -> ~10MB. Same rows, ~2.7x less transfer. + edges = px.DataFrame('dx_order_edges', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + edges = edges[edges.order_id == order_id] + edges = edges[edges.src_table == base_table] + edges = edges[['unique_id']] + src = px.DataFrame(base_table, clickhouse_dsn=clickhouse_dsn, start_time=start_time) + j = src.merge(edges, how='inner', left_on=['unique_id'], right_on=['unique_id'], suffixes=['', '_e']) + return j.drop(['unique_id', 'event_time']) + + def evidence_graph(start_time: str, clickhouse_dsn: str, table: str): df = px.DataFrame(table, clickhouse_dsn=clickhouse_dsn, start_time=start_time) orders = px.DataFrame('dx_orders', clickhouse_dsn=clickhouse_dsn, start_time=start_time) @@ -49,7 +64,7 @@ def evidence_graph(start_time: str, clickhouse_dsn: str, table: str): def orders(start_time: str, clickhouse_dsn: str, graph_table: str): df = px.DataFrame('dx_orders', clickhouse_dsn=clickhouse_dsn, start_time=start_time) alerts = px.DataFrame(graph_table, clickhouse_dsn=clickhouse_dsn, start_time=start_time) - alerts = alerts[['uniqueID', 'rule', 'mitre_tactic', 'mitre_technique', 'alert']] + alerts = alerts[['uniqueID', 'rule', 'mitre_tactic', 'mitre_technique']] df = df.merge(alerts, how='left', left_on=['kubescape_uid', 'rule_id'], right_on=['uniqueID', 'rule'], suffixes=['', '_k']) df.order = px.script_reference(df.order_id, 'dx/evidence_graph', { @@ -58,7 +73,8 @@ def orders(start_time: str, clickhouse_dsn: str, graph_table: str): 'graph_table': graph_table, 'order_id': df.order_id, }) - return df[['order', 'rule_id', 'disc', 'mitre_tactic', 'mitre_technique', 'alert', 'pod']] + df.Alert = df.disc + return df[['order', 'rule_id', 'Alert', 'mitre_tactic', 'mitre_technique', 'pod']] def kubescape(start_time: str, clickhouse_dsn: str, order_id: str): @@ -68,7 +84,7 @@ def kubescape(start_time: str, clickhouse_dsn: str, order_id: str): k = px.DataFrame('dx_src__kubescape_mitre', clickhouse_dsn=clickhouse_dsn, start_time=start_time) j = k.merge(orders, how='inner', left_on=['uniqueID', 'RuleID'], right_on=['kubescape_uid', 'rule_id'], suffixes=['', '_ord']) - return j.drop(['row_time', 'event_time', 'kubescape_uid', 'rule_id']) + return j.drop(['row_time', 'event_time', 'kubescape_uid', 'rule_id', 'uniqueID']) def conn(start_time: str, clickhouse_dsn: str, order_id: str): @@ -96,7 +112,7 @@ def mysql(start_time: str, clickhouse_dsn: str, order_id: str): def dc_snoop(start_time: str, clickhouse_dsn: str, order_id: str): - return _ord(start_time, clickhouse_dsn, 'dx_ord__dc_snoop', order_id) + return _bridge(start_time, clickhouse_dsn, 'dc_snoop', order_id) def stack_trace(start_time: str, clickhouse_dsn: str, order_id: str): From 4c7cb8eeb17a1d8ea96b5ac02199426f22dc035d Mon Sep 17 00:00:00 2001 From: entlein Date: Fri, 21 Aug 2026 20:08:26 +0200 Subject: [PATCH 58/66] deploy: pin AE 0.14.19-aeprod75 (bridge views + MITRE views) + dx 0.5.0-keepset-rc1 (keepset collect-all image) --- k8s/vizier/adaptive_export/kustomization.yaml | 2 +- k8s/vizier/dx/dx-daemon.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/k8s/vizier/adaptive_export/kustomization.yaml b/k8s/vizier/adaptive_export/kustomization.yaml index 2818e8f62bb..35d478ecc96 100644 --- a/k8s/vizier/adaptive_export/kustomization.yaml +++ b/k8s/vizier/adaptive_export/kustomization.yaml @@ -7,4 +7,4 @@ resources: images: - name: vizier-adaptive_export_image newName: ghcr.io/k8sstormcenter/vizier-adaptive_export_image - newTag: 0.14.19-aeprod72 + newTag: 0.14.19-aeprod75 diff --git a/k8s/vizier/dx/dx-daemon.yaml b/k8s/vizier/dx/dx-daemon.yaml index 98256fb38a7..8b9f4a73791 100644 --- a/k8s/vizier/dx/dx-daemon.yaml +++ b/k8s/vizier/dx/dx-daemon.yaml @@ -23,7 +23,7 @@ spec: # (WITHOUT -tiny — -tiny's pclntab stripping SIGSEGV'd under load). Passes the # obfuscation gate AND survives the kill-chain (restarts=0, 4 rounds). Carries the # evidence-manifest + DX_FOREST_PUSHDOWN code. - image: docker.io/entlein/dx-daemon:0.4.0-ssotforest-rc23 + image: docker.io/entlein/dx-daemon:0.5.0-keepset-rc1 ports: - { name: findings, containerPort: 9099, hostPort: 9099 } env: From dc317b048cef0226489147e0895e6b7597b17dbe Mon Sep 17 00:00:00 2001 From: entlein Date: Fri, 21 Aug 2026 20:15:56 +0200 Subject: [PATCH 59/66] bazel: add sha256-verified mirror for org_libc_musl (musl.libc.org is down; same pinned sha 7d5b0b60 from sources.openwrt.org) --- bazel/repository_locations.bzl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 4584d725f9b..c8bda4fbeac 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -456,7 +456,10 @@ REPOSITORY_LOCATIONS = dict( org_libc_musl = dict( sha256 = "7d5b0b6062521e4627e099e4c9dc8248d32a30285e959b7eecaa780cf8cfd4a4", strip_prefix = "musl-1.2.3", - urls = ["http://musl.libc.org/releases/musl-1.2.3.tar.gz"], + urls = [ + "https://sources.openwrt.org/musl-1.2.3.tar.gz", + "http://musl.libc.org/releases/musl-1.2.3.tar.gz", + ], manual_license_name = "libc/musl", ), rules_cc = dict( From 9827dbe58de6acde201268718b48207309d8d8ba Mon Sep 17 00:00:00 2001 From: entlein Date: Fri, 21 Aug 2026 20:16:09 +0200 Subject: [PATCH 60/66] deploy: pin AE 0.14.19-aeprod76 (aeprod75 CI died on musl outage, image never published) --- k8s/vizier/adaptive_export/kustomization.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/vizier/adaptive_export/kustomization.yaml b/k8s/vizier/adaptive_export/kustomization.yaml index 35d478ecc96..35c0987d520 100644 --- a/k8s/vizier/adaptive_export/kustomization.yaml +++ b/k8s/vizier/adaptive_export/kustomization.yaml @@ -7,4 +7,4 @@ resources: images: - name: vizier-adaptive_export_image newName: ghcr.io/k8sstormcenter/vizier-adaptive_export_image - newTag: 0.14.19-aeprod75 + newTag: 0.14.19-aeprod76 From 6512b6985b1994719c9c9aeceb8c70df906b784f Mon Sep 17 00:00:00 2001 From: entlein Date: Fri, 21 Aug 2026 20:27:41 +0200 Subject: [PATCH 61/66] dx/evidence_graph: fix _bridge unique_id compile error via passthrough view px.DataFrame('dc_snoop') uses the registered base-table relation, which omits the fork-added unique_id column (present physically), so src.merge(on unique_id) failed to compile. A VIEW is inferred from ClickHouse via DESCRIBE instead, so it exposes unique_id. _bridge now reads a passthrough view dx_base__dc_snoop (= SELECT * FROM dc_snoop) for the base rows; src_name still filters dx_order_edges.src_table. Keeps the ~2.7x dc_snoop transfer win. Requires views dx_base__dc_snoop + pid/ppid in dx_src__kubescape_mitre (created on rig 6a881932; DDL saved for baking into AE schema). --- .../dx/evidence_graph/evidence_graph.pxl | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/src/pxl_scripts/dx/evidence_graph/evidence_graph.pxl b/src/pxl_scripts/dx/evidence_graph/evidence_graph.pxl index 2abdf3c19a0..fd656be1247 100644 --- a/src/pxl_scripts/dx/evidence_graph/evidence_graph.pxl +++ b/src/pxl_scripts/dx/evidence_graph/evidence_graph.pxl @@ -27,17 +27,20 @@ def _ord(start_time: str, clickhouse_dsn: str, view: str, order_id: str): return df.drop(['order_id', 'row_time', 'event_time']) -def _bridge(start_time: str, clickhouse_dsn: str, base_table: str, order_id: str): +def _bridge(start_time: str, clickhouse_dsn: str, src_name: str, base_view: str, order_id: str): # Fast path for high-fan-out tables (dc_snoop): the pre-joined dx_ord__ view # multiplies a base row by every order that consulted it (dc_snoop 6.3k -> 160k), # and the Pixie CH connector can't push the order_id filter down, so _ord pulls - # the full fanned-out view (~28MB). Instead join the SMALL base table to just + # the full fanned-out view (~28MB). Instead join the SMALL base rows to just # this order's narrow edge set -> ~10MB. Same rows, ~2.7x less transfer. + # base_view is a passthrough VIEW over the base table (px infers unique_id from + # a view via DESCRIBE; the registered base-table relation omits it). src_name is + # the base table name as stored in dx_order_edges.src_table. edges = px.DataFrame('dx_order_edges', clickhouse_dsn=clickhouse_dsn, start_time=start_time) edges = edges[edges.order_id == order_id] - edges = edges[edges.src_table == base_table] + edges = edges[edges.src_table == src_name] edges = edges[['unique_id']] - src = px.DataFrame(base_table, clickhouse_dsn=clickhouse_dsn, start_time=start_time) + src = px.DataFrame(base_view, clickhouse_dsn=clickhouse_dsn, start_time=start_time) j = src.merge(edges, how='inner', left_on=['unique_id'], right_on=['unique_id'], suffixes=['', '_e']) return j.drop(['unique_id', 'event_time']) @@ -112,7 +115,7 @@ def mysql(start_time: str, clickhouse_dsn: str, order_id: str): def dc_snoop(start_time: str, clickhouse_dsn: str, order_id: str): - return _bridge(start_time, clickhouse_dsn, 'dc_snoop', order_id) + return _bridge(start_time, clickhouse_dsn, 'dc_snoop', 'dx_base__dc_snoop', order_id) def stack_trace(start_time: str, clickhouse_dsn: str, order_id: str): From 5b6e0ee429e98dca4be198218ebda307a99feee2 Mon Sep 17 00:00:00 2001 From: entlein Date: Fri, 21 Aug 2026 20:41:40 +0200 Subject: [PATCH 62/66] dx/evidence_graph: order-centric graph, revert dc_snoop to _ord, native profiler for stacks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - GRAPH: rebuilt order-centric — start from dx_orders, INNER JOIN the anomaly detail on (kubescape_uid, rule_id); every edge is an order with a valid order_id + deep-link. Returns 31 edges (CH-verified). edgeHoverInfo uniqueID -> order_id. - dc_snoop: reverted to _ord; removed the _bridge helper + dropped the dx_base__dc_snoop passthrough view (per 'stop making views'). dc_snoop perf needs a data-model fix (bridge fan-out / connector pushdown), not a UI view. - STACKTRACE: forensic_db.stack_trace is empty (profiler->CH export not running). Read the native Pixie profiler stack_traces.beta instead (never empty); scope to the order's pod, differential over the order window via px.time_to_int64. NOTE: stack panels need UI verification — px CLI auth expired on the rig (refresh requested from makefile-agent); graph + all CH panels are verified. --- .../dx/evidence_graph/evidence_graph.pxl | 55 +++++++++---------- src/pxl_scripts/dx/evidence_graph/vis.json | 2 +- 2 files changed, 27 insertions(+), 30 deletions(-) diff --git a/src/pxl_scripts/dx/evidence_graph/evidence_graph.pxl b/src/pxl_scripts/dx/evidence_graph/evidence_graph.pxl index fd656be1247..f0740c7d9ce 100644 --- a/src/pxl_scripts/dx/evidence_graph/evidence_graph.pxl +++ b/src/pxl_scripts/dx/evidence_graph/evidence_graph.pxl @@ -27,30 +27,15 @@ def _ord(start_time: str, clickhouse_dsn: str, view: str, order_id: str): return df.drop(['order_id', 'row_time', 'event_time']) -def _bridge(start_time: str, clickhouse_dsn: str, src_name: str, base_view: str, order_id: str): - # Fast path for high-fan-out tables (dc_snoop): the pre-joined dx_ord__ view - # multiplies a base row by every order that consulted it (dc_snoop 6.3k -> 160k), - # and the Pixie CH connector can't push the order_id filter down, so _ord pulls - # the full fanned-out view (~28MB). Instead join the SMALL base rows to just - # this order's narrow edge set -> ~10MB. Same rows, ~2.7x less transfer. - # base_view is a passthrough VIEW over the base table (px infers unique_id from - # a view via DESCRIBE; the registered base-table relation omits it). src_name is - # the base table name as stored in dx_order_edges.src_table. - edges = px.DataFrame('dx_order_edges', clickhouse_dsn=clickhouse_dsn, start_time=start_time) - edges = edges[edges.order_id == order_id] - edges = edges[edges.src_table == src_name] - edges = edges[['unique_id']] - src = px.DataFrame(base_view, clickhouse_dsn=clickhouse_dsn, start_time=start_time) - j = src.merge(edges, how='inner', left_on=['unique_id'], right_on=['unique_id'], suffixes=['', '_e']) - return j.drop(['unique_id', 'event_time']) - - def evidence_graph(start_time: str, clickhouse_dsn: str, table: str): - df = px.DataFrame(table, clickhouse_dsn=clickhouse_dsn, start_time=start_time) + # Order-centric graph: start from dx_orders and INNER JOIN the anomaly detail on + # the order keys (kubescape_uid, rule_id), so every edge IS an order and carries a + # valid order_id + deep-link. subject_pod -> target, labelled ruleID + MITRE. orders = px.DataFrame('dx_orders', clickhouse_dsn=clickhouse_dsn, start_time=start_time) - orders = orders[['kubescape_uid', 'rule_id', 'order_id']] - df = df.merge(orders, how='left', left_on=['uniqueID', 'rule'], - right_on=['kubescape_uid', 'rule_id'], suffixes=['', '_ord']) + orders = orders[['order_id', 'kubescape_uid', 'rule_id', 'pod']] + anom = px.DataFrame(table, clickhouse_dsn=clickhouse_dsn, start_time=start_time) + df = orders.merge(anom, how='inner', left_on=['kubescape_uid', 'rule_id'], + right_on=['uniqueID', 'rule'], suffixes=['', '_a']) df.order_link = px.script_reference(df.order_id, 'dx/evidence_graph', { 'start_time': start_time, 'clickhouse_dsn': clickhouse_dsn, @@ -60,7 +45,7 @@ def evidence_graph(start_time: str, clickhouse_dsn: str, table: str): df.from_entity = px.Pod(df.subject_pod) df.to_entity = df.target return df[['from_entity', 'to_entity', 'order_link', 'rule_mitre', 'mitre_tactic', - 'mitre_technique', 'uniqueID', 'rule', 'process', 'target', 'target_kind', + 'mitre_technique', 'order_id', 'rule', 'process', 'target', 'target_kind', 'severity', 'alert', 'subject_pod']] @@ -115,22 +100,34 @@ def mysql(start_time: str, clickhouse_dsn: str, order_id: str): def dc_snoop(start_time: str, clickhouse_dsn: str, order_id: str): - return _bridge(start_time, clickhouse_dsn, 'dc_snoop', 'dx_base__dc_snoop', order_id) + return _ord(start_time, clickhouse_dsn, 'dx_ord__dc_snoop', order_id) def stack_trace(start_time: str, clickhouse_dsn: str, order_id: str): - return _ord(start_time, clickhouse_dsn, 'dx_ord__stack_trace', order_id) + # Native Pixie profiler (stack_traces.beta) is always populated, unlike the + # ClickHouse forensic_db.stack_trace which is only filled if the retention export + # runs. Scope to the order's pod (looked up from dx_orders). + w = px.DataFrame('dx_orders', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + w = w[w.order_id == order_id] + w = w[['pod']] + st = px.DataFrame(table='stack_traces.beta', start_time=start_time) + st.pod = st.ctx['pod'] + st = st.merge(w, how='inner', left_on=['pod'], right_on=['pod'], suffixes=['', '_w']) + st = st.groupby(['pod', 'stack_trace']).agg(count=('count', px.sum)) + return st[['pod', 'stack_trace', 'count']] def stack_diff(start_time: str, clickhouse_dsn: str, order_id: str): # Differential flame graph for ONE order's pod: stacks DURING the attack window - # [lo,hi] vs BASELINE stacks before the attack (row_time < lo). lo/hi come from - # dx_orders_win (event_time +/- 300s, Int64). delta = attack - baseline drives the - # StackTraceFlameGraph colouring (red = spiked during the attack). + # [lo,hi] vs BASELINE stacks before the attack. Reads the native profiler + # (stack_traces.beta) — never empty — and takes the order's pod + lo/hi from + # dx_orders_win. delta = attack - baseline drives the red/blue colouring. orders = px.DataFrame('dx_orders_win', clickhouse_dsn=clickhouse_dsn, start_time=start_time) orders = orders[orders.order_id == order_id] orders = orders[['pod', 'lo', 'hi']] - st = px.DataFrame('dx_src__stack_trace', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + st = px.DataFrame(table='stack_traces.beta', start_time=start_time) + st.pod = st.ctx['pod'] + st.row_time = px.time_to_int64(st.time_) st = st.merge(orders, how='inner', left_on=['pod'], right_on=['pod'], suffixes=['', '_o']) base = st[st.row_time < st.lo] diff --git a/src/pxl_scripts/dx/evidence_graph/vis.json b/src/pxl_scripts/dx/evidence_graph/vis.json index f3b883ca085..56e2fc9988b 100644 --- a/src/pxl_scripts/dx/evidence_graph/vis.json +++ b/src/pxl_scripts/dx/evidence_graph/vis.json @@ -20,7 +20,7 @@ {"outputName": "g_stackdiff", "func": {"name": "stack_diff", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "order_id", "variable": "order_id"}]}} ], "widgets": [ - {"name": "Evidence graph (subject pod -> target; edge = ruleID + MITRE technique; click for details + order link)", "position": {"x": 0, "y": 0, "w": 12, "h": 4}, "globalFuncOutputName": "g_graph", "displaySpec": {"@type": "types.px.dev/px.vispb.Graph", "adjacencyList": {"fromColumn": "from_entity", "toColumn": "to_entity"}, "edgeWeightColumn": "severity", "edgeColorColumn": "severity", "edgeLabelColumn": "rule_mitre", "edgeThresholds": {"mediumThreshold": 5, "highThreshold": 8}, "edgeHoverInfo": ["order_link", "rule", "mitre_tactic", "mitre_technique", "alert", "process", "target", "severity", "uniqueID"], "edgeLength": 500}}, + {"name": "Evidence graph (subject pod -> target; edge = ruleID + MITRE technique; click for details + order link)", "position": {"x": 0, "y": 0, "w": 12, "h": 4}, "globalFuncOutputName": "g_graph", "displaySpec": {"@type": "types.px.dev/px.vispb.Graph", "adjacencyList": {"fromColumn": "from_entity", "toColumn": "to_entity"}, "edgeWeightColumn": "severity", "edgeColorColumn": "severity", "edgeLabelColumn": "rule_mitre", "edgeThresholds": {"mediumThreshold": 5, "highThreshold": 8}, "edgeHoverInfo": ["order_link", "rule", "mitre_tactic", "mitre_technique", "alert", "process", "target", "severity", "order_id"], "edgeLength": 500}}, {"name": "ORDERS (click an order to filter all panels)", "position": {"x": 0, "y": 4, "w": 12, "h": 3}, "globalFuncOutputName": "g_orders", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, {"name": "kubescape_logs for order (MITRE tactic + technique)", "position": {"x": 0, "y": 7, "w": 12, "h": 4}, "globalFuncOutputName": "g_kube", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, {"name": "conn_stats", "position": {"x": 0, "y": 11, "w": 6, "h": 4}, "globalFuncOutputName": "g_conn", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, From 1c2052449781102a596139cb53030fa11b3d05dd Mon Sep 17 00:00:00 2001 From: entlein Date: Fri, 21 Aug 2026 21:26:16 +0200 Subject: [PATCH 63/66] deploy: pin dx 0.5.0-keepset-rc2 (http collected as pod/ns metadata) --- k8s/vizier/dx/dx-daemon.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/vizier/dx/dx-daemon.yaml b/k8s/vizier/dx/dx-daemon.yaml index 8b9f4a73791..7c6c96cf130 100644 --- a/k8s/vizier/dx/dx-daemon.yaml +++ b/k8s/vizier/dx/dx-daemon.yaml @@ -23,7 +23,7 @@ spec: # (WITHOUT -tiny — -tiny's pclntab stripping SIGSEGV'd under load). Passes the # obfuscation gate AND survives the kill-chain (restarts=0, 4 rounds). Carries the # evidence-manifest + DX_FOREST_PUSHDOWN code. - image: docker.io/entlein/dx-daemon:0.5.0-keepset-rc1 + image: docker.io/entlein/dx-daemon:0.5.0-keepset-rc2 ports: - { name: findings, containerPort: 9099, hostPort: 9099 } env: From 703ebef47f6230959295de604a408f6ee3bc44b6 Mon Sep 17 00:00:00 2001 From: entlein Date: Fri, 21 Aug 2026 22:05:50 +0200 Subject: [PATCH 64/66] deploy: pin dx 0.5.0-keepset-rc4 (dns hops + dc_snoop union + redis heartbeat filter) --- k8s/vizier/dx/dx-daemon.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/vizier/dx/dx-daemon.yaml b/k8s/vizier/dx/dx-daemon.yaml index 7c6c96cf130..2bbea0e30bb 100644 --- a/k8s/vizier/dx/dx-daemon.yaml +++ b/k8s/vizier/dx/dx-daemon.yaml @@ -23,7 +23,7 @@ spec: # (WITHOUT -tiny — -tiny's pclntab stripping SIGSEGV'd under load). Passes the # obfuscation gate AND survives the kill-chain (restarts=0, 4 rounds). Carries the # evidence-manifest + DX_FOREST_PUSHDOWN code. - image: docker.io/entlein/dx-daemon:0.5.0-keepset-rc2 + image: docker.io/entlein/dx-daemon:0.5.0-keepset-rc4 ports: - { name: findings, containerPort: 9099, hostPort: 9099 } env: From a736fc2ad6f59cc6a52ea3cf9a93581409c567e9 Mon Sep 17 00:00:00 2001 From: entlein Date: Fri, 21 Aug 2026 22:08:40 +0200 Subject: [PATCH 65/66] dx/evidence_graph: differential stack window +/-30s with matched baseline stack_diff window was event_time +/-300s (600s attack) with an unbounded ~6h baseline -> too wide + asymmetric. Now: ATTACK = [event_time-30s, event_time+30s] and a MATCHED 60s BASELINE immediately before it [event_time-90s, event_time-30s), computed as Int64 offsets from dx_orders_win.lo (no float division, so it compares against px.time_to_int64 row_time). delta = attack - baseline is now like-for-like. px-verified on a recent order (real redis stacks). NOTE: only populated for attacks within Pixie profiler retention (~1h); older attacks have no native profiler stacks. --- .../dx/evidence_graph/evidence_graph.pxl | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/src/pxl_scripts/dx/evidence_graph/evidence_graph.pxl b/src/pxl_scripts/dx/evidence_graph/evidence_graph.pxl index f0740c7d9ce..ff78300ce61 100644 --- a/src/pxl_scripts/dx/evidence_graph/evidence_graph.pxl +++ b/src/pxl_scripts/dx/evidence_graph/evidence_graph.pxl @@ -125,16 +125,24 @@ def stack_diff(start_time: str, clickhouse_dsn: str, order_id: str): orders = px.DataFrame('dx_orders_win', clickhouse_dsn=clickhouse_dsn, start_time=start_time) orders = orders[orders.order_id == order_id] orders = orders[['pod', 'lo', 'hi']] + # dx_orders_win.lo = event_time - 300s. Re-center to a tight +/-30s ATTACK + # window [event_time-30s, event_time+30s] and a MATCHED 60s BASELINE just + # before it [event_time-90s, event_time-30s), as Int64 offsets from lo (no + # division -> stays Int64 so it compares against row_time). + orders.alo = orders.lo + 270000000000 + orders.ahi = orders.lo + 330000000000 + orders.blo = orders.lo + 210000000000 st = px.DataFrame(table='stack_traces.beta', start_time=start_time) st.pod = st.ctx['pod'] st.row_time = px.time_to_int64(st.time_) st = st.merge(orders, how='inner', left_on=['pod'], right_on=['pod'], suffixes=['', '_o']) - base = st[st.row_time < st.lo] + base = st[st.row_time >= st.blo] + base = base[base.row_time < base.alo] base = base.groupby(['pod', 'stack_trace']).agg(count=('count', px.sum)) - atk = st[st.row_time >= st.lo] - atk = atk[atk.row_time <= atk.hi] + atk = st[st.row_time >= st.alo] + atk = atk[atk.row_time <= atk.ahi] atk = atk.groupby(['pod', 'stack_trace']).agg(count=('count', px.sum)) diff = base.merge(atk, how='right', left_on=['stack_trace'], right_on=['stack_trace'], From e12e8642d4402bc68e4f243f8314ecb76d9e249e Mon Sep 17 00:00:00 2001 From: entlein Date: Fri, 21 Aug 2026 22:28:10 +0200 Subject: [PATCH 66/66] deploy: pin dx 0.5.0-keepset-rc5 (dc_snoop union: comm + tree-pid + pod/ns-minus-stat) --- k8s/vizier/dx/dx-daemon.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/k8s/vizier/dx/dx-daemon.yaml b/k8s/vizier/dx/dx-daemon.yaml index 2bbea0e30bb..e7d5df897b3 100644 --- a/k8s/vizier/dx/dx-daemon.yaml +++ b/k8s/vizier/dx/dx-daemon.yaml @@ -23,7 +23,7 @@ spec: # (WITHOUT -tiny — -tiny's pclntab stripping SIGSEGV'd under load). Passes the # obfuscation gate AND survives the kill-chain (restarts=0, 4 rounds). Carries the # evidence-manifest + DX_FOREST_PUSHDOWN code. - image: docker.io/entlein/dx-daemon:0.5.0-keepset-rc4 + image: docker.io/entlein/dx-daemon:0.5.0-keepset-rc5 ports: - { name: findings, containerPort: 9099, hostPort: 9099 } env: