diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 4584d725f9b..c8bda4fbeac 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -456,7 +456,10 @@ REPOSITORY_LOCATIONS = dict( org_libc_musl = dict( sha256 = "7d5b0b6062521e4627e099e4c9dc8248d32a30285e959b7eecaa780cf8cfd4a4", strip_prefix = "musl-1.2.3", - urls = ["http://musl.libc.org/releases/musl-1.2.3.tar.gz"], + urls = [ + "https://sources.openwrt.org/musl-1.2.3.tar.gz", + "http://musl.libc.org/releases/musl-1.2.3.tar.gz", + ], manual_license_name = "libc/musl", ), rules_cc = dict( diff --git a/ci/artifact_utils.sh b/ci/artifact_utils.sh index a1eec1a7760..e6d7c0dca26 100644 --- a/ci/artifact_utils.sh +++ b/ci/artifact_utils.sh @@ -107,7 +107,15 @@ create_manifest_update() { tag_name="release/${component}/v${version}" # actions/checkout doesn't get the tag annotation properly. git fetch origin tag "${tag_name}" -f - timestamp="$(git tag -l --format "%(taggerdate:raw)" "${tag_name}" | awk '{print $1}' | jq '. | todate')" + # taggerdate is empty for a LIGHTWEIGHT tag → produces `timestamp: ,` → jq syntax + # error → release-metadata step fails even though the image built fine. Fall back to + # the tagged commit's committer date so the manifest is well-formed regardless of how + # the release tag was cut (annotated vs lightweight). + raw_ts="$(git tag -l --format "%(taggerdate:raw)" "${tag_name}" | awk '{print $1}')" + if [ -z "${raw_ts}" ]; then + raw_ts="$(git log -1 --format="%ct" "${tag_name}")" + fi + timestamp="$(printf '%s' "${raw_ts}" | jq '. | todate')" jq -s \ "[{name: \"${component}\", artifact: [{timestamp: ${timestamp}, commitHash: \"${commit_hash}\", versionStr: \"${version}\", availableArtifactMirrors: .}]}]" \ diff --git a/k8s/vizier/adaptive_export/kustomization.yaml b/k8s/vizier/adaptive_export/kustomization.yaml new file mode 100644 index 00000000000..35c0987d520 --- /dev/null +++ b/k8s/vizier/adaptive_export/kustomization.yaml @@ -0,0 +1,10 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: pl +resources: + - ../bootstrap/adaptive_export_role.yaml + - ../bootstrap/adaptive_export_deployment.yaml +images: + - name: vizier-adaptive_export_image + newName: ghcr.io/k8sstormcenter/vizier-adaptive_export_image + newTag: 0.14.19-aeprod76 diff --git a/k8s/vizier/bootstrap/adaptive_export_deployment.yaml b/k8s/vizier/bootstrap/adaptive_export_deployment.yaml index 2db195ff408..251ceed69ce 100644 --- a/k8s/vizier/bootstrap/adaptive_export_deployment.yaml +++ b/k8s/vizier/bootstrap/adaptive_export_deployment.yaml @@ -1,115 +1,97 @@ --- +# adaptive-export: node-local forensic capture operator. DaemonSet so each pod +# queries its own node's vizier-pem (pem-direct). Secret seeded per-cluster. apiVersion: apps/v1 -kind: Deployment +kind: DaemonSet metadata: name: adaptive-export + labels: { name: adaptive-export, plane: control } spec: - replicas: 0 selector: - matchLabels: - name: adaptive-export + matchLabels: { name: adaptive-export } template: metadata: - labels: - name: adaptive-export - plane: control + labels: { name: adaptive-export, plane: control } spec: affinity: nodeAffinity: requiredDuringSchedulingIgnoredDuringExecution: nodeSelectorTerms: - # The beta.kubernetes.io/os label has been deprecated since - # k8s v1.14; every modern kubelet sets kubernetes.io/os. The - # single term below is enough — kept both ORed terms in the - # past for pre-1.14 compatibility. - matchExpressions: - - key: kubernetes.io/os - operator: In - values: - - linux + - { key: kubernetes.io/os, operator: In, values: [linux] } serviceAccountName: pl-adaptive-export-service-account containers: - name: adaptive-export image: vizier-adaptive_export_image:latest - # Bounded so AE can never memory-pressure a node (measured: AE uses - # only ~16-38Mi steady; passthrough with the raised 1M-row cap can - # spike, so 1Gi caps the worst case). CPU was pinned at the old 300m - # limit under concurrent passthrough → raised to 1 core. + ports: + - { name: control, containerPort: 9100, hostPort: 9100 } resources: - requests: - cpu: 200m - memory: 128Mi - limits: - cpu: "1" - memory: 1Gi + requests: { cpu: 100m, memory: 128Mi } + limits: { cpu: "1", memory: 1Gi } env: + - name: HOST_IP + valueFrom: { fieldRef: { fieldPath: status.hostIP } } + - name: ADAPTIVE_VIZIER_DIRECT_ADDR + value: "$(HOST_IP):50305" + - name: PL_JWT_SIGNING_KEY + valueFrom: { secretKeyRef: { name: pl-cluster-secrets, key: jwt-signing-key } } + - name: PX_DISABLE_TLS + value: "1" - name: PL_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace + valueFrom: { fieldRef: { fieldPath: metadata.namespace } } + - name: NODE_NAME + valueFrom: { fieldRef: { fieldPath: spec.nodeName } } - name: PIXIE_API_KEY - valueFrom: - secretKeyRef: - name: pl-adaptive-export-secrets - key: pixie-api-key + valueFrom: { secretKeyRef: { name: pl-adaptive-export-secrets, key: pixie-api-key } } - name: CLICKHOUSE_DSN - valueFrom: - secretKeyRef: - name: pl-adaptive-export-secrets - key: clickhouse-dsn - - name: VERBOSE - value: "true" - - name: DETECTION_INTERVAL_SEC - value: "10" - - name: DETECTION_LOOKBACK_SEC - value: "30" - # EXPORT_MODE controls the reconcile behaviour: - # auto - detection drives on/off (default) - # always - plugin always enabled (bypass detection) - # never - plugin always disabled and ch-* scripts purged - - name: EXPORT_MODE - value: "auto" - # Number of consecutive empty detection ticks before auto-disable fires. - - name: EXPORT_QUIET_TICKS - value: "6" - # Optional overrides for the ClickHouse PxL scripts. When unset they are - # parsed from CLICKHOUSE_DSN. Individual fields win over the parsed DSN. - # Defaults below match soc/tree/clickhouse-lab (forensic-soc-db CHI, - # ingest_writer user, forensic_db database). + valueFrom: { secretKeyRef: { name: pl-adaptive-export-secrets, key: clickhouse-dsn } } - name: KUBESCAPE_TABLE value: "kubescape_logs" - # - name: CLICKHOUSE_HOST - # value: "clickhouse-forensic-soc-db.clickhouse.svc.cluster.local" - # - name: CLICKHOUSE_PORT - # value: "9000" - # - name: CLICKHOUSE_USER - # value: "ingest_writer" - # - name: CLICKHOUSE_PASSWORD - # value: "changeme-ingest" - # - name: CLICKHOUSE_DATABASE - # value: "forensic_db" - # TLS for the control surface (CONTROL_TLS=true). server.crt/key from the - # same service-tls-certs secret the broker/PEM use; without this the dx - # bearer JWT crosses the CNI in cleartext. Harmless when control is off. + - name: EXPORT_MODE + value: "never" + # Control surface is secure-by-default (#96): TLS + bearer-JWT auth are ON + # out of the box. The service-tls-certs keypair mounted at /certs below is + # used for TLS (else AE self-generates an ephemeral in-memory cert), and + # PL_JWT_SIGNING_KEY above turns on auth. CONTROL_TLS / CONTROL_REQUIRE_AUTH + # are deprecated no-ops; set CONTROL_INSECURE=true only to opt out (dev). + - name: CONTROL_ADDR + value: ":9100" + - name: ADAPTIVE_PUSH_PIXIE_ROWS + value: "true" + - name: ADAPTIVE_RECONCILE + value: "true" + - name: DEPLOY_TRACEPOINTS + value: "true" + - name: INSTALL_PRESET_SCRIPTS + value: "false" + - name: ADAPTIVE_MAX_INFLIGHT_QUERIES_GLOBAL + value: "4" + - name: ADAPTIVE_ORDER_CHUNK_SEC + value: "600" + - name: VERBOSE + value: "true" volumeMounts: - - name: certs - mountPath: /certs - readOnly: true + - { name: certs, mountPath: /certs, readOnly: true } securityContext: allowPrivilegeEscalation: false - capabilities: - drop: - - ALL - seccompProfile: - type: RuntimeDefault + capabilities: { drop: [ALL] } + seccompProfile: { type: RuntimeDefault } volumes: - name: certs - secret: - secretName: service-tls-certs + secret: { secretName: service-tls-certs } securityContext: runAsUser: 10100 runAsGroup: 10100 fsGroup: 10100 runAsNonRoot: true - seccompProfile: - type: RuntimeDefault + seccompProfile: { type: RuntimeDefault } +--- +apiVersion: v1 +kind: Service +metadata: + name: adaptive-export-control +spec: + selector: { name: adaptive-export } + internalTrafficPolicy: Local # dx reaches its co-located (same-node) AE + ports: + - { name: control, port: 9100, targetPort: 9100 } diff --git a/k8s/vizier/dx/dx-daemon.yaml b/k8s/vizier/dx/dx-daemon.yaml new file mode 100644 index 00000000000..e7d5df897b3 --- /dev/null +++ b/k8s/vizier/dx/dx-daemon.yaml @@ -0,0 +1,79 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: { name: dx-daemon, namespace: honey } +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: dx-daemon + namespace: honey + labels: { app: dx-daemon } +spec: + selector: { matchLabels: { app: dx-daemon } } + template: + metadata: + labels: { app: dx-daemon } + spec: + serviceAccountName: dx-daemon + tolerations: [{ operator: Exists }] + terminationGracePeriodSeconds: 35 + containers: + - name: dx-daemon + # OBFUSCATED release rc13 (entlein/dx#138 fault 2 RESOLVED): garble -literals + # (WITHOUT -tiny — -tiny's pclntab stripping SIGSEGV'd under load). Passes the + # obfuscation gate AND survives the kill-chain (restarts=0, 4 rounds). Carries the + # evidence-manifest + DX_FOREST_PUSHDOWN code. + image: docker.io/entlein/dx-daemon:0.5.0-keepset-rc5 + ports: + - { name: findings, containerPort: 9099, hostPort: 9099 } + env: + - { name: NODE_NAME, valueFrom: { fieldRef: { fieldPath: spec.nodeName } } } + - { name: HOST_IP, valueFrom: { fieldRef: { fieldPath: status.hostIP } } } + - { name: DX_RECEIVER_TLS, value: "1" } + # AE control surface is TLS-by-default (#96); the dx client TLS-skip-verifies + # the in-cluster (self-signed/shared) cert and attaches its bearer JWT. + - { name: AE_CONTROL_ADDR, value: "https://adaptive-export-control.pl.svc.cluster.local:9100" } + - { name: PX_API_KEY, valueFrom: { secretKeyRef: { name: dx-pixie-auth, key: api-key, optional: true } } } + - { name: PX_CLUSTER_ID, valueFrom: { secretKeyRef: { name: dx-pixie-auth, key: cluster-id, optional: true } } } + - { name: PX_CLOUD_ADDR, valueFrom: { secretKeyRef: { name: dx-pixie-auth, key: cloud-addr, optional: true } } } + - { name: DX_BENCH, value: "pemdirect" } + - { name: PL_JWT_SIGNING_KEY, valueFrom: { secretKeyRef: { name: dx-vizier-direct, key: jwt-signing-key, optional: true } } } + - { name: DX_VIZIER_DIRECT_ADDR, value: "vizier-query-broker-svc.pl.svc.cluster.local:50300" } + - { name: PX_DISABLE_TLS, value: "1" } + - { name: DX_CLUSTER_MALIGNANT_HTTP, valueFrom: { secretKeyRef: { name: dx-metastasis-ch, key: http-url, optional: true } } } + - { name: DX_PX_TIMEOUT_S, value: "90" } + - { name: DX_TELEMETRY_CACHE, value: "1" } + - { name: DX_WORKERS, value: "4" } + # evidence-graph: forest-scope the evidence, write the per-anomaly edge set, + # sink it straight to forensic_db.dx_evidence_graph (soc ingest_writer). + - { name: DX_FOREST_SCOPE, value: "1" } + # FOREST_PUSHDOWN (entlein/dx#138 fault 3): push the dc_snoop ppid-lineage filter + # INTO the PxL so dx pulls only the alert pod's subtree, not the whole node — + # frees the node-local PEM so AE can export dc_snoop under load (validated: 0→1777). + - { name: DX_FOREST_PUSHDOWN, value: "1" } + - { name: DX_FOREST_PUSHDOWN_DEPTH, value: "4" } + - { name: DX_PRECORRELATE_GRAPH, value: "1" } + - { name: DX_EVIDENCE_GRAPH_CH, value: "http://ingest_writer:changeme-ingest@clickhouse-forensic-soc-db.clickhouse.svc.cluster.local:8123/forensic_db" } + readinessProbe: + httpGet: { path: /healthz, port: 9099, scheme: HTTPS } + initialDelaySeconds: 3 + periodSeconds: 10 + resources: + # memory: the precorrelate/full-evidence workup (DX_PRECORRELATE_GRAPH) + pemdirect + # gRPC result streams pull the per-anomaly evidence set into memory; at 1Gi dx is + # OOM-killed mid-workup (exit 137) BEFORE it writes the graph → crash-loop, empty + # graph. Measured peak ~1.3GB/round under the redis kill-chain (entlein/dx#138 + # fault 1); 3Gi clears it reliably on an 8GiB node (validated: restarts=0 over 6+ rounds). + requests: { cpu: 50m, memory: 1Gi } + limits: { cpu: "2", memory: 3Gi } +--- +apiVersion: v1 +kind: Service +metadata: + name: dx-daemon + namespace: honey +spec: + selector: { app: dx-daemon } + internalTrafficPolicy: Local + ports: + - { name: findings, port: 9099, targetPort: 9099 } diff --git a/k8s/vizier/dx/kustomization.yaml b/k8s/vizier/dx/kustomization.yaml new file mode 100644 index 00000000000..7e7edebde60 --- /dev/null +++ b/k8s/vizier/dx/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: honey +resources: + - dx-daemon.yaml diff --git a/skaffold/skaffold_adaptive_export.yaml b/skaffold/skaffold_adaptive_export.yaml new file mode 100644 index 00000000000..b04d8550990 --- /dev/null +++ b/skaffold/skaffold_adaptive_export.yaml @@ -0,0 +1,44 @@ +--- +# Deploy-only Skaffold for the adaptive_export DaemonSet using a prebuilt image +# (lab / review), overlaying an already-running vizier. Run from the repo root: +# skaffold deploy -f skaffold/skaffold_adaptive_export.yaml +# Bump the image via newTag in k8s/vizier/adaptive_export/kustomization.yaml. +apiVersion: skaffold/v4beta11 +kind: Config +metadata: + name: adaptive-export +manifests: + kustomize: + paths: + - k8s/vizier/adaptive_export + buildArgs: + - --load-restrictor=LoadRestrictionsNone +deploy: + kubectl: + defaultNamespace: pl + hooks: + before: + - host: + command: + - bash + - -c + - | + set -e + # PL_CLOUD_ADDR must carry an explicit :443 or the AE cloud client crashloops. + CA=$(kubectl -n pl get cm pl-cloud-config -o jsonpath='{.data.PL_CLOUD_ADDR}' 2>/dev/null || true) + case "$CA" in ""|*:*) ;; *) kubectl -n pl patch cm pl-cloud-config --type merge -p "{\"data\":{\"PL_CLOUD_ADDR\":\"$CA:443\"}}";; esac + # Seed pl-adaptive-export-secrets ONLY when a key is supplied; never clobber a good secret with an empty one. + API="${PIXIE_API_KEY:-${PX_API_KEY:-}}" + CH_DSN="${AE_CH_DSN:-ingest_writer:changeme-ingest@clickhouse-forensic-soc-db.clickhouse.svc.cluster.local:9000/forensic_db}" + if [ -n "$API" ]; then + kubectl -n pl create secret generic pl-adaptive-export-secrets \ + --from-literal=pixie-api-key="$API" \ + --from-literal=clickhouse-dsn="$CH_DSN" \ + --dry-run=client -o yaml | kubectl apply -f - + elif ! kubectl -n pl get secret pl-adaptive-export-secrets >/dev/null 2>&1; then + echo "ERROR: set PIXIE_API_KEY (or source keys.env) to seed pl-adaptive-export-secrets" >&2 + exit 1 + else + echo "pl-adaptive-export-secrets exists; PIXIE_API_KEY unset -> leaving it untouched" + fi + os: [linux, darwin] diff --git a/skaffold/skaffold_dx.yaml b/skaffold/skaffold_dx.yaml new file mode 100644 index 00000000000..ea3ee0c7e14 --- /dev/null +++ b/skaffold/skaffold_dx.yaml @@ -0,0 +1,42 @@ +--- +# Deploy-only Skaffold for the dx-daemon DaemonSet (prebuilt image), overlaying an +# already-running vizier + soc stack. Run from the repo root, AFTER adaptive_export +# (the hook mirrors pl-adaptive-export-secrets into honey): +# skaffold deploy -f skaffold/skaffold_dx.yaml +apiVersion: skaffold/v4beta11 +kind: Config +metadata: + name: dx-daemon +manifests: + kustomize: + paths: + - k8s/vizier/dx +deploy: + kubectl: + defaultNamespace: honey + hooks: + before: + - host: + command: + - bash + - -c + - | + set -e + kubectl create namespace honey --dry-run=client -o yaml | kubectl apply -f - + JWT=$(kubectl -n pl get secret pl-cluster-secrets -o jsonpath='{.data.jwt-signing-key}' | base64 -d) + CID=$(kubectl -n pl get secret pl-cluster-secrets -o jsonpath='{.data.cluster-id}' | base64 -d) + CA=$(kubectl -n pl get cm pl-cloud-config -o jsonpath='{.data.PL_CLOUD_ADDR}') + API=$(kubectl -n pl get secret pl-adaptive-export-secrets -o jsonpath='{.data.pixie-api-key}' 2>/dev/null | base64 -d) + CH_URL="${DX_CH_HTTP_URL:-http://ingest_writer:changeme-ingest@clickhouse-forensic-soc-db.clickhouse.svc.cluster.local:8123/?database=forensic_db}" + kubectl -n honey create secret generic dx-vizier-direct \ + --from-literal=jwt-signing-key="$JWT" \ + --dry-run=client -o yaml | kubectl apply -f - + kubectl -n honey create secret generic dx-pixie-auth \ + --from-literal=api-key="$API" \ + --from-literal=cluster-id="$CID" \ + --from-literal=cloud-addr="$CA" \ + --dry-run=client -o yaml | kubectl apply -f - + kubectl -n honey create secret generic dx-metastasis-ch \ + --from-literal=http-url="$CH_URL" \ + --dry-run=client -o yaml | kubectl apply -f - + os: [linux, darwin] diff --git a/src/pxl_scripts/Makefile b/src/pxl_scripts/Makefile index 1cca03f4dc5..4e8a3562658 100644 --- a/src/pxl_scripts/Makefile +++ b/src/pxl_scripts/Makefile @@ -15,7 +15,7 @@ # SPDX-License-Identifier: Apache-2.0 # Update dir name here if you want to add a new directory. -dirs := bpftrace px pxbeta sotw +dirs := bpftrace dx px pxbeta sotw script_files := $(foreach dir,$(dirs),$(wildcard $(dir)/**/*)) EXECUTABLES ?= px diff --git a/src/pxl_scripts/dx/evidence_graph/evidence_graph.pxl b/src/pxl_scripts/dx/evidence_graph/evidence_graph.pxl new file mode 100644 index 00000000000..b7c96c70e53 --- /dev/null +++ b/src/pxl_scripts/dx/evidence_graph/evidence_graph.pxl @@ -0,0 +1,160 @@ +# Copyright 2018- The Pixie Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# SPDX-License-Identifier: Apache-2.0 +# +# SOC pixie evidence graph — works only with ClickHouse enabled. + +import px + + +def _ord(start_time: str, clickhouse_dsn: str, view: str, order_id: str): + df = px.DataFrame(view, clickhouse_dsn=clickhouse_dsn, start_time=start_time) + df = df[df.order_id == order_id] + return df.drop(['order_id', 'row_time', 'event_time']) + + +def _bridge(start_time: str, clickhouse_dsn: str, src_name: str, base_view: str, order_id: str): + # Fast path for the high-fan-out table (dc_snoop): the pre-joined dx_ord__ view + # multiplies a base row by every order; px pulls it whole (no filter push-down). + # Join the small base rows to just this order's narrow edge set instead (~2x). + # base_view is a passthrough VIEW so px infers unique_id (the registered base + # relation omits it). + edges = px.DataFrame('dx_order_edges', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + edges = edges[edges.order_id == order_id] + edges = edges[edges.src_table == src_name] + edges = edges[['unique_id']] + src = px.DataFrame(base_view, clickhouse_dsn=clickhouse_dsn, start_time=start_time) + j = src.merge(edges, how='inner', left_on=['unique_id'], right_on=['unique_id'], suffixes=['', '_e']) + return j.drop(['unique_id', 'event_time']) + + +def evidence_graph(start_time: str, clickhouse_dsn: str, table: str): + orders = px.DataFrame('dx_orders', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + orders = orders[['order_id', 'kubescape_uid', 'rule_id', 'pod']] + anom = px.DataFrame(table, clickhouse_dsn=clickhouse_dsn, start_time=start_time) + df = orders.merge(anom, how='inner', left_on=['kubescape_uid', 'rule_id'], + right_on=['uniqueID', 'rule'], suffixes=['', '_a']) + df.order_link = px.script_reference(df.order_id, 'dx/evidence_graph', { + 'start_time': start_time, + 'clickhouse_dsn': clickhouse_dsn, + 'graph_table': table, + 'order_id': df.order_id, + }) + df.from_entity = px.Pod(df.subject_pod) + df.to_entity = df.target + return df[['from_entity', 'to_entity', 'order_link', 'rule_mitre', 'mitre_tactic', + 'mitre_technique', 'order_id', 'rule', 'process', 'target', 'target_kind', + 'severity', 'alert', 'subject_pod']] + + +def orders(start_time: str, clickhouse_dsn: str, graph_table: str): + df = px.DataFrame('dx_orders', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + alerts = px.DataFrame(graph_table, clickhouse_dsn=clickhouse_dsn, start_time=start_time) + alerts = alerts[['uniqueID', 'rule', 'mitre_tactic', 'mitre_technique']] + df = df.merge(alerts, how='left', left_on=['kubescape_uid', 'rule_id'], + right_on=['uniqueID', 'rule'], suffixes=['', '_k']) + df.order = px.script_reference(df.order_id, 'dx/evidence_graph', { + 'start_time': start_time, + 'clickhouse_dsn': clickhouse_dsn, + 'graph_table': graph_table, + 'order_id': df.order_id, + }) + df.Alert = df.disc + return df[['order', 'rule_id', 'Alert', 'mitre_tactic', 'mitre_technique', 'pod']] + + +def kubescape(start_time: str, clickhouse_dsn: str, order_id: str): + orders = px.DataFrame('dx_orders', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + orders = orders[orders.order_id == order_id] + orders = orders[['kubescape_uid', 'rule_id']] + k = px.DataFrame('dx_src__kubescape_mitre', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + j = k.merge(orders, how='inner', left_on=['uniqueID', 'RuleID'], + right_on=['kubescape_uid', 'rule_id'], suffixes=['', '_ord']) + return j.drop(['row_time', 'event_time', 'kubescape_uid', 'rule_id', 'uniqueID']) + + +def conn(start_time: str, clickhouse_dsn: str, order_id: str): + return _ord(start_time, clickhouse_dsn, 'dx_ord__conn_stats', order_id) + + +def redis(start_time: str, clickhouse_dsn: str, order_id: str): + return _ord(start_time, clickhouse_dsn, 'dx_ord__redis_events', order_id) + + +def http(start_time: str, clickhouse_dsn: str, order_id: str): + return _ord(start_time, clickhouse_dsn, 'dx_ord__http_events', order_id) + + +def dns(start_time: str, clickhouse_dsn: str, order_id: str): + return _ord(start_time, clickhouse_dsn, 'dx_ord__dns_events', order_id) + + +def pgsql(start_time: str, clickhouse_dsn: str, order_id: str): + return _ord(start_time, clickhouse_dsn, 'dx_ord__pgsql_events', order_id) + + +def mysql(start_time: str, clickhouse_dsn: str, order_id: str): + return _ord(start_time, clickhouse_dsn, 'dx_ord__mysql_events', order_id) + + +def dc_snoop(start_time: str, clickhouse_dsn: str, order_id: str): + return _bridge(start_time, clickhouse_dsn, 'dc_snoop', 'dx_base__dc_snoop', order_id) + + +def stack_trace(start_time: str, clickhouse_dsn: str, order_id: str): + # Native profiler (never empty); the ClickHouse stack_trace export is not running. + w = px.DataFrame('dx_orders', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + w = w[w.order_id == order_id] + w = w[['pod']] + st = px.DataFrame(table='stack_traces.beta', start_time=start_time) + st.pod = st.ctx['pod'] + st = st.merge(w, how='inner', left_on=['pod'], right_on=['pod'], suffixes=['', '_w']) + st = st.groupby(['pod', 'stack_trace']).agg(count=('count', px.sum)) + return st[['pod', 'stack_trace', 'count']] + + +def stack_diff(start_time: str, clickhouse_dsn: str, order_id: str): + # ATTACK [event_time-30s, +30s] vs MATCHED 60s BASELINE before it; Int64 offsets + # from dx_orders_win.lo (no float division -> compares against time_to_int64). + orders = px.DataFrame('dx_orders_win', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + orders = orders[orders.order_id == order_id] + orders = orders[['pod', 'lo', 'hi']] + orders.alo = orders.lo + 270000000000 + orders.ahi = orders.lo + 330000000000 + orders.blo = orders.lo + 210000000000 + st = px.DataFrame(table='stack_traces.beta', start_time=start_time) + st.pod = st.ctx['pod'] + st.row_time = px.time_to_int64(st.time_) + st = st.merge(orders, how='inner', left_on=['pod'], right_on=['pod'], suffixes=['', '_o']) + + base = st[st.row_time >= st.blo] + base = base[base.row_time < base.alo] + base = base.groupby(['pod', 'stack_trace']).agg(count=('count', px.sum)) + + atk = st[st.row_time >= st.alo] + atk = atk[atk.row_time <= atk.ahi] + atk = atk.groupby(['pod', 'stack_trace']).agg(count=('count', px.sum)) + + diff = base.merge(atk, how='right', left_on=['stack_trace'], right_on=['stack_trace'], + suffixes=['_base', '_atk']) + diff.pod = diff.pod_atk + diff.stack_trace = px.replace(' ', diff.stack_trace_atk, '') + diff.count = diff.count_atk + diff.delta = diff.count_atk - diff.count_base + + total = atk.groupby(['pod']).agg(total=('count', px.sum)) + merged = diff.merge(total, how='inner', left_on=['pod'], right_on=['pod'], suffixes=['', '_t']) + merged.percent = 100 * merged.count / merged.total + return merged[['stack_trace', 'count', 'delta', 'percent', 'pod']] diff --git a/src/pxl_scripts/dx/evidence_graph/manifest.yaml b/src/pxl_scripts/dx/evidence_graph/manifest.yaml new file mode 100644 index 00000000000..59572ec64e3 --- /dev/null +++ b/src/pxl_scripts/dx/evidence_graph/manifest.yaml @@ -0,0 +1,4 @@ +--- +short: SOC Evidence Graph +long: > + SOC pixie, works only with clickhouse enabled. diff --git a/src/pxl_scripts/dx/evidence_graph/vis.json b/src/pxl_scripts/dx/evidence_graph/vis.json new file mode 100644 index 00000000000..2c6da3db83e --- /dev/null +++ b/src/pxl_scripts/dx/evidence_graph/vis.json @@ -0,0 +1,36 @@ +{ + "variables": [ + {"name": "start_time", "type": "PX_STRING", "description": "Window start.", "defaultValue": "-6h"}, + {"name": "clickhouse_dsn", "type": "PX_STRING", "description": "forensic_db DSN: user:pass@host:port/db.", "defaultValue": "forensic_analyst:changeme-analyst@clickhouse-forensic-soc-db.clickhouse.svc.cluster.local:9000/forensic_db"}, + {"name": "graph_table", "type": "PX_STRING", "description": "L1 kill-chain graph source (MITRE-enriched).", "defaultValue": "dx_kubescape_mitre"}, + {"name": "order_id", "type": "PX_STRING", "description": "Set by clicking an order link in the graph popup or the ORDERS panel; every panel snaps to that order.", "defaultValue": ""} + ], + "globalFuncs": [ + {"outputName": "g_graph", "func": {"name": "evidence_graph", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "table", "variable": "graph_table"}]}}, + {"outputName": "g_orders", "func": {"name": "orders", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "graph_table", "variable": "graph_table"}]}}, + {"outputName": "g_kube", "func": {"name": "kubescape", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "order_id", "variable": "order_id"}]}}, + {"outputName": "g_conn", "func": {"name": "conn", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "order_id", "variable": "order_id"}]}}, + {"outputName": "g_redis", "func": {"name": "redis", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "order_id", "variable": "order_id"}]}}, + {"outputName": "g_http", "func": {"name": "http", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "order_id", "variable": "order_id"}]}}, + {"outputName": "g_dns", "func": {"name": "dns", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "order_id", "variable": "order_id"}]}}, + {"outputName": "g_pgsql", "func": {"name": "pgsql", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "order_id", "variable": "order_id"}]}}, + {"outputName": "g_mysql", "func": {"name": "mysql", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "order_id", "variable": "order_id"}]}}, + {"outputName": "g_dcsnoop", "func": {"name": "dc_snoop", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "order_id", "variable": "order_id"}]}}, + {"outputName": "g_stack", "func": {"name": "stack_trace", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "order_id", "variable": "order_id"}]}}, + {"outputName": "g_stackdiff", "func": {"name": "stack_diff", "args": [{"name": "start_time", "variable": "start_time"}, {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, {"name": "order_id", "variable": "order_id"}]}} + ], + "widgets": [ + {"name": "Evidence graph (subject pod -> target; edge = ruleID + MITRE technique; click for details + order link)", "position": {"x": 0, "y": 0, "w": 12, "h": 4}, "globalFuncOutputName": "g_graph", "displaySpec": {"@type": "types.px.dev/px.vispb.Graph", "adjacencyList": {"fromColumn": "from_entity", "toColumn": "to_entity"}, "edgeWeightColumn": "severity", "edgeColorColumn": "severity", "edgeLabelColumn": "rule_mitre", "edgeThresholds": {"mediumThreshold": 5, "highThreshold": 8}, "edgeHoverInfo": ["order_link", "rule", "mitre_tactic", "mitre_technique", "alert", "process", "target", "severity", "order_id"], "edgeLength": 500}}, + {"name": "ORDERS (click an order to filter all panels)", "position": {"x": 0, "y": 4, "w": 12, "h": 3}, "globalFuncOutputName": "g_orders", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, + {"name": "kubescape_logs for order (MITRE tactic + technique)", "position": {"x": 0, "y": 7, "w": 12, "h": 4}, "globalFuncOutputName": "g_kube", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, + {"name": "conn_stats", "position": {"x": 0, "y": 11, "w": 6, "h": 4}, "globalFuncOutputName": "g_conn", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, + {"name": "redis_events", "position": {"x": 6, "y": 11, "w": 6, "h": 4}, "globalFuncOutputName": "g_redis", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, + {"name": "http_events", "position": {"x": 0, "y": 15, "w": 6, "h": 4}, "globalFuncOutputName": "g_http", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, + {"name": "dns_events", "position": {"x": 6, "y": 15, "w": 6, "h": 4}, "globalFuncOutputName": "g_dns", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, + {"name": "pgsql_events", "position": {"x": 0, "y": 19, "w": 6, "h": 4}, "globalFuncOutputName": "g_pgsql", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, + {"name": "mysql_events", "position": {"x": 6, "y": 19, "w": 6, "h": 4}, "globalFuncOutputName": "g_mysql", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, + {"name": "dc_snoop (file access)", "position": {"x": 0, "y": 23, "w": 6, "h": 4}, "globalFuncOutputName": "g_dcsnoop", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, + {"name": "stack_trace (profiler)", "position": {"x": 6, "y": 23, "w": 6, "h": 4}, "globalFuncOutputName": "g_stack", "displaySpec": {"@type": "types.px.dev/px.vispb.Table"}}, + {"name": "Differential stack trace (attack window vs baseline; red = spiked during attack)", "position": {"x": 0, "y": 27, "w": 12, "h": 4}, "globalFuncOutputName": "g_stackdiff", "displaySpec": {"@type": "types.px.dev/px.vispb.StackTraceFlameGraph", "stacktraceColumn": "stack_trace", "countColumn": "count", "percentageColumn": "percent", "podColumn": "pod", "differenceColumn": "delta"}} + ] +} diff --git a/src/pxl_scripts/px/dx_evidence_graph/README.md b/src/pxl_scripts/px/dx_evidence_graph/README.md new file mode 100644 index 00000000000..ea27bccd455 --- /dev/null +++ b/src/pxl_scripts/px/dx_evidence_graph/README.md @@ -0,0 +1,107 @@ +# DX Evidence Graph — 3-level zoom (`px/dx_evidence_graph`) + +A **standalone** Pixie Live View bundle (PxL + `vis.json`, no Pixie UI source +changes) that renders the dx evidence graph in Pixie's existing `GraphWidget` and +lets an analyst zoom from an investigation down to the individual forensic rows dx +consulted. + +## What it shows — the three levels + +| Level | Widget | PxL func | Content | +|------|--------|----------|---------| +| 1 | Graph | `evidence_graph` | Severity-weighted, all-protocol **pod → pod** edge list for the malignant (ruled-in) investigations. Edge weight = `confidence`, colour = `max_severity`, label = `edge_kind`, hover = investigation_id / condition / criteria / num_findings. | +| 2 | Table | `investigation_detail` | The **manifest** row(s) for the zoomed investigation: `verdict`, `condition`, `confidence`/`posterior`, case-window bounds (`win_lo`/`win_hi`, plucked from the `case_window` JSON), `evidence_hash`, raw `findings`. | +| 3 | Table | `consulted_rows` | The **§H reconstruction**: the raw `dc_snoop` (default) process rows for the alert pod in the window — the individual rows dx consulted. Repoint `raw_table` at `redis_events` / `kubescape_logs` for the other §H tables. | + +## The drill-down model + +- **Pod-node double-click → `px/pod` (built-in, no code change).** `evidence_graph` + stamps the `from_entity`/`to_entity` node columns with the pod semantic type via + `px.Pod(...)` (registered as a `STRING → ST_POD_NAME` cast in + `src/carnot/planner/objects/pixie_module.cc:560`). The GraphWidget's built-in + `doubleClickCallback` → `deepLinkURLFromSemanticType` (`graph.tsx` ~line 170) then + deep-links any `ST_POD_NAME` node to `px/pod`. We rely on that path; nothing under + `src/ui` is modified. Caveat: the column is stamped pod-typed even when an endpoint + resolves to a service/IP (pod > service > ip fallback), so a non-pod node double-click + deep-links to `px/pod?pod=` — harmless, and in the demo every endpoint is a pod. +- **Investigation zoom = the `investigation_id` script var.** Copy an `investigation_id` + from a graph-edge hover into the `investigation_id` variable (and set `pod_filter` to + the alert pod). Levels 2 and 3 re-run scoped to that investigation. `investigation_filter` + independently narrows the graph itself. + +## How to load it into a running Pixie UI (no rebuild) + +This is a self-contained scripts bundle — deploy it without touching the UI: + +1. **Custom Live View (fastest).** In the Live UI, open the script editor (the + `` **Scratch Pad** / "Edit script" pane), paste `evidence_graph.pxl` into the + **PxL** tab and `vis.json` into the **Vis Spec** tab, set the variables + (at minimum `clickhouse_dsn`), and Run. +2. **Bundled script.** The directory (`evidence_graph.pxl` + `vis.json` + + `manifest.yaml`) is globbed into `bundle-oss.json` by + `src/pxl_scripts/BUILD.bazel` (the `**/*.pxl|json|yaml` filegroup), so it ships as + the script id **`px/dx_evidence_graph`** wherever that bundle is served. No registry + edit is required. +3. **`px` CLI.** `px run -f evidence_graph.pxl` (table output) for a non-UI smoke test. + +Set the `clickhouse_dsn` variable to your forensic_db DSN (default: +`ingest_writer:changeme-ingest@clickhouse-forensic-soc-db.clickhouse.svc.cluster.local:9000/forensic_db`). + +## Feasibility: can a PxL Live View read ClickHouse via `clickhouse_dsn`? — **YES** + +Confirmed against the fork, not assumed: + +- `px.DataFrame(table, clickhouse_dsn=..., start_time=...)` is a first-class reader: + the arg is registered on the DataFrame op + (`src/carnot/planner/objects/dataframe.cc:189-197,558-559`) and executed by the PEM + through `ClickHouseSourceNode` (`src/carnot/exec/clickhouse_source_node.cc`). +- It is **already in production use** by the shipped `px/dx_evidence_graph` bundle + reading these exact tables, and `schema.sql` documents the tables as *"read by the + Pixie dx_evidence_graph UI via px.DataFrame(clickhouse_dsn=...)"*. +- The reader maps `String / Int8..64 / UInt8..64 / Float32/64 / DateTime / DateTime64` + → Pixie types (`clickhouse_source_node.cc:112-370`). Every column projected here is + in that set. + +So this bundle is built directly on `clickhouse_dsn`. **No alternative execution path +is needed.** + +### Real constraints (documented, not blockers) + +1. **Templated read, not arbitrary SQL.** The reader issues + `SELECT … FROM WHERE >= … [AND hostname = ] ORDER BY LIMIT …`. + It **cannot** run the `JSONExtract*` / `ARRAY JOIN` SQL from demo.md §H. The bundle + therefore re-implements the reconstruction in PxL: JSON columns + (`case_window`, `findings`) are parsed with `px.pluck_int64` / `px.pluck`, and + row-scoping is done with PxL filters (`px.contains`) instead of a SQL join. +2. **`hostname` partition filter.** When a `hostname` column exists the reader appends + `AND hostname = ` (`clickhouse_source_node.cc:429-434`). + Rows are only visible from the PEM whose host wrote them — a multi-node caveat. + `dc_snoop` is node-scoped so this is expected; for the dx tables ensure the reading + PEM matches the writing host (the shipped `px/dx_evidence_graph` operates under the + same rule). +3. **`start_time` is a no-op on nanosecond `event_time` tables.** The reader converts + `start_time` ns→seconds (`clickhouse_source_node.cc:69-75`) and compares it to + `event_time`. For tables whose `event_time` is `UInt64` **nanoseconds** + (`dx_evidence_graph`, `dx_evidence_manifest`, `kubescape_logs`) the seconds-scale + threshold is always ≤ the nanosecond values, so **all** in-TTL rows return (no time + narrowing — bounded by the 30-day TTL + `LIMIT`). On `dc_snoop` (`DateTime64(9)`) + `start_time` filtering does apply. This is why Level 3's exact window is read from + the **manifest** (`win_lo`/`win_hi`, Level 2) rather than from `start_time`. + +## Needs live-UI validation on a cluster with the dx evidence tables + +Statically validated here: `vis.json` is valid JSON; every variable is referenced by +a `globalFunc`; every widget binds to a declared `globalFunc`; func names and per-arg +names match the PxL signatures; projected columns exist in `schema.sql`; the `px.Pod` +/ `px.pluck_int64` / `px.contains` builtins used all exist in the fork. + +Cannot be run against a live PEM from here — validate on a cluster carrying +`forensic_db` (e.g. a PG with the SOC stack + AE): + +- All three funcs **compile and execute** through the PEM's `clickhouse_dsn` reader. +- `px.Pod(...)` produces `ST_POD_NAME` nodes and a **double-click deep-links to `px/pod`**. +- `px.pluck_int64(case_window, 'lo'|'hi')` returns the correct window bounds (Level 2). +- Level 3 `raw_table` swaps (`redis_events`, `kubescape_logs`) project without a + column-name error (their schemas differ from `dc_snoop`; adjust the projection if so). +- Graph rendering: `edgeColorColumn`/`edgeThresholds` colour by `max_severity`, hover + shows the investigation fields. diff --git a/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl b/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl index c5860a94a2c..b8c136a6244 100644 --- a/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl +++ b/src/pxl_scripts/px/dx_evidence_graph/dx_evidence_graph.pxl @@ -17,18 +17,82 @@ import px -def dx_evidence_graph(start_time: str, clickhouse_dsn: str, table: str): +def _ord(start_time: str, clickhouse_dsn: str, view: str, order_id: str): + df = px.DataFrame(view, clickhouse_dsn=clickhouse_dsn, start_time=start_time) + df = df[df.order_id == order_id] + return df.drop(['order_id', 'row_time', 'event_time']) + + +def evidence_graph(start_time: str, clickhouse_dsn: str, table: str): df = px.DataFrame(table, clickhouse_dsn=clickhouse_dsn, start_time=start_time) - df.requestor = px.select(df.requestor_pod == '', - px.select(df.requestor_service == '', df.requestor_ip, df.requestor_service), - df.requestor_pod) - df.responder = px.select(df.responder_pod == '', - px.select(df.responder_service == '', df.responder_ip, df.responder_service), - df.responder_pod) - return df[['requestor', 'responder', - 'requestor_pod', 'responder_pod', - 'requestor_service', 'responder_service', - 'requestor_ip', 'responder_ip', - 'weight', 'max_severity', 'confidence', - 'edge_kind', 'condition', 'criteria', 'num_findings', - 'investigation_id']] + orders = px.DataFrame('dx_orders', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + orders = orders[['kubescape_uid', 'rule_id', 'order_id']] + df = df.merge(orders, how='left', left_on=['uniqueID', 'rule'], + right_on=['kubescape_uid', 'rule_id'], suffixes=['', '_ord']) + df.order_link = px.script_reference(df.order_id, 'px/dx_evidence_graph', { + 'start_time': start_time, + 'clickhouse_dsn': clickhouse_dsn, + 'graph_table': table, + 'order_id': df.order_id, + }) + df.from_entity = px.Pod(df.subject_pod) + df.to_entity = df.target + return df[['from_entity', 'to_entity', 'order_link', 'uniqueID', 'rule', 'process', + 'target', 'target_kind', 'severity', 'alert', 'subject_pod']] + + +def orders(start_time: str, clickhouse_dsn: str, graph_table: str): + df = px.DataFrame('dx_orders', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + alerts = px.DataFrame(graph_table, clickhouse_dsn=clickhouse_dsn, start_time=start_time) + alerts = alerts[['uniqueID', 'rule', 'alert']] + df = df.merge(alerts, how='left', left_on=['kubescape_uid', 'rule_id'], + right_on=['uniqueID', 'rule'], suffixes=['', '_k']) + df.order = px.script_reference(df.order_id, 'px/dx_evidence_graph', { + 'start_time': start_time, + 'clickhouse_dsn': clickhouse_dsn, + 'graph_table': graph_table, + 'order_id': df.order_id, + }) + return df[['order', 'rule_id', 'disc', 'alert', 'pod']] + + +def kubescape(start_time: str, clickhouse_dsn: str, order_id: str): + orders = px.DataFrame('dx_orders', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + orders = orders[orders.order_id == order_id] + orders = orders[['kubescape_uid', 'rule_id']] + k = px.DataFrame('dx_src__kubescape_logs', clickhouse_dsn=clickhouse_dsn, start_time=start_time) + j = k.merge(orders, how='inner', left_on=['uniqueID', 'RuleID'], + right_on=['kubescape_uid', 'rule_id'], suffixes=['', '_ord']) + return j.drop(['row_time', 'event_time', 'kubescape_uid', 'rule_id']) + + +def conn(start_time: str, clickhouse_dsn: str, order_id: str): + return _ord(start_time, clickhouse_dsn, 'dx_ord__conn_stats', order_id) + + +def redis(start_time: str, clickhouse_dsn: str, order_id: str): + return _ord(start_time, clickhouse_dsn, 'dx_ord__redis_events', order_id) + + +def http(start_time: str, clickhouse_dsn: str, order_id: str): + return _ord(start_time, clickhouse_dsn, 'dx_ord__http_events', order_id) + + +def dns(start_time: str, clickhouse_dsn: str, order_id: str): + return _ord(start_time, clickhouse_dsn, 'dx_ord__dns_events', order_id) + + +def pgsql(start_time: str, clickhouse_dsn: str, order_id: str): + return _ord(start_time, clickhouse_dsn, 'dx_ord__pgsql_events', order_id) + + +def mysql(start_time: str, clickhouse_dsn: str, order_id: str): + return _ord(start_time, clickhouse_dsn, 'dx_ord__mysql_events', order_id) + + +def dc_snoop(start_time: str, clickhouse_dsn: str, order_id: str): + return _ord(start_time, clickhouse_dsn, 'dx_ord__dc_snoop', order_id) + + +def stack_trace(start_time: str, clickhouse_dsn: str, order_id: str): + return _ord(start_time, clickhouse_dsn, 'dx_ord__stack_trace', order_id) diff --git a/src/pxl_scripts/px/dx_evidence_graph/vis.json b/src/pxl_scripts/px/dx_evidence_graph/vis.json index 90befd97383..dba495a3f9e 100644 --- a/src/pxl_scripts/px/dx_evidence_graph/vis.json +++ b/src/pxl_scripts/px/dx_evidence_graph/vis.json @@ -1,75 +1,195 @@ { - "variables": [ - { - "name": "start_time", - "type": "PX_STRING", - "description": "Start time of the window.", - "defaultValue": "-15m" - }, - { - "name": "clickhouse_dsn", - "type": "PX_STRING", - "description": "ClickHouse DSN: user:pass@host:port/db.", - "defaultValue": "forensic_analyst:changeme-analyst@clickhouse-forensic-soc-db.clickhouse.svc.cluster.local:9000/forensic_db" - }, - { - "name": "table", - "type": "PX_STRING", - "description": "dx_evidence_graph", - "defaultValue": "dx_evidence_graph_malignant" - } - ], - "globalFuncs": [ - { - "outputName": "dx_graph", - "func": { - "name": "dx_evidence_graph", - "args": [ - {"name": "start_time", "variable": "start_time"}, - {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, - {"name": "table", "variable": "table"} - ] - } - } - ], - "widgets": [ - { - "name": "DX Evidence Graph", - "position": {"x": 0, "y": 0, "w": 12, "h": 5}, - "globalFuncOutputName": "dx_graph", - "displaySpec": { - "@type": "types.px.dev/px.vispb.Graph", - "adjacencyList": { - "fromColumn": "requestor", - "toColumn": "responder" - }, - "edgeWeightColumn": "weight", - "edgeColorColumn": "max_severity", - "edgeLabelColumn": "edge_kind", - "edgeThresholds": { - "mediumThreshold": 3, - "highThreshold": 4 - }, - "edgeHoverInfo": [ - "edge_kind", - "condition", - "criteria", - "weight", - "max_severity", - "confidence", - "num_findings", - "investigation_id" - ], - "edgeLength": 500 - } - }, - { - "name": "Edges", - "position": {"x": 0, "y": 5, "w": 12, "h": 4}, - "globalFuncOutputName": "dx_graph", - "displaySpec": { - "@type": "types.px.dev/px.vispb.Table" - } - } - ] + "variables": [ + { + "name": "start_time", + "type": "PX_STRING", + "description": "Window start.", + "defaultValue": "-6h" + }, + { + "name": "clickhouse_dsn", + "type": "PX_STRING", + "description": "forensic_db DSN: user:pass@host:port/db.", + "defaultValue": "forensic_analyst:changeme-analyst@clickhouse-forensic-soc-db.clickhouse.svc.cluster.local:9000/forensic_db" + }, + { + "name": "graph_table", + "type": "PX_STRING", + "description": "L1 kill-chain graph source.", + "defaultValue": "dx_kubescape_anomalies" + }, + { + "name": "order_id", + "type": "PX_STRING", + "description": "Set by clicking an order link in the graph popup or the ORDERS panel; every protocol panel snaps to that order's consulted records.", + "defaultValue": "" + } + ], + "globalFuncs": [ + { + "outputName": "g_graph", + "func": {"name": "evidence_graph", "args": [ + {"name": "start_time", "variable": "start_time"}, + {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, + {"name": "table", "variable": "graph_table"} + ]} + }, + { + "outputName": "g_orders", + "func": {"name": "orders", "args": [ + {"name": "start_time", "variable": "start_time"}, + {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, + {"name": "graph_table", "variable": "graph_table"} + ]} + }, + { + "outputName": "g_kube", + "func": {"name": "kubescape", "args": [ + {"name": "start_time", "variable": "start_time"}, + {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, + {"name": "order_id", "variable": "order_id"} + ]} + }, + { + "outputName": "g_conn", + "func": {"name": "conn", "args": [ + {"name": "start_time", "variable": "start_time"}, + {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, + {"name": "order_id", "variable": "order_id"} + ]} + }, + { + "outputName": "g_redis", + "func": {"name": "redis", "args": [ + {"name": "start_time", "variable": "start_time"}, + {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, + {"name": "order_id", "variable": "order_id"} + ]} + }, + { + "outputName": "g_http", + "func": {"name": "http", "args": [ + {"name": "start_time", "variable": "start_time"}, + {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, + {"name": "order_id", "variable": "order_id"} + ]} + }, + { + "outputName": "g_dns", + "func": {"name": "dns", "args": [ + {"name": "start_time", "variable": "start_time"}, + {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, + {"name": "order_id", "variable": "order_id"} + ]} + }, + { + "outputName": "g_pgsql", + "func": {"name": "pgsql", "args": [ + {"name": "start_time", "variable": "start_time"}, + {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, + {"name": "order_id", "variable": "order_id"} + ]} + }, + { + "outputName": "g_mysql", + "func": {"name": "mysql", "args": [ + {"name": "start_time", "variable": "start_time"}, + {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, + {"name": "order_id", "variable": "order_id"} + ]} + }, + { + "outputName": "g_dcsnoop", + "func": {"name": "dc_snoop", "args": [ + {"name": "start_time", "variable": "start_time"}, + {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, + {"name": "order_id", "variable": "order_id"} + ]} + }, + { + "outputName": "g_stack", + "func": {"name": "stack_trace", "args": [ + {"name": "start_time", "variable": "start_time"}, + {"name": "clickhouse_dsn", "variable": "clickhouse_dsn"}, + {"name": "order_id", "variable": "order_id"} + ]} + } + ], + "widgets": [ + { + "name": "Evidence graph (subject pod -> target; click an edge for details + order link)", + "position": {"x": 0, "y": 0, "w": 12, "h": 4}, + "globalFuncOutputName": "g_graph", + "displaySpec": { + "@type": "types.px.dev/px.vispb.Graph", + "adjacencyList": {"fromColumn": "from_entity", "toColumn": "to_entity"}, + "edgeWeightColumn": "severity", + "edgeColorColumn": "severity", + "edgeLabelColumn": "rule", + "edgeThresholds": {"mediumThreshold": 5, "highThreshold": 8}, + "edgeHoverInfo": ["order_link", "rule", "alert", "process", "target", "severity", "uniqueID"], + "edgeLength": 500 + } + }, + { + "name": "ORDERS (click an order to filter all panels)", + "position": {"x": 0, "y": 4, "w": 12, "h": 3}, + "globalFuncOutputName": "g_orders", + "displaySpec": {"@type": "types.px.dev/px.vispb.Table"} + }, + { + "name": "kubescape_logs (anomalies) for order", + "position": {"x": 0, "y": 7, "w": 12, "h": 4}, + "globalFuncOutputName": "g_kube", + "displaySpec": {"@type": "types.px.dev/px.vispb.Table"} + }, + { + "name": "conn_stats (consulted records for order)", + "position": {"x": 0, "y": 11, "w": 6, "h": 4}, + "globalFuncOutputName": "g_conn", + "displaySpec": {"@type": "types.px.dev/px.vispb.Table"} + }, + { + "name": "redis_events (consulted records for order)", + "position": {"x": 6, "y": 11, "w": 6, "h": 4}, + "globalFuncOutputName": "g_redis", + "displaySpec": {"@type": "types.px.dev/px.vispb.Table"} + }, + { + "name": "http_events (consulted records for order)", + "position": {"x": 0, "y": 15, "w": 6, "h": 4}, + "globalFuncOutputName": "g_http", + "displaySpec": {"@type": "types.px.dev/px.vispb.Table"} + }, + { + "name": "dns_events (consulted records for order)", + "position": {"x": 6, "y": 15, "w": 6, "h": 4}, + "globalFuncOutputName": "g_dns", + "displaySpec": {"@type": "types.px.dev/px.vispb.Table"} + }, + { + "name": "pgsql_events (consulted records for order)", + "position": {"x": 0, "y": 19, "w": 6, "h": 4}, + "globalFuncOutputName": "g_pgsql", + "displaySpec": {"@type": "types.px.dev/px.vispb.Table"} + }, + { + "name": "mysql_events (consulted records for order)", + "position": {"x": 6, "y": 19, "w": 6, "h": 4}, + "globalFuncOutputName": "g_mysql", + "displaySpec": {"@type": "types.px.dev/px.vispb.Table"} + }, + { + "name": "dc_snoop (file access) for order", + "position": {"x": 0, "y": 23, "w": 6, "h": 4}, + "globalFuncOutputName": "g_dcsnoop", + "displaySpec": {"@type": "types.px.dev/px.vispb.Table"} + }, + { + "name": "stack_trace (profiler) for order", + "position": {"x": 6, "y": 23, "w": 6, "h": 4}, + "globalFuncOutputName": "g_stack", + "displaySpec": {"@type": "types.px.dev/px.vispb.Table"} + } + ] } diff --git a/src/ui/src/containers/live-widgets/graph/graph.tsx b/src/ui/src/containers/live-widgets/graph/graph.tsx index a8a642eec84..685931bf571 100644 --- a/src/ui/src/containers/live-widgets/graph/graph.tsx +++ b/src/ui/src/containers/live-widgets/graph/graph.tsx @@ -46,6 +46,7 @@ import { } from './graph-utils'; import { formatByDataType, formatBySemType } from '../../format-data/format-data'; import { deepLinkURLFromSemanticType } from '../utils/live-view-params'; +import { ScriptReference } from '../utils/script-reference'; interface AdjacencyList { toColumn: string; @@ -157,8 +158,11 @@ export const Graph = React.memo(({ const [graph, setGraph] = React.useState(null); const [pinned, setPinned] = React.useState>([]); const pinSeq = React.useRef(0); + const [edgeScriptRefs, setEdgeScriptRefs] = React.useState< + Map>(() => new Map()); const [edgeLabels, setEdgeLabels] = React.useState>(() => new Map()); const [labelOffsets, setLabelOffsets] = React.useState>(() => new Map()); @@ -198,6 +202,7 @@ export const Graph = React.memo(({ const nodes = new visData.DataSet(); const idToSemType = {}; const labelMap = new Map(); + const scriptRefMap = new Map(); const selfLoopCounts = new Map(); const selfLoopRank = new Map(); @@ -257,8 +262,17 @@ export const Graph = React.memo(({ if (edgeHoverInfo && edgeHoverInfo.length > 0) { let edgeInfo = ''; - edgeHoverInfo.forEach((info, i) => { + edgeHoverInfo.forEach((info) => { if (info != null) { + // Script-reference columns become the deep link in the pinned popup, + // not a line in the hover text (a hover tooltip can't be clicked). + if (info.semType === SemanticType.ST_SCRIPT_REFERENCE) { + const ref = d[info.name]; + if (ref && ref.script) { + scriptRefMap.set(edgeId, { label: ref.label, script: ref.script, args: ref.args }); + } + return; + } let val: string; if (info.semType === SemanticType.ST_NONE || info.semType === SemanticType.ST_UNSPECIFIED) { val = formatByDataType(info.type, d[info.name]); @@ -266,7 +280,7 @@ export const Graph = React.memo(({ const valWithUnits = formatBySemType(info.semType, d[info.name]); val = `${valWithUnits.val} ${valWithUnits.units}`; } - edgeInfo = `${edgeInfo}${i === 0 ? '' : '
'} ${info.name}: ${val}`; + edgeInfo = `${edgeInfo}${edgeInfo === '' ? '' : '
'} ${info.name}: ${val}`; } }); edge.title = edgeInfo; @@ -279,6 +293,7 @@ export const Graph = React.memo(({ nodes, edges, idToSemType, }); setEdgeLabels(labelMap); + setEdgeScriptRefs(scriptRefMap); setLabelOffsets((prev) => { const next = new Map(); selfLoopRank.forEach((rank, edgeId) => { @@ -331,6 +346,7 @@ export const Graph = React.memo(({ title: String(edgeData?.title ?? ''), x: rect.left + params.pointer.DOM.x, y: rect.top + params.pointer.DOM.y, + scriptRef: edgeScriptRefs.get(edgeId), }]); } }); @@ -401,7 +417,7 @@ export const Graph = React.memo(({ const onPinPointerDown = React.useCallback((key: number, initialX: number, initialY: number) => (e: React.PointerEvent) => { - if ((e.target as HTMLElement).closest('[data-pin-close]')) return; + if ((e.target as HTMLElement).closest('a, [data-pin-close]')) return; e.stopPropagation(); const startX = e.clientX; const startY = e.clientY; @@ -490,6 +506,17 @@ export const Graph = React.memo(({ touchAction: 'none', }} > + {p.scriptRef && ( +
+ +
+ )}
target), deduped by uniqueID. +CREATE VIEW IF NOT EXISTS forensic_db.dx_kubescape_anomalies AS +SELECT JSONExtractString(BaseRuntimeMetadata, 'uniqueID') AS uniqueID, + concat(JSONExtractString(RuntimeK8sDetails, 'podNamespace'), '/', JSONExtractString(RuntimeK8sDetails, 'podName')) AS subject_pod, + RuleID AS rule, + JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'process'), 'name') AS process, + multiIf(JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'dns'), 'domain') != '', JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'dns'), 'domain'), JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'network'), 'dstIP') != '', JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'network'), 'dstIP'), JSONExtractString(JSONExtractRaw(BaseRuntimeMetadata, 'arguments'), 'path') != '', JSONExtractString(JSONExtractRaw(BaseRuntimeMetadata, 'arguments'), 'path'), JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'file'), 'name') != '', concat(JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'file'), 'directory'), '/', JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'file'), 'name')), 'unknown') AS target, + multiIf(JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'dns'), 'domain') != '', 'domain', JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'network'), 'dstIP') != '', 'endpoint', (JSONExtractString(JSONExtractRaw(BaseRuntimeMetadata, 'arguments'), 'path') != '') OR (JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'file'), 'name') != ''), 'file', 'other') AS target_kind, + toInt8OrZero(JSONExtractString(BaseRuntimeMetadata, 'severity')) AS severity, + message AS alert, hostname, event_time +FROM forensic_db.kubescape_logs +WHERE RuleID != '' AND JSONExtractString(BaseRuntimeMetadata, 'uniqueID') != '' +LIMIT 1 BY uniqueID; + +-- dx_src__kubescape_logs: anomaly detail (process tree comm/cmdline/pcomm) per panel. +CREATE VIEW IF NOT EXISTS forensic_db.dx_src__kubescape_logs AS +SELECT toString(fromUnixTimestamp64Nano(toInt64(event_time))) AS ts, toInt64(event_time) AS row_time, event_time, + RuleID, JSONExtractString(BaseRuntimeMetadata, 'uniqueID') AS uniqueID, + JSONExtractString(JSONExtractRaw(RuntimeProcessDetails, 'processTree'), 'comm') AS comm, + JSONExtractString(JSONExtractRaw(RuntimeProcessDetails, 'processTree'), 'pcomm') AS parent, + JSONExtractString(JSONExtractRaw(RuntimeProcessDetails, 'processTree'), 'cmdline') AS cmdline, + message AS alert, + concat(JSONExtractString(RuntimeK8sDetails, 'podNamespace'), '/', JSONExtractString(RuntimeK8sDetails, 'podName')) AS pod, hostname +FROM forensic_db.kubescape_logs WHERE RuleID != ''; + +-- dx_src__: original protocol schema + ts/row_time/event_time, encrypted/ssl dropped. +CREATE VIEW IF NOT EXISTS forensic_db.dx_src__redis_events AS +SELECT toString(time_) AS ts, toInt64(toUnixTimestamp64Nano(time_)) AS row_time, toUInt64(toUnixTimestamp64Nano(event_time)) AS event_time, + namespace, pod, remote_addr, remote_port, trace_role, req_cmd, req_args, resp, latency, hostname +FROM forensic_db.redis_events; + +CREATE VIEW IF NOT EXISTS forensic_db.dx_src__conn_stats AS +SELECT toString(time_) AS ts, toInt64(toUnixTimestamp64Nano(time_)) AS row_time, toUInt64(toUnixTimestamp64Nano(event_time)) AS event_time, + namespace, pod, remote_addr, remote_port, protocol, conn_open, conn_close, conn_active, bytes_sent, bytes_recv, hostname +FROM forensic_db.conn_stats; + +CREATE VIEW IF NOT EXISTS forensic_db.dx_src__http_events AS +SELECT toString(time_) AS ts, toInt64(toUnixTimestamp64Nano(time_)) AS row_time, toUInt64(toUnixTimestamp64Nano(event_time)) AS event_time, + namespace, pod, remote_addr, remote_port, req_method, req_path, req_body, resp_status, resp_body, latency, hostname +FROM forensic_db.http_events; + +CREATE VIEW IF NOT EXISTS forensic_db.dx_src__dns_events AS +SELECT toString(time_) AS ts, toInt64(toUnixTimestamp64Nano(time_)) AS row_time, toUInt64(toUnixTimestamp64Nano(event_time)) AS event_time, + namespace, pod, remote_addr, remote_port, req_body, resp_body, latency, hostname +FROM forensic_db.dns_events; + +CREATE VIEW IF NOT EXISTS forensic_db.dx_src__pgsql_events AS +SELECT toString(time_) AS ts, toInt64(toUnixTimestamp64Nano(time_)) AS row_time, toUInt64(toUnixTimestamp64Nano(event_time)) AS event_time, + namespace, pod, remote_addr, remote_port, req, resp, latency, hostname +FROM forensic_db.pgsql_events; + +CREATE VIEW IF NOT EXISTS forensic_db.dx_src__mysql_events AS +SELECT toString(time_) AS ts, toInt64(toUnixTimestamp64Nano(time_)) AS row_time, toUInt64(toUnixTimestamp64Nano(event_time)) AS event_time, + namespace, pod, remote_addr, remote_port, req_cmd, req_body, resp_status, resp_body, latency, hostname +FROM forensic_db.mysql_events; + +CREATE VIEW IF NOT EXISTS forensic_db.dx_src__dc_snoop AS +SELECT toString(time_) AS ts, toInt64(toUnixTimestamp64Nano(time_)) AS row_time, toUInt64(toUnixTimestamp64Nano(event_time)) AS event_time, + pid, comm, t, file, namespace, pod, container, hostname +FROM forensic_db.dc_snoop; + +CREATE VIEW IF NOT EXISTS forensic_db.dx_src__stack_trace AS +SELECT toString(time_) AS ts, toInt64(toUnixTimestamp64Nano(time_)) AS row_time, toUInt64(toUnixTimestamp64Nano(event_time)) AS event_time, + namespace, pod, container, stack_trace_id, stack_trace, count, hostname +FROM forensic_db.stack_trace; + +CREATE VIEW IF NOT EXISTS forensic_db.dx_ord__redis_events AS +SELECT + e.order_id AS order_id, + toString(c.time_) AS ts, + toInt64(toUnixTimestamp64Nano(c.time_)) AS row_time, + toUInt64(toUnixTimestamp64Nano(c.event_time)) AS event_time, + c.namespace AS namespace, + c.pod AS pod, + c.remote_addr AS remote_addr, + c.remote_port AS remote_port, + c.trace_role AS trace_role, + c.req_cmd AS req_cmd, + c.req_args AS req_args, + c.resp AS resp, + c.latency AS latency, + e.hostname AS hostname +FROM forensic_db.dx_order_edges AS e +INNER JOIN forensic_db.redis_events AS c ON c.unique_id = e.unique_id +WHERE e.src_table = 'redis_events'; + +CREATE VIEW IF NOT EXISTS forensic_db.dx_ord__http_events AS +SELECT + e.order_id AS order_id, + toString(c.time_) AS ts, + toInt64(toUnixTimestamp64Nano(c.time_)) AS row_time, + toUInt64(toUnixTimestamp64Nano(c.event_time)) AS event_time, + c.namespace AS namespace, + c.pod AS pod, + c.remote_addr AS remote_addr, + c.remote_port AS remote_port, + c.req_method AS req_method, + c.req_path AS req_path, + c.req_body AS req_body, + c.resp_status AS resp_status, + c.resp_body AS resp_body, + c.latency AS latency, + e.hostname AS hostname +FROM forensic_db.dx_order_edges AS e +INNER JOIN forensic_db.http_events AS c ON c.unique_id = e.unique_id +WHERE e.src_table = 'http_events'; + +CREATE VIEW IF NOT EXISTS forensic_db.dx_ord__dns_events AS +SELECT + e.order_id AS order_id, + toString(c.time_) AS ts, + toInt64(toUnixTimestamp64Nano(c.time_)) AS row_time, + toUInt64(toUnixTimestamp64Nano(c.event_time)) AS event_time, + c.namespace AS namespace, + c.pod AS pod, + c.remote_addr AS remote_addr, + c.remote_port AS remote_port, + c.req_body AS req_body, + c.resp_body AS resp_body, + c.latency AS latency, + e.hostname AS hostname +FROM forensic_db.dx_order_edges AS e +INNER JOIN forensic_db.dns_events AS c ON c.unique_id = e.unique_id +WHERE e.src_table = 'dns_events'; + +CREATE VIEW IF NOT EXISTS forensic_db.dx_ord__pgsql_events AS +SELECT + e.order_id AS order_id, + toString(c.time_) AS ts, + toInt64(toUnixTimestamp64Nano(c.time_)) AS row_time, + toUInt64(toUnixTimestamp64Nano(c.event_time)) AS event_time, + c.namespace AS namespace, + c.pod AS pod, + c.remote_addr AS remote_addr, + c.remote_port AS remote_port, + c.req AS req, + c.resp AS resp, + c.latency AS latency, + e.hostname AS hostname +FROM forensic_db.dx_order_edges AS e +INNER JOIN forensic_db.pgsql_events AS c ON c.unique_id = e.unique_id +WHERE e.src_table = 'pgsql_events'; + +CREATE VIEW IF NOT EXISTS forensic_db.dx_ord__mysql_events AS +SELECT + e.order_id AS order_id, + toString(c.time_) AS ts, + toInt64(toUnixTimestamp64Nano(c.time_)) AS row_time, + toUInt64(toUnixTimestamp64Nano(c.event_time)) AS event_time, + c.namespace AS namespace, + c.pod AS pod, + c.remote_addr AS remote_addr, + c.remote_port AS remote_port, + c.req_cmd AS req_cmd, + c.req_body AS req_body, + c.resp_status AS resp_status, + c.resp_body AS resp_body, + c.latency AS latency, + e.hostname AS hostname +FROM forensic_db.dx_order_edges AS e +INNER JOIN forensic_db.mysql_events AS c ON c.unique_id = e.unique_id +WHERE e.src_table = 'mysql_events'; + +CREATE VIEW IF NOT EXISTS forensic_db.dx_ord__dc_snoop AS +SELECT + e.order_id AS order_id, + toString(c.time_) AS ts, + toInt64(toUnixTimestamp64Nano(c.time_)) AS row_time, + toUInt64(toUnixTimestamp64Nano(c.event_time)) AS event_time, + c.pid AS pid, + c.comm AS comm, + c.t AS t, + c.file AS file, + c.namespace AS namespace, + c.pod AS pod, + c.container AS container, + e.hostname AS hostname +FROM forensic_db.dx_order_edges AS e +INNER JOIN forensic_db.dc_snoop AS c ON c.unique_id = e.unique_id +WHERE e.src_table = 'dc_snoop'; + +CREATE VIEW IF NOT EXISTS forensic_db.dx_ord__stack_trace AS +SELECT + e.order_id AS order_id, + toString(c.time_) AS ts, + toInt64(toUnixTimestamp64Nano(c.time_)) AS row_time, + toUInt64(toUnixTimestamp64Nano(c.event_time)) AS event_time, + c.namespace AS namespace, + c.pod AS pod, + c.container AS container, + c.stack_trace_id AS stack_trace_id, + c.stack_trace AS stack_trace, + c.count AS count, + e.hostname AS hostname +FROM forensic_db.dx_order_edges AS e +INNER JOIN forensic_db.stack_trace AS c ON c.unique_id = e.unique_id +WHERE e.src_table = 'stack_trace'; + +-- ── MITRE ATT&CK enrichment over kubescape_logs (px/dx_evidence_graph) ──────── +-- dx_kubescape_mitre: L1 graph source — one row per (uniqueID, rule) so orders +-- keyed on (uniqueID, RuleID) all join; MITRE + resolved target from BaseRuntimeMetadata. +CREATE VIEW IF NOT EXISTS forensic_db.dx_kubescape_mitre AS +SELECT JSONExtractString(BaseRuntimeMetadata, 'uniqueID') AS uniqueID, + concat(JSONExtractString(RuntimeK8sDetails, 'podNamespace'), '/', JSONExtractString(RuntimeK8sDetails, 'podName')) AS subject_pod, + RuleID AS rule, + JSONExtractString(BaseRuntimeMetadata, 'mitreTactic') AS mitre_tactic, + JSONExtractString(BaseRuntimeMetadata, 'mitreTechnique') AS mitre_technique, + concat(RuleID, ' · ', JSONExtractString(BaseRuntimeMetadata, 'mitreTechnique')) AS rule_mitre, + JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'process'), 'name') AS process, + multiIf(JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'dns'), 'domain') != '', +JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'dns'), 'domain'), +JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'network'), 'dstIP') != '', +JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'network'), 'dstIP'), JSONExtractString(JSONExtractRaw(BaseRuntimeMetadata, +'arguments'), 'path') != '', JSONExtractString(JSONExtractRaw(BaseRuntimeMetadata, 'arguments'), 'path'), +JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'file'), 'name') != '', +concat(JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'file'), 'directory'), '/', +JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'file'), 'name')), 'unknown') AS target, + multiIf(JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'dns'), 'domain') != '', 'domain', +JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, 'identifiers'), 'network'), 'dstIP') != '', 'endpoint', +(JSONExtractString(JSONExtractRaw(BaseRuntimeMetadata, 'arguments'), 'path') != '') OR (JSONExtractString(JSONExtractRaw(JSONExtractRaw(BaseRuntimeMetadata, +'identifiers'), 'file'), 'name') != ''), 'file', 'other') AS target_kind, + toInt8OrZero(JSONExtractString(BaseRuntimeMetadata, 'severity')) AS severity, + message AS alert, hostname, event_time +FROM forensic_db.kubescape_logs +WHERE RuleID != '' AND JSONExtractString(BaseRuntimeMetadata, 'uniqueID') != '' +LIMIT 1 BY uniqueID, rule; + +-- dx_src__kubescape_mitre: kubescape detail panel — MITRE cols after RuleID, plus +-- process tree (comm/pcomm/cmdline). ts/row_time/event_time px-connector convention. +CREATE VIEW IF NOT EXISTS forensic_db.dx_src__kubescape_mitre AS +SELECT toString(fromUnixTimestamp64Nano(toInt64(event_time))) AS ts, toInt64(event_time) AS row_time, event_time, + RuleID, + JSONExtractString(BaseRuntimeMetadata, 'mitreTactic') AS mitre_tactic, + JSONExtractString(BaseRuntimeMetadata, 'mitreTechnique') AS mitre_technique, + JSONExtractString(BaseRuntimeMetadata, 'uniqueID') AS uniqueID, + JSONExtractString(JSONExtractRaw(RuntimeProcessDetails, 'processTree'), 'comm') AS comm, + JSONExtractInt(JSONExtractRaw(RuntimeProcessDetails, 'processTree'), 'pid') AS pid, + JSONExtractString(JSONExtractRaw(RuntimeProcessDetails, 'processTree'), 'pcomm') AS parent, + JSONExtractInt(JSONExtractRaw(RuntimeProcessDetails, 'processTree'), 'ppid') AS ppid, + JSONExtractString(JSONExtractRaw(RuntimeProcessDetails, 'processTree'), 'cmdline') AS cmdline, + message AS alert, + concat(JSONExtractString(RuntimeK8sDetails, 'podNamespace'), '/', JSONExtractString(RuntimeK8sDetails, 'podName')) AS pod, hostname +FROM forensic_db.kubescape_logs WHERE RuleID != ''; + +-- dx_orders_win: per-order ±300s baseline/attack window for the differential +-- flamegraph (stack_diff). hostname carried so the px node-shard resolves. +CREATE VIEW IF NOT EXISTS forensic_db.dx_orders_win AS +SELECT order_id, pod, + toInt64(event_time) - 300000000000 AS lo, + toInt64(event_time) + 300000000000 AS hi, + event_time, hostname +FROM forensic_db.dx_orders; + +-- dx_base__dc_snoop: passthrough so PxL infers unique_id (the registered dc_snoop +-- relation omits it), enabling the dc_snoop panel's fast base+edges bridge join. +CREATE VIEW IF NOT EXISTS forensic_db.dx_base__dc_snoop AS +SELECT * FROM forensic_db.dc_snoop; diff --git a/src/vizier/services/adaptive_export/internal/control/BUILD.bazel b/src/vizier/services/adaptive_export/internal/control/BUILD.bazel index c22b1b8ba71..6253c6a54d5 100644 --- a/src/vizier/services/adaptive_export/internal/control/BUILD.bazel +++ b/src/vizier/services/adaptive_export/internal/control/BUILD.bazel @@ -19,19 +19,26 @@ load("//bazel:pl_build_system.bzl", "pl_go_test") go_library( name = "control", - srcs = ["server.go"], + srcs = [ + "server.go", + "tls.go", + ], importpath = "px.dev/pixie/src/vizier/services/adaptive_export/internal/control", visibility = ["//src/vizier/services/adaptive_export:__subpackages__"], deps = [ "//src/shared/services/utils", "//src/vizier/services/adaptive_export/internal/activeset", "//src/vizier/services/adaptive_export/internal/anomaly", + "@com_github_sirupsen_logrus//:logrus", ], ) pl_go_test( name = "control_test", - srcs = ["server_test.go"], + srcs = [ + "server_test.go", + "tls_test.go", + ], embed = [":control"], deps = [ "//src/shared/services/utils", diff --git a/src/vizier/services/adaptive_export/internal/control/server.go b/src/vizier/services/adaptive_export/internal/control/server.go index 96292715fcb..99831bd339b 100644 --- a/src/vizier/services/adaptive_export/internal/control/server.go +++ b/src/vizier/services/adaptive_export/internal/control/server.go @@ -34,6 +34,8 @@ import ( "strings" "time" + log "github.com/sirupsen/logrus" + jwtutils "px.dev/pixie/src/shared/services/utils" "px.dev/pixie/src/vizier/services/adaptive_export/internal/activeset" "px.dev/pixie/src/vizier/services/adaptive_export/internal/anomaly" @@ -67,6 +69,10 @@ type exportAller interface { // anomaly is comfortably inside the pulled slice. const controlExportLookback = 600 * time.Second +// A /query window narrower than this is widened to controlExportLookback (a +// point window keyed on one finding's timestamp matches no pixie rows). +const minControlQueryWindow = 5 * time.Second + // The control API carries timestamps in the pipeline's ONE unit: unix // NANOSECONDS — the same unit as forensic_db.*.event_time and dx's referral // windows. Read them with time.Unix(0, ns). (This spot previously did @@ -86,12 +92,21 @@ type manifestWriter interface { WriteEvidenceManifest(ctx context.Context, jsonEachRow []byte) error } +// rowsWriter persists dx-handed pixie base rows (loop 1: conn_stats with a +// pre-stamped unique_id) into forensic_db.
through the SAME sink the +// controller capture path uses (sink.ClickHouseHTTP.WritePixieRows). +// nil → /dx/rows 501s. +type rowsWriter interface { + WritePixieRows(ctx context.Context, table string, rows []map[string]any) error +} + // Server is the control HTTP surface. type Server struct { set exporter runner queryRunner // may be nil; /query then returns 501 graph graphWriter // may be nil; /dx/evidence_graph then returns 501 manifest manifestWriter // may be nil; /dx/evidence_manifest then returns 501 + rows rowsWriter // may be nil; /dx/rows then returns 501 mux *http.ServeMux verify func(bearer string) error // nil → auth disabled; set via SetAuth } @@ -106,6 +121,7 @@ func New(set exporter, runner queryRunner) *Server { s.mux.HandleFunc("/query", s.handleQuery) s.mux.HandleFunc("/dx/evidence_graph", s.handleDXEvidenceGraph) s.mux.HandleFunc("/dx/evidence_manifest", s.handleDXEvidenceManifest) + s.mux.HandleFunc("/dx/rows", s.handleDXRows) return s } @@ -115,6 +131,9 @@ func (s *Server) SetGraphWriter(g graphWriter) { s.graph = g } // SetManifestWriter wires the dx_evidence_manifest sink. func (s *Server) SetManifestWriter(m manifestWriter) { s.manifest = m } +// SetRowsWriter wires the /dx/rows base-row sink (loop 1). +func (s *Server) SetRowsWriter(rw rowsWriter) { s.rows = rw } + // SetAuth turns on bearer-JWT auth for the control surface, verified with the // SAME shared lib + signing key the vizier broker/PEM use (px.dev/pixie/src/ // shared/services/utils). dx already mints a service JWT (GenerateJWTForService, @@ -179,6 +198,59 @@ func (s *Server) handleDXEvidenceGraph(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusAccepted) } +// dxRowsAllowedTables guards /dx/rows against arbitrary-table writes: only the +// bridged tables dx hands base rows for (each carries a pre-stamped unique_id and +// a dx_ord__ view) are accepted. Mirrors evidencegraph.UIDColsByTable on the dx side. +var dxRowsAllowedTables = map[string]bool{ + "conn_stats": true, + "redis_events": true, + "http_events": true, + "dns_events": true, + "pgsql_events": true, + "mysql_events": true, + "dc_snoop": true, + "stack_trace": true, +} + +// dxRowsReq is the /dx/rows wire body: dx-handed base rows for one table. +type dxRowsReq struct { + Table string `json:"table"` + Rows []map[string]any `json:"rows"` +} + +// handleDXRows ingests dx-handed base rows (loop 1: conn_stats carrying a +// pre-stamped content-hash unique_id, a hex String) and writes them to +// forensic_db.
via the same sink path the controller capture uses. +// decodeNumber (UseNumber) keeps large integer columns as json.Number so the +// fast encoder emits exact decimal text; the shared decode() would cast them to +// float64, and the sink's appendFloat renders large values in scientific +// notation, which ClickHouse rejects for Int64/UInt64 columns (whole batch 502). +func (s *Server) handleDXRows(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + w.WriteHeader(http.StatusMethodNotAllowed) + return + } + if s.rows == nil { + w.WriteHeader(http.StatusNotImplemented) + return + } + var req dxRowsReq + if !decodeNumber(w, r, &req) || !dxRowsAllowedTables[req.Table] { + w.WriteHeader(http.StatusBadRequest) + return + } + if len(req.Rows) == 0 { + w.WriteHeader(http.StatusAccepted) + return + } + if err := s.rows.WritePixieRows(r.Context(), req.Table, req.Rows); err != nil { + log.WithField("table", req.Table).WithField("rows", len(req.Rows)).WithError(err).Error("dx/rows: WritePixieRows failed") + w.WriteHeader(http.StatusBadGateway) + return + } + w.WriteHeader(http.StatusAccepted) +} + // dxManifest mirrors the wire shape of dx's manifest.Manifest (internal/manifest). // Scalars map to typed forensic_db.dx_evidence_manifest columns; the nested // collections are held as raw JSON and persisted as JSON text in String columns @@ -295,6 +367,14 @@ func decode(w http.ResponseWriter, r *http.Request, v any) bool { return json.NewDecoder(r.Body).Decode(v) == nil } +func decodeNumber(w http.ResponseWriter, r *http.Request, v any) bool { + defer r.Body.Close() + r.Body = http.MaxBytesReader(w, r.Body, maxControlBodyBytes) + dec := json.NewDecoder(r.Body) + dec.UseNumber() + return dec.Decode(v) == nil +} + // ── handlers ────────────────────────────────────────────────────────── func (s *Server) handleHealth(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusOK) @@ -353,8 +433,12 @@ func (s *Server) handleQuery(w http.ResponseWriter, r *http.Request) { w.WriteHeader(http.StatusBadRequest) return } - err := s.runner.OrderQuery(req.target(), req.Table, - time.Unix(0, req.Window[0]).UTC(), time.Unix(0, req.Window[1]).UTC(), req.QueryID) + hi := time.Unix(0, req.Window[1]).UTC() + lo := time.Unix(0, req.Window[0]).UTC() + if hi.Sub(lo) < minControlQueryWindow { + lo = hi.Add(-controlExportLookback) // widen a point window + } + err := s.runner.OrderQuery(req.target(), req.Table, lo, hi, req.QueryID) if err != nil { w.WriteHeader(http.StatusBadGateway) return diff --git a/src/vizier/services/adaptive_export/internal/control/server_test.go b/src/vizier/services/adaptive_export/internal/control/server_test.go index 15b3122ad0e..c84d940b951 100644 --- a/src/vizier/services/adaptive_export/internal/control/server_test.go +++ b/src/vizier/services/adaptive_export/internal/control/server_test.go @@ -21,6 +21,7 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "strconv" "strings" "testing" "time" @@ -45,12 +46,14 @@ func (f *fakeExporter) Remove(k activeset.Key) { f.removes = append(f.removes, k // fakeRunner records OrderQuery calls; err controls the failure path. type fakeRunner struct { - calls []string // "table|ns/pod|queryID" - err error + calls []string // "table|ns/pod|queryID" + lastStart, lastEnd time.Time + err error } func (f *fakeRunner) OrderQuery(t anomaly.Target, table string, start, end time.Time, qid string) error { f.calls = append(f.calls, table+"|"+t.Namespace+"/"+t.Pod+"|"+qid) + f.lastStart, f.lastEnd = start, end return f.err } @@ -311,3 +314,48 @@ func TestEvidenceManifest(t *testing.T) { t.Fatalf("writer error: got %d, want 502", r.StatusCode) } } + +// TestQueryWidensNarrowWindow — a control client that sends a near-zero window +// (lo≈hi, e.g. dx keying on a single finding's event_time) would capture nothing; +// the handler widens it to controlExportLookback ending at hi so the evidence +// leading up to the referral is still captured. +func TestQueryWidensNarrowWindow(t *testing.T) { + rn := &fakeRunner{} + srv := New(&fakeExporter{}, rn) + // hi = 10_000_000_000 ns, lo = hi - 512ns → a 512ns window (passes lo= %v; got %v", minControlQueryWindow, got) + } + // hi must be preserved (we widen the lower bound only). + if rn.lastEnd.UnixNano() != hi { + t.Errorf("hi must be preserved; want %d got %d", hi, rn.lastEnd.UnixNano()) + } +} + +// A comfortably-wide window is passed through unchanged (no over-widening). +func TestQueryWideWindowUnchanged(t *testing.T) { + rn := &fakeRunner{} + srv := New(&fakeExporter{}, rn) + hi := int64(1_000_000_000_000) // 1000s in ns, so lo stays positive + lo := hi - int64(120*time.Second) + resp := do(t, srv, http.MethodPost, "/query", + `{"pod":"p","namespace":"redis-demo","table":"dc_snoop","query_id":"q2","window":[`+ + itoa(lo)+`,`+itoa(hi)+`]}`) + if resp.StatusCode != http.StatusAccepted { + t.Fatalf("want 202, got %d", resp.StatusCode) + } + if got := rn.lastEnd.Sub(rn.lastStart); got != 120*time.Second { + t.Errorf("wide window must pass through unchanged; want 120s got %v", got) + } +} + +func itoa(n int64) string { return strconv.FormatInt(n, 10) } diff --git a/src/vizier/services/adaptive_export/internal/control/tls.go b/src/vizier/services/adaptive_export/internal/control/tls.go new file mode 100644 index 00000000000..cc59c8455c1 --- /dev/null +++ b/src/vizier/services/adaptive_export/internal/control/tls.go @@ -0,0 +1,128 @@ +// Copyright 2018- The Pixie Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package control + +import ( + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "math/big" + "net" + "os" + "time" +) + +// TLSConfig builds the server-side *tls.Config for the control surface. +// +// If BOTH certFile and keyFile exist and load, the mounted keypair is used +// (the shared service-tls-certs the broker/PEM already carry). Otherwise an +// ephemeral in-memory self-signed cert is generated so TLS works with zero +// extra secrets — dx skip-verifies the in-cluster cert, so a self-signed cert +// is sufficient to stop the bearer JWT crossing the CNI in cleartext. +// +// The bool return reports whether the cert was self-generated (true) vs +// loaded from disk (false), for the caller's boot log. +func TLSConfig(certFile, keyFile string, hostnames ...string) (*tls.Config, bool, error) { + if certFile != "" && keyFile != "" && fileExists(certFile) && fileExists(keyFile) { + cert, err := tls.LoadX509KeyPair(certFile, keyFile) + if err != nil { + return nil, false, fmt.Errorf("load mounted keypair %s/%s: %w", certFile, keyFile, err) + } + return &tls.Config{Certificates: []tls.Certificate{cert}, MinVersion: tls.VersionTLS12}, false, nil + } + cert, err := selfSignedCert(hostnames...) + if err != nil { + return nil, false, err + } + return &tls.Config{Certificates: []tls.Certificate{cert}, MinVersion: tls.VersionTLS12}, true, nil +} + +func fileExists(p string) bool { + fi, err := os.Stat(p) + return err == nil && !fi.IsDir() +} + +// selfSignedCert mints an ephemeral in-memory self-signed certificate: +// ECDSA P-256, 1y validity, SAN covering localhost + 127.0.0.1 + ::1 and any +// extra hostnames (the pod/node name). Nothing is written to disk; the key +// lives only in the returned tls.Certificate. +func selfSignedCert(hostnames ...string) (tls.Certificate, error) { + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return tls.Certificate{}, fmt.Errorf("generate ecdsa key: %w", err) + } + serial, err := rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) + if err != nil { + return tls.Certificate{}, fmt.Errorf("generate serial: %w", err) + } + now := time.Now() + tmpl := x509.Certificate{ + SerialNumber: serial, + Subject: pkix.Name{CommonName: "adaptive-export-control"}, + NotBefore: now.Add(-time.Hour), + NotAfter: now.AddDate(1, 0, 0), // 1y validity + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + DNSNames: []string{"localhost"}, + IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback}, + } + for _, h := range hostnames { + if h == "" { + continue + } + if ip := net.ParseIP(h); ip != nil { + tmpl.IPAddresses = append(tmpl.IPAddresses, ip) + } else { + tmpl.DNSNames = append(tmpl.DNSNames, h) + } + } + der, err := x509.CreateCertificate(rand.Reader, &tmpl, &tmpl, &key.PublicKey, key) + if err != nil { + return tls.Certificate{}, fmt.Errorf("create certificate: %w", err) + } + return tls.Certificate{ + Certificate: [][]byte{der}, + PrivateKey: key, + Leaf: &tmpl, + }, nil +} + +// certToPEM renders a tls.Certificate (as produced by selfSignedCert, holding a +// single DER cert + an *ecdsa.PrivateKey) as PEM cert + PEM key bytes — the +// on-disk shape of a mounted /certs/server.{crt,key} keypair. +func certToPEM(cert tls.Certificate) ([]byte, []byte, error) { + if len(cert.Certificate) == 0 { + return nil, nil, fmt.Errorf("certToPEM: empty certificate chain") + } + certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: cert.Certificate[0]}) + ec, ok := cert.PrivateKey.(*ecdsa.PrivateKey) + if !ok { + return nil, nil, fmt.Errorf("certToPEM: private key is not *ecdsa.PrivateKey") + } + der, err := x509.MarshalECPrivateKey(ec) + if err != nil { + return nil, nil, fmt.Errorf("marshal ec private key: %w", err) + } + keyPEM := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: der}) + return certPEM, keyPEM, nil +} diff --git a/src/vizier/services/adaptive_export/internal/control/tls_test.go b/src/vizier/services/adaptive_export/internal/control/tls_test.go new file mode 100644 index 00000000000..e6f53680c83 --- /dev/null +++ b/src/vizier/services/adaptive_export/internal/control/tls_test.go @@ -0,0 +1,194 @@ +// Copyright 2018- The Pixie Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package control + +import ( + "crypto/tls" + "net" + "net/http" + "os" + "path/filepath" + "strings" + "testing" + "time" + + jwtutils "px.dev/pixie/src/shared/services/utils" +) + +// serveTLS starts the control server over TLS on 127.0.0.1:0 using the given +// *tls.Config and returns the base https URL + a shutdown func. +func serveTLS(t *testing.T, cfg *tls.Config, srv *Server) (string, func()) { + t.Helper() + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + httpSrv := &http.Server{Handler: srv.Handler(), TLSConfig: cfg} + go func() { _ = httpSrv.ServeTLS(ln, "", "") }() + return "https://" + ln.Addr().String(), func() { _ = httpSrv.Close() } +} + +func skipVerifyClient() *http.Client { + return &http.Client{ + Timeout: 3 * time.Second, + Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}, //nolint:gosec // test: dx skip-verifies the in-cluster self-signed cert + } +} + +// TestTLSConfigSelfSigned: with no mounted cert files, TLSConfig self-generates +// an in-memory cert (the default secure path when /certs is absent). +func TestTLSConfigSelfSigned(t *testing.T) { + cfg, selfSigned, err := TLSConfig("/no/such/cert.crt", "/no/such/key.key", "some-pod") + if err != nil { + t.Fatalf("TLSConfig self-gen: %v", err) + } + if !selfSigned { + t.Fatal("expected selfSigned=true when cert files are absent") + } + if cfg == nil || len(cfg.Certificates) != 1 { + t.Fatalf("expected exactly one in-memory certificate, got %+v", cfg) + } +} + +// TestTLSServesHealthz: the server serves TLS by default (self-gen path) and a +// TLS client can reach /healthz. This is T1's "no cleartext by default". +func TestTLSServesHealthz(t *testing.T) { + cfg, selfSigned, err := TLSConfig("", "", "localhost") + if err != nil { + t.Fatalf("TLSConfig: %v", err) + } + if !selfSigned { + t.Fatal("expected self-signed cert") + } + base, stop := serveTLS(t, cfg, New(&fakeExporter{}, nil)) + defer stop() + + resp, err := skipVerifyClient().Get(base + "/healthz") + if err != nil { + t.Fatalf("TLS GET /healthz: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("healthz over TLS = %d, want 200", resp.StatusCode) + } +} + +// TestTLSRejectsUnauthenticated: over TLS with a signing key configured, an +// unauthenticated control request is rejected (401). This is T1's "requires +// the bearer JWT when a signing key is present" — verified end-to-end on the +// real TLS listener, not just the handler. +func TestTLSRejectsUnauthenticated(t *testing.T) { + const key = "0123456789abcdef0123456789abcdef" + srv := New(&fakeExporter{}, nil) + srv.SetAuth(key, "vizier") + + cfg, _, err := TLSConfig("", "", "localhost") + if err != nil { + t.Fatalf("TLSConfig: %v", err) + } + base, stop := serveTLS(t, cfg, srv) + defer stop() + client := skipVerifyClient() + + // No bearer → 401. + resp, err := client.Post(base+"/export/start", "application/json", strings.NewReader(`{"pod":"p","t_end":1}`)) + if err != nil { + t.Fatalf("TLS POST: %v", err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusUnauthorized { + t.Fatalf("unauthenticated over TLS = %d, want 401", resp.StatusCode) + } + + // Valid bearer → not 401. + good, err := jwtutils.SignJWTClaims(jwtutils.GenerateJWTForService("dx", "vizier"), key) + if err != nil { + t.Fatalf("mint token: %v", err) + } + req, _ := http.NewRequest(http.MethodPost, base+"/export/start", strings.NewReader(`{"namespace":"n","pod":"p","t_end":1}`)) + req.Header.Set("Authorization", "Bearer "+good) + resp2, err := client.Do(req) + if err != nil { + t.Fatalf("TLS POST authed: %v", err) + } + resp2.Body.Close() + if resp2.StatusCode == http.StatusUnauthorized { + t.Fatal("valid bearer wrongly rejected over TLS") + } +} + +// TestTLSConfigMountedCert: when cert+key files exist, TLSConfig loads them +// (selfSigned=false) — the /certs/server.{crt,key} shared-cert path. +func TestTLSConfigMountedCert(t *testing.T) { + dir := t.TempDir() + certPath := filepath.Join(dir, "server.crt") + keyPath := filepath.Join(dir, "server.key") + writePEMKeypair(t, certPath, keyPath) + + cfg, selfSigned, err := TLSConfig(certPath, keyPath, "localhost") + if err != nil { + t.Fatalf("TLSConfig mounted: %v", err) + } + if selfSigned { + t.Fatal("expected selfSigned=false when cert files exist") + } + if cfg == nil || len(cfg.Certificates) != 1 { + t.Fatalf("expected one loaded certificate, got %+v", cfg) + } +} + +// TestPlaintextPathServes: the CONTROL_INSECURE opt-out serves plain HTTP. This +// mirrors main.go's insecure branch (httpSrv.ListenAndServe with the same +// handler) — a plaintext client reaches /healthz. +func TestPlaintextPathServes(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatalf("listen: %v", err) + } + httpSrv := &http.Server{Handler: New(&fakeExporter{}, nil).Handler()} + go func() { _ = httpSrv.Serve(ln) }() + defer httpSrv.Close() + + resp, err := (&http.Client{Timeout: 3 * time.Second}).Get("http://" + ln.Addr().String() + "/healthz") + if err != nil { + t.Fatalf("plaintext GET /healthz: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("plaintext healthz = %d, want 200", resp.StatusCode) + } +} + +// writePEMKeypair mints a self-signed cert via the same helper and writes it as +// PEM cert+key files, so the mounted-cert load path can be exercised. +func writePEMKeypair(t *testing.T, certPath, keyPath string) { + t.Helper() + cert, err := selfSignedCert("localhost") + if err != nil { + t.Fatalf("selfSignedCert: %v", err) + } + certPEM, keyPEM, err := certToPEM(cert) + if err != nil { + t.Fatalf("certToPEM: %v", err) + } + if err := os.WriteFile(certPath, certPEM, 0o600); err != nil { + t.Fatalf("write cert: %v", err) + } + if err := os.WriteFile(keyPath, keyPEM, 0o600); err != nil { + t.Fatalf("write key: %v", err) + } +} diff --git a/src/vizier/services/adaptive_export/internal/controller/BUILD.bazel b/src/vizier/services/adaptive_export/internal/controller/BUILD.bazel index 6024b7ce5c7..dc4bdca23b1 100644 --- a/src/vizier/services/adaptive_export/internal/controller/BUILD.bazel +++ b/src/vizier/services/adaptive_export/internal/controller/BUILD.bazel @@ -36,12 +36,14 @@ pl_go_test( name = "controller_test", srcs = [ "controller_test.go", + "order_chunk_test.go", "order_query_test.go", ], embed = [":controller"], deps = [ "//src/vizier/services/adaptive_export/internal/anomaly", "//src/vizier/services/adaptive_export/internal/kubescape", + "//src/vizier/services/adaptive_export/internal/reconcile", "//src/vizier/services/adaptive_export/internal/sink", ], ) diff --git a/src/vizier/services/adaptive_export/internal/controller/controller.go b/src/vizier/services/adaptive_export/internal/controller/controller.go index 5a4b44ec1c4..01e317359c0 100644 --- a/src/vizier/services/adaptive_export/internal/controller/controller.go +++ b/src/vizier/services/adaptive_export/internal/controller/controller.go @@ -33,7 +33,9 @@ import ( "context" "errors" "fmt" + "strings" "sync" + "sync/atomic" "time" log "github.com/sirupsen/logrus" @@ -135,6 +137,10 @@ type Config struct { // captures over overlapping windows. Defaulted to 30s in defaulted(). ExportAllFloor time.Duration + // OrderChunk is the sub-window the ordered path walks the capture window in. + // Defaulted in defaulted(); env ADAPTIVE_ORDER_CHUNK_SEC overrides. + OrderChunk time.Duration + // === Throughput-protection knobs === // // At high anomaly rates (many concurrent active hashes), the default @@ -207,9 +213,20 @@ func (c *Config) defaulted() Config { if out.ExportAllFloor == 0 { out.ExportAllFloor = 30 * time.Second } + if out.OrderChunk == 0 { + out.OrderChunk = defaultOrderChunk + } return out } +const ( + // defaultOrderChunk = the full control lookback: one query per table, subdividing + // only on timeout (pre-chunking every table 10x-amplifies queries on one PEM). + defaultOrderChunk = 600 * time.Second + // orderMinChunk is the adaptive-subdivision floor; a span this small that still fails is surfaced. + orderMinChunk = 1 * time.Second +) + // Controller is the live orchestrator. One instance per operator process. type Controller struct { trig Trigger @@ -243,8 +260,17 @@ type Controller struct { exportAllMu sync.Mutex exportAllAt map[string]time.Time // per-target floor for OrderExportAll (steer-all) + + // Consecutive transient failures on the ordered path; any success resets it. Above + // orderBreakerTrip captureSpan stops subdividing so a saturated PEM isn't flooded. + orderTimeoutStreak atomic.Int32 } +const ( + maxOrderSplitDepth = 3 // cap captureSpan recursion (≤2^depth leaves/chunk) + orderBreakerTrip = 8 // consecutive timeouts above which subdivision stops +) + // New wires a Controller. nil clock falls through to RealClock. // nil querier disables the rev-1 push path (controller will only // write attribution rows; expects cloud's retention plugin to write @@ -295,23 +321,78 @@ func (c *Controller) OrderQuery(target anomaly.Target, table string, start, end return errors.New("controller: no pixie querier (operator-side push disabled)") } now := c.clock.Now() - q, err := pxl.QueryFor(table, target, start, end, now) - if err != nil { - return err + chunk := c.cfg.OrderChunk + if chunk <= 0 { + chunk = defaultOrderChunk + } + // Walk the window in OrderChunk sub-windows; captureSpan subdivides any that time out. + var readTotal, wroteTotal int + var firstErr error + for s := start; s.Before(end); s = s.Add(chunk) { + e := s.Add(chunk) + if e.After(end) { + e = end + } + qid := fmt.Sprintf("%s:%d-%d", queryID, s.Unix(), e.Unix()) + r, w, err := c.captureSpan(target, table, s, e, qid, 0) + readTotal += r + wroteTotal += w + if err != nil && firstErr == nil { + firstErr = err + } + } + recErr := "" + if firstErr != nil { + recErr = firstErr.Error() + } + // One reconcile row per table, aggregating every chunk. + c.cfg.Rec.Record(context.Background(), reconcile.Row{ + TS: now, Mode: "ordered", Table: table, + Namespace: target.Namespace, Pod: target.Pod, + WinStart: start, WinEnd: end, + ReadCount: int64(readTotal), WroteCount: int64(wroteTotal), + WriteErr: recErr, Hostname: c.cfg.Hostname, + }) + return firstErr +} + +// captureSpan captures [start,end) for one table, subdividing a transient failure into +// half-spans down to orderMinChunk. Bounded by maxOrderSplitDepth + the breaker so a +// saturated PEM isn't stormed; overlapping retries dedupe in the ReplacingMergeTree tables. +func (c *Controller) captureSpan(target anomaly.Target, table string, start, end time.Time, queryID string, depth int) (int, int, error) { + r, w, e := c.orderQuerySlice(target, table, start, end, queryID) + if e == nil || !isRetriableSpanErr(e) || end.Sub(start) <= orderMinChunk { + return r, w, e + } + if depth >= maxOrderSplitDepth { + return r, w, e // depth-capped: don't amplify a persistently-failing span + } + if c.orderTimeoutStreak.Load() > orderBreakerTrip { + // PEM saturated (sustained timeouts) — splitting would only add load. + log.WithFields(log.Fields{"table": table, "pod": target.Pod}). + Warn("ordered capture: circuit-breaker open (PEM saturated), not subdividing") + return r, w, e + } + log.WithError(e).WithFields(log.Fields{ + "table": table, "pod": target.Pod, "span": end.Sub(start).String(), "depth": depth, + }).Warn("ordered capture: transient failure, subdividing span") + mid := start.Add(end.Sub(start) / 2) + r1, w1, e1 := c.captureSpan(target, table, start, mid, queryID+".l", depth+1) + r2, w2, e2 := c.captureSpan(target, table, mid, end, queryID+".r", depth+1) + if e1 != nil { + return r1 + r2, w1 + w2, e1 + } + return r1 + r2, w1 + w2, e2 +} + +// orderQuerySlice runs one (target, table, [start,end)) capture and writes the rows. +// It records no reconcile row — the OrderQuery driver aggregates and records once. +func (c *Controller) orderQuerySlice(target anomaly.Target, table string, start, end time.Time, queryID string) (int, int, error) { + now := c.clock.Now() + q, qerr := pxl.QueryFor(table, target, start, end, now) + if qerr != nil { + return 0, 0, qerr } - // Background ctx with per-op timeouts mirroring pushPixieRows: a control-ordered - // capture must complete independently of any anomaly window's lifecycle. - var readCount, wroteCount int - var recErr string - defer func() { - c.cfg.Rec.Record(context.Background(), reconcile.Row{ - TS: now, Mode: "ordered", Table: table, - Namespace: target.Namespace, Pod: target.Pod, - WinStart: start, WinEnd: end, - ReadCount: int64(readCount), WroteCount: int64(wroteCount), - WriteErr: recErr, Hostname: c.cfg.Hostname, - }) - }() if c.globalSem != nil { c.globalSem <- struct{}{} defer func() { <-c.globalSem }() @@ -320,25 +401,46 @@ func (c *Controller) OrderQuery(target anomaly.Target, table string, start, end rows, qerr := c.querier.Query(qctx, q) cancel() if qerr != nil { - recErr = qerr.Error() - return qerr + if isRetriableSpanErr(qerr) { + c.orderTimeoutStreak.Add(1) // feed the saturation breaker + } + return 0, 0, qerr } - readCount = len(rows) + c.orderTimeoutStreak.Store(0) // a completed query clears the breaker if len(rows) == 0 { - return nil // nothing to persist; the read/0-wrote reconcile row still records it + return 0, 0, nil } wctx, wcancel := context.WithTimeout(context.Background(), 60*time.Second) werr := c.sink.WritePixieRows(wctx, table, rows) wcancel() if werr != nil { - recErr = werr.Error() - return werr + return len(rows), 0, werr } - wroteCount = len(rows) log.WithFields(log.Fields{ "table": table, "rows": len(rows), "pod": target.Pod, "query_id": queryID, }).Info("ordered pixie rows written to forensic_db (dx→AE /query)") - return nil + return len(rows), len(rows), nil +} + +// isRetriableSpanErr reports whether an error is a transient timeout/overload (vs. a +// structural error like a missing table) — covers ctx deadlines and gRPC status strings. +func isRetriableSpanErr(err error) bool { + if err == nil { + return false + } + if errors.Is(err, context.DeadlineExceeded) { + return true + } + s := strings.ToLower(err.Error()) + for _, m := range []string{ + "deadline", "timeout", "exceeded", "resourceexhausted", + "resource exhausted", "unavailable", "context canceled", "context cancelled", + } { + if strings.Contains(s, m) { + return true + } + } + return false } // OrderExportAll runs a one-shot OrderQuery for EVERY configured pixie table for diff --git a/src/vizier/services/adaptive_export/internal/controller/order_chunk_test.go b/src/vizier/services/adaptive_export/internal/controller/order_chunk_test.go new file mode 100644 index 00000000000..d9d0acb4921 --- /dev/null +++ b/src/vizier/services/adaptive_export/internal/controller/order_chunk_test.go @@ -0,0 +1,223 @@ +/* + * Copyright 2018- The Pixie Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +package controller + +// Tests for the CHUNKED + adaptively-subdividing ordered capture path — the +// durable fix for heavy tables (dc_snoop) losing the per-query deadline race under +// the OrderExportAll fan-out. Each chunk is a both-sides bounded pixie query; a +// chunk that still times out under contention is halved down to orderMinChunk. + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "px.dev/pixie/src/vizier/services/adaptive_export/internal/reconcile" +) + +// countingQuerier counts Query calls and can fail the first failN calls with a +// configurable error (to exercise adaptive subdivision) or fail every call. +type countingQuerier struct { + mu sync.Mutex + calls int + rows []map[string]any + failN int // fail the first failN calls, then succeed + failAll bool // fail every call + failErr error // error to return on a failed call +} + +func (q *countingQuerier) Query(context.Context, string) ([]map[string]any, error) { + q.mu.Lock() + defer q.mu.Unlock() + q.calls++ + if q.failAll || q.calls <= q.failN { + return nil, q.failErr + } + return q.rows, nil +} + +func (q *countingQuerier) callCount() int { + q.mu.Lock() + defer q.mu.Unlock() + return q.calls +} + +// recordingRec captures every reconcile Row so a test can assert the ordered path +// records exactly ONE aggregated row per table (not one per chunk). +type recordingRec struct { + mu sync.Mutex + rows []reconcile.Row +} + +func (r *recordingRec) Record(_ context.Context, row reconcile.Row) { + r.mu.Lock() + defer r.mu.Unlock() + r.rows = append(r.rows, row) +} + +func chunkCtl(snk Sink, q PixieQuerier, rec reconcile.Recorder, chunk time.Duration) *Controller { + cfg := defaultCfg() + cfg.OrderChunk = chunk + cfg.Rec = rec + c := New(newFakeTrigger(), snk, cfg, &fakeClock{t: canonicalEventTime}) + if q != nil { + c = c.WithPixieQuerier(q) + } + return c +} + +var errDeadline = errors.New("rpc error: code = DeadlineExceeded desc = context deadline exceeded") + +// A wide window is walked in OrderChunk-sized slices: one pixie query per chunk, +// each writing its rows. 180s window / 60s chunk = 3 bounded queries. +func TestOrderQueryChunksWideWindow(t *testing.T) { + snk := newRecordingSink() + q := &countingQuerier{rows: []map[string]any{{"comm": "whoami"}}} + end := canonicalEventTime + start := end.Add(-180 * time.Second) + if err := chunkCtl(snk, q, reconcile.Nop{}, 60*time.Second). + OrderQuery(oqTarget, "dc_snoop", start, end, "qid-w"); err != nil { + t.Fatalf("OrderQuery: %v", err) + } + if got := q.callCount(); got != 3 { + t.Errorf("want 3 chunk queries for a 180s/60s window, got %d", got) + } + if got := snk.count("dc_snoop"); got != 3 { + t.Errorf("want 3 rows written (one per chunk), got %d", got) + } +} + +// The ordered path records exactly ONE reconcile row per table, aggregating the +// per-chunk read/wrote counts — a forensic dump reads per-table, not per-chunk. +func TestOrderQuerySingleReconcileRowPerTable(t *testing.T) { + snk := newRecordingSink() + rec := &recordingRec{} + q := &countingQuerier{rows: []map[string]any{{"comm": "cat"}}} + end := canonicalEventTime + start := end.Add(-120 * time.Second) // 2 chunks + if err := chunkCtl(snk, q, rec, 60*time.Second). + OrderQuery(oqTarget, "dc_snoop", start, end, "qid-r"); err != nil { + t.Fatalf("OrderQuery: %v", err) + } + if len(rec.rows) != 1 { + t.Fatalf("want 1 aggregated reconcile row, got %d", len(rec.rows)) + } + if rec.rows[0].ReadCount != 2 || rec.rows[0].WroteCount != 2 { + t.Errorf("want aggregated read=2 wrote=2 across chunks, got read=%d wrote=%d", + rec.rows[0].ReadCount, rec.rows[0].WroteCount) + } + if rec.rows[0].WriteErr != "" { + t.Errorf("clean capture must record no error, got %q", rec.rows[0].WriteErr) + } +} + +// A chunk that fails with a TRANSIENT (deadline) error is retried as narrower +// half-spans and recovers — the flaky-capture fix. The querier fails only its first +// call, so the initial full-chunk query subdivides and the halves succeed. +func TestCaptureSpanSubdividesOnTransientError(t *testing.T) { + snk := newRecordingSink() + q := &countingQuerier{rows: []map[string]any{{"comm": "getent"}}, failN: 1, failErr: errDeadline} + end := canonicalEventTime + start := end.Add(-8 * time.Second) // single 60s chunk covers it → one initial query + if err := chunkCtl(snk, q, reconcile.Nop{}, 60*time.Second). + OrderQuery(oqTarget, "dc_snoop", start, end, "qid-t"); err != nil { + t.Fatalf("transient failure must recover via subdivision, got %v", err) + } + // call 1 (8s span) fails → split into two 4s halves (calls 2 & 3), both succeed. + if got := q.callCount(); got != 3 { + t.Errorf("want 3 calls (1 failed + 2 half-span retries), got %d", got) + } + if got := snk.count("dc_snoop"); got != 2 { + t.Errorf("want 2 half-span writes after subdivision, got %d", got) + } +} + +// A NON-transient error (e.g. a missing dark-vector table) surfaces immediately — +// no wasteful subdivision. Exactly one query per chunk, error returned. +func TestCaptureSpanDoesNotSplitNonTransient(t *testing.T) { + snk := newRecordingSink() + q := &countingQuerier{failAll: true, failErr: errors.New("table 'dx_bpf' not found")} + end := canonicalEventTime + start := end.Add(-30 * time.Second) // < one chunk → single chunk + err := chunkCtl(snk, q, reconcile.Nop{}, 60*time.Second). + OrderQuery(oqTarget, "dx_bpf", start, end, "qid-n") + if err == nil { + t.Fatal("non-transient error must surface") + } + if got := q.callCount(); got != 1 { + t.Errorf("non-transient error must NOT subdivide; want 1 call, got %d", got) + } +} + +// A persistently-timing-out span subdivides down to orderMinChunk and then surfaces +// the error instead of looping forever — the recursion terminates at the floor. +func TestCaptureSpanTerminatesAtMinChunk(t *testing.T) { + snk := newRecordingSink() + q := &countingQuerier{failAll: true, failErr: errDeadline} + end := canonicalEventTime + start := end.Add(-4 * time.Second) // 4s → 2s → 1s (floor), bounded call count + err := chunkCtl(snk, q, reconcile.Nop{}, 60*time.Second). + OrderQuery(oqTarget, "dc_snoop", start, end, "qid-f") + if err == nil { + t.Fatal("a span that never succeeds must ultimately surface the error") + } + // 4s→(2s,2s)→each (1s,1s): calls = 1 + 2 + 4 = 7, finite. Assert it stayed bounded. + if got := q.callCount(); got == 0 || got > 15 { + t.Errorf("subdivision must terminate at orderMinChunk with a bounded call count, got %d", got) + } +} + +// A persistently-timing-out multi-chunk window must NOT explode into a query storm. +// Without guards, 10 chunks each subdividing 60s→1s ≈ 10×64 = 640 queries against a +// saturated PEM. The depth cap (≤2^3 leaves/chunk) + circuit-breaker (stop +// subdividing after orderBreakerTrip consecutive timeouts) bound it hard. +func TestOrderQueryCircuitBreakerBoundsStorm(t *testing.T) { + snk := newRecordingSink() + q := &countingQuerier{failAll: true, failErr: errDeadline} + end := canonicalEventTime + start := end.Add(-600 * time.Second) // 10 chunks @ 60s, all time out + _ = chunkCtl(snk, q, reconcile.Nop{}, 60*time.Second). + OrderQuery(oqTarget, "dc_snoop", start, end, "qid-storm") + got := q.callCount() + if got > 60 { + t.Errorf("depth-cap + circuit-breaker must bound the storm; got %d calls (want <=60, ungrafted would be ~640)", got) + } + if got < 10 { + t.Errorf("must still attempt each of the 10 chunks at least once; got %d", got) + } +} + +// A healthy PEM (queries succeed) must NOT trip the breaker — subdivision stays +// available for genuinely-oversized windows. A querier that fails ONCE then succeeds +// still subdivides and recovers (breaker reset by the success). +func TestCircuitBreakerResetsOnSuccess(t *testing.T) { + snk := newRecordingSink() + q := &countingQuerier{rows: []map[string]any{{"comm": "cat"}}, failN: 1, failErr: errDeadline} + end := canonicalEventTime + start := end.Add(-8 * time.Second) + if err := chunkCtl(snk, q, reconcile.Nop{}, 60*time.Second). + OrderQuery(oqTarget, "dc_snoop", start, end, "qid-reset"); err != nil { + t.Fatalf("single transient failure must recover (breaker must not latch); got %v", err) + } + if snk.count("dc_snoop") < 1 { + t.Errorf("recovered subdivision must write rows; got %d", snk.count("dc_snoop")) + } +} diff --git a/src/vizier/services/adaptive_export/internal/pxl/compile.go b/src/vizier/services/adaptive_export/internal/pxl/compile.go index cdd21c5313c..06d5f79b1b3 100644 --- a/src/vizier/services/adaptive_export/internal/pxl/compile.go +++ b/src/vizier/services/adaptive_export/internal/pxl/compile.go @@ -108,12 +108,23 @@ func PodEnrichPxL(table string) string { return "proc = px.DataFrame(table='process_stats', start_time='" + darkProcStatsWindow + "')\n" + "proc.pod = proc.ctx['pod']\n" + "proc.namespace = proc.ctx['namespace']\n" + + // node resolved from the SAME process_stats upid the pod/ns come from, so + // dark-vector rows (dc_snoop et al.) carry hostname and become px-readable + // (#136). Transient attack pids that miss process_stats resolve blank — + // the same accepted limitation as pod/ns above. + "proc.node = px.upid_to_node_name(proc.upid)\n" + "proc.pid = px.upid_to_pid(proc.upid)\n" + - "proc = proc.groupby(['pod', 'namespace', 'pid']).agg()\n" + - "df = df.merge(proc, how='left', left_on=['pid'], right_on=['pid'], suffixes=['', '_x'])\n" + "proc = proc.groupby(['pod', 'namespace', 'node', 'pid']).agg()\n" + + "df = df.merge(proc, how='left', left_on=['pid'], right_on=['pid'], suffixes=['', '_x'])\n" + + "df.hostname = df.node\n" } return "df.namespace = px.upid_to_namespace(df.upid)\n" + - "df.pod = px.upid_to_pod_name(df.upid)\n" + "df.pod = px.upid_to_pod_name(df.upid)\n" + + // hostname = the capture node — the leading ORDER BY column on every + // socket_tracer table. AE left it empty (only stack_trace stamped it), so + // px reads of these tables (and the #136 order-UUID views) could not filter + // by hostname and the pushdown prefix (hostname,event_time) was unusable. + "df.hostname = px.upid_to_node_name(df.upid)\n" } // Render fills a CompilePassthrough template with the precise [sliceStart, diff --git a/src/vizier/services/adaptive_export/internal/pxl/queryfor.go b/src/vizier/services/adaptive_export/internal/pxl/queryfor.go index 4f9d8d6d37c..18c877c9db7 100644 --- a/src/vizier/services/adaptive_export/internal/pxl/queryfor.go +++ b/src/vizier/services/adaptive_export/internal/pxl/queryfor.go @@ -63,21 +63,17 @@ func QueryFor(table string, t anomaly.Target, sliceStart, sliceEnd, now time.Tim var b strings.Builder b.WriteString(pxSetMaxRows) b.WriteString("import px\n") - b.WriteString("df = px.DataFrame(table='" + pixieSourceFor(table) + "', start_time='" + relStart + "')\n") + // Bound the source scan on both sides; the df.time_ < sliceEnd filter trims the exact upper bound. + dfArgs := "table='" + pixieSourceFor(table) + "', start_time='" + relStart + "'" + if relEnd := relEndBound(now, sliceEnd); relEnd != "" { + dfArgs += ", end_time='" + relEnd + "'" + } + b.WriteString("df = px.DataFrame(" + dfArgs + ")\n") b.WriteString("df = df[df.time_ >= px.int64_to_time(" + strconv.FormatInt(sliceStart.UnixNano(), 10) + ")]\n") b.WriteString("df = df[df.time_ < px.int64_to_time(" + strconv.FormatInt(sliceEnd.UnixNano(), 10) + ")]\n") - // Native tables: px.upid_to_pod_name returns "/" (carnot: - // metadata_ops.h UPIDToPodNameUDF::Exec → absl::Substitute("$0/$1", ns, name)), - // not the bare pod name. Dark-vector tracepoint tables (pid-keyed) resolve pod - // via a process_stats pid-merge instead and yield a BARE pod name (dx#126). + // px.upid_to_pod_name yields "/"; dark-vector tables resolve pod via a pid-merge (bare name). if table == "stack_trace" { - // stack_trace is the CANONICAL native continuous profiler (stack_traces.beta, - // upid-keyed — NOT a pid tracepoint, so NOT a dark-vector pid-merge). Resolve - // pod/namespace/container/hostname exactly like the export preset - // (script/presets/stack_trace.pxl) and stamp event_time = time_ so the CH - // stack_trace row is complete. df.ctx['pod'] is the NAMESPACED "/" - // key (verified live), so the pod filter is namespaced — same as the native - // upid_to_pod_name path below. + // Native profiler (stack_traces.beta): resolve pod/ns/container from ctx, stamp event_time. b.WriteString("df.namespace = df.ctx['namespace']\n") b.WriteString("df.pod = df.ctx['pod']\n") b.WriteString("df.container = df.ctx['container']\n") @@ -94,22 +90,11 @@ func QueryFor(table string, t anomaly.Target, sliceStart, sliceEnd, now time.Tim } } } else if IsDarkVector(table) { - // Dark-vector tracepoints emit a RAW kernel pid. The malignant transient - // pids an incident actually produces — an attack's whoami/cat/getent - // children — are too short-lived to land in process_stats, so their - // pod/namespace resolves BLANK; a pod (or even namespace) filter drops - // exactly the evidence, which is why the dark tables came back empty. - // The AE is node-local (pem-direct → the node's own PEM), so the query is - // already scoped to the alert's node. - // - // ORDER MATTERS: drop the infra/self comms FIRST (env-driven, no recompile), - // THEN do the process_stats pid-merge. The node's dark stream is huge - // (Formatter/vector/runc/... thousands of rows per window); merging every - // one against process_stats is the query that timed out and silently - // dropped dc_snoop. Filtering comm first shrinks the merge to the handful - // of workload rows (bash/redis/whoami/cat), so the dark capture completes. + // Node-scoped (transient attack pids resolve blank ns, so no pod filter). Drop + // own-stack comms before the pid-merge to keep it cheap, then drop infra namespaces. b.WriteString(darkCommExclusion(table)) b.WriteString(PodEnrichPxL(table)) + b.WriteString(darkNamespaceExclusion()) } else { b.WriteString(PodEnrichPxL(table)) if t.Namespace != "" { @@ -130,11 +115,16 @@ func QueryFor(table string, t anomaly.Target, sliceStart, sliceEnd, now time.Tim return b.String(), nil } -// pixieSourceFor returns the Pixie table a builtin is sourced FROM when it -// differs from the ClickHouse table it is written TO. stack_trace is written to -// CH as 'stack_trace' but sourced from the CANONICAL native continuous profiler -// 'stack_traces.beta' — the always-on Pixie profiler, NOT an AE-invented table. -// (Dotted-name DataFrames compile fine in a direct query; verified live.) +// relEndBound returns a relative end_time ("-s"), or "" when sliceEnd is at/after now. +func relEndBound(now, sliceEnd time.Time) string { + gap := now.Sub(sliceEnd) + if gap < time.Second { + return "" // at/after now → default end_time (scan to now) + } + return "-" + strconv.FormatInt(int64(gap/time.Second), 10) + "s" +} + +// pixieSourceFor maps a CH table to the pixie table it's read from (stack_trace ← stack_traces.beta). func pixieSourceFor(table string) string { if table == "stack_trace" { return "stack_traces.beta" @@ -142,18 +132,15 @@ func pixieSourceFor(table string) string { return table } -// darkVectorHasComm lists the dark-vector tables that carry a `comm` column, so -// the infra-comm exclusion only emits for those (stack_trace is upid-only). +// Dark-vector tables carrying a comm column (so the comm exclusion applies). var darkVectorHasComm = map[string]bool{ "dc_snoop": true, "creds_change": true, "dx_vfs_events": true, "dx_unlink": true, "dx_dlookup": true, "dx_mprotect": true, "dx_bpf": true, "dx_ptrace": true, } -// darkExcludeCommsDefault is the node's own infra/self comms dropped from the -// node-scoped dark capture so the workload's activity stands out. Overridable at -// runtime via DC_SNOOP_EXCLUDE_COMMS (csv) — a process can be added without a -// recompile. Kept in sync with script.presets defaultExcludeComms. +// Own-stack + node/system comms dropped from the node-scoped dark capture; workload +// comms (redis-*, etc.) are never listed. Override via DC_SNOOP_EXCLUDE_COMMS (csv). var darkExcludeCommsDefault = []string{ "pem", "kelvin", "containerd", "containerd-shim", "runc", "node-agent", "runc:[2:INIT]", "runc:[1:CHILD]", @@ -165,10 +152,26 @@ var darkExcludeCommsDefault = []string{ "ConfigReloader", "clickhouse-oper", "Formatter", "(setup.sh)", "cmd", "vector-worker", "metrics-server", "local-path-prov", "portmap", "(udev-worker)", "systemd-resolve", "systemd-timesyn", + "systemd-udevd", "systemd-sysctl", "host-local", "bridge", "flannel", + "loopback", "bandwidth", "dbus-daemon", "mount", "umount", "tailscaled", + "grpc_health_pro", "kubevuln", "opm", "(spawn)", "kube-proxy", + "pause", "systemd-logind", +} + +// Kernel-thread families whose names carry a variable suffix (kworker/u8:3) that +// exact match misses; dropped via px.contains. +var darkExcludeCommSubstrings = []string{ + "kworker", "ksoftirqd", "migration", "rcu_", "kthreadd", "kdevtmpfs", + "kcompactd", "khugepaged", "kswapd", "watchdog", "cpuhp", "ksmd", "irq/", +} + +// Infra namespaces dropped from the node-scoped dark capture. Blank-namespace rows +// (transient attack children) survive. Override via DC_SNOOP_EXCLUDE_NAMESPACES. +var darkExcludeNamespacesDefault = []string{ + "pl", "honey", "px-operator", "olm", "clickhouse", "socdemo", "socdemo-ch", + "kube-system", "kube-public", "kube-node-lease", "local-path-storage", } -// darkCommExclusion builds the infra-comm drop filter for a dark-vector table -// that has a comm column. Returns "" for comm-less tables (stack_trace). func darkCommExclusion(table string) string { if !darkVectorHasComm[table] { return "" @@ -186,6 +189,26 @@ func darkCommExclusion(table string) string { for _, c := range comms { b.WriteString("df = df[df.comm != '" + escapePxL(c) + "']\n") } + for _, s := range darkExcludeCommSubstrings { + b.WriteString("df = df[px.logicalNot(px.contains(df.comm, '" + escapePxL(s) + "'))]\n") + } + return b.String() +} + +func darkNamespaceExclusion() string { + nss := darkExcludeNamespacesDefault + if v := strings.TrimSpace(os.Getenv("DC_SNOOP_EXCLUDE_NAMESPACES")); v != "" { + nss = nil + for _, s := range strings.Split(v, ",") { + if s = strings.TrimSpace(s); s != "" { + nss = append(nss, s) + } + } + } + var b strings.Builder + for _, ns := range nss { + b.WriteString("df = df[df.namespace != '" + escapePxL(ns) + "']\n") + } return b.String() } diff --git a/src/vizier/services/adaptive_export/internal/pxl/queryfor_test.go b/src/vizier/services/adaptive_export/internal/pxl/queryfor_test.go index 562ea794cc0..6ca0023fd61 100644 --- a/src/vizier/services/adaptive_export/internal/pxl/queryfor_test.go +++ b/src/vizier/services/adaptive_export/internal/pxl/queryfor_test.go @@ -18,6 +18,7 @@ package pxl import ( "errors" + "strconv" "strings" "testing" "time" @@ -257,14 +258,15 @@ func TestEscapePxL_TableDriven(t *testing.T) { // df = df[df.time_ < ...] 1 // df.namespace = px.upid_to_namespace(...) 1 // df.pod = px.upid_to_pod_name(...) 1 +// df.hostname = px.upid_to_node_name(...) 1 // df = df[df.namespace == '...'] 1 // df = df[df.pod == '...'] 1 // px.display(df, '...') 1 -// (trailing newline → empty 11th split) 1 +// (trailing newline → empty 12th split) 1 // -// Total: 10 statements + trailing empty == strings.Split == 11 entries. +// Total: 11 statements + trailing empty == strings.Split == 12 entries. func TestQueryFor_RejectsInjectionInTargetFields(t *testing.T) { - const wantLines = 11 + const wantLines = 12 cases := []struct { name string @@ -336,7 +338,105 @@ func TestQueryFor_PodOnlyRegexEscapesQuoteMetaInjection(t *testing.T) { if err != nil { t.Fatalf("QueryFor: %v", err) } - if strings.Contains(q, "exec(") || strings.Count(q, "\n") > 9 { + if strings.Contains(q, "exec(") || strings.Count(q, "\n") > 10 { t.Fatalf("pod-only path injection succeeded:\n%s", q) } } + +// TestQueryFor_EndTimeBoundsPastWindow — a window whose upper bound is in the past +// must emit a relative end_time so the PEM scan is bounded on BOTH sides (not +// [sliceStart, now]). The precise upper bound is still enforced by the df.time_ < +// nanos post-filter. +func TestQueryFor_EndTimeBoundsPastWindow(t *testing.T) { + // sliceEnd 2 minutes before now → end_time must appear. + end := fixedNow.Add(-2 * time.Minute) + start := fixedNow.Add(-7 * time.Minute) + q, err := QueryFor("dc_snoop", target, start, end, fixedNow) + if err != nil { + t.Fatalf("QueryFor: %v", err) + } + if !strings.Contains(q, "end_time='-120s'") { + t.Fatalf("past-window query must bound the source scan with end_time='-120s'; got:\n%s", q) + } + // exact upper bound still trimmed precisely in nanos. + if !strings.Contains(q, "df = df[df.time_ < px.int64_to_time("+ + strconv.FormatInt(end.UnixNano(), 10)+")]") { + t.Fatalf("precise nanos upper-bound filter must remain; got:\n%s", q) + } +} + +// TestQueryFor_NoEndTimeAtLiveEdge — a window that reaches now must NOT emit +// end_time (scan to the live edge), preserving the pre-chunking behavior for the +// most-recent slice. +func TestQueryFor_NoEndTimeAtLiveEdge(t *testing.T) { + q, err := QueryFor("dc_snoop", target, fixedNow.Add(-1*time.Minute), fixedNow, fixedNow) + if err != nil { + t.Fatalf("QueryFor: %v", err) + } + if strings.Contains(q, "end_time=") { + t.Fatalf("live-edge window must not bound end_time; got:\n%s", q) + } +} + +// TestQueryFor_DarkNamespaceExclusion — the node-scoped dark capture (dc_snoop) +// must drop infra namespaces (pl, kube-system, …) while KEEPING blank-namespace +// transient rows (the attack's short-lived children). Mirrors the shipped preset. +func TestQueryFor_DarkNamespaceExclusion(t *testing.T) { + q, err := QueryFor("dc_snoop", target, fixedStart, fixedEnd, fixedNow) + if err != nil { + t.Fatalf("QueryFor: %v", err) + } + // namespace drops present for infra + for _, ns := range []string{"pl", "kube-system", "clickhouse"} { + if !strings.Contains(q, "df = df[df.namespace != '"+ns+"']") { + t.Errorf("dark capture must drop infra namespace %q; got:\n%s", ns, q) + } + } + // must NOT pin to the alert pod's namespace (node-scoped keeps blank + other workloads) + if strings.Contains(q, "df = df[df.namespace == '") { + t.Errorf("dark capture must not pin df.namespace ==; got:\n%s", q) + } + // host/CNI comm drops present + for _, c := range []string{"host-local", "systemd-udevd", "tailscaled", "kubevuln"} { + if !strings.Contains(q, "df = df[df.comm != '"+c+"']") { + t.Errorf("dark capture must drop host/CNI comm %q; got:\n%s", c, q) + } + } +} + +// TestQueryFor_DarkNamespaceExclusion_EnvOverride — DC_SNOOP_EXCLUDE_NAMESPACES +// replaces the default list. +func TestQueryFor_DarkNamespaceExclusion_EnvOverride(t *testing.T) { + t.Setenv("DC_SNOOP_EXCLUDE_NAMESPACES", "foo,bar") + q, err := QueryFor("dc_snoop", target, fixedStart, fixedEnd, fixedNow) + if err != nil { + t.Fatalf("QueryFor: %v", err) + } + if !strings.Contains(q, "df = df[df.namespace != 'foo']") || !strings.Contains(q, "df = df[df.namespace != 'bar']") { + t.Errorf("env override must emit foo/bar drops; got:\n%s", q) + } + if strings.Contains(q, "df = df[df.namespace != 'pl']") { + t.Errorf("env override must REPLACE the default (no 'pl'); got:\n%s", q) + } +} + +// dc_snoop drops kernel-thread families (variable suffix) via px.logicalNot(px.contains), +// keeps workload comms (redis-server), and doesn't pin the alert pod's namespace. +func TestQueryFor_DarkCommSubstringExclusion(t *testing.T) { + q, err := QueryFor("dc_snoop", target, fixedStart, fixedEnd, fixedNow) + if err != nil { + t.Fatalf("QueryFor: %v", err) + } + for _, sub := range []string{"kworker", "ksoftirqd", "rcu_"} { + want := "df = df[px.logicalNot(px.contains(df.comm, '" + sub + "'))]" + if !strings.Contains(q, want) { + t.Errorf("want kernel-thread drop %q; got:\n%s", want, q) + } + } + if strings.Contains(q, "df.comm != 'redis-server'") || strings.Contains(q, "df.comm, 'redis") { + t.Errorf("workload comm redis-* must NOT be excluded; got:\n%s", q) + } + if !strings.Contains(q, "df = df[df.comm != 'pause']") { + t.Errorf("want exact drop of 'pause'; got:\n%s", q) + } +} diff --git a/src/vizier/services/adaptive_export/internal/trigger/BUILD.bazel b/src/vizier/services/adaptive_export/internal/trigger/BUILD.bazel index 0445d9211f4..8ffeb29b8e4 100644 --- a/src/vizier/services/adaptive_export/internal/trigger/BUILD.bazel +++ b/src/vizier/services/adaptive_export/internal/trigger/BUILD.bazel @@ -21,6 +21,8 @@ go_library( name = "trigger", srcs = [ "clickhouse.go", + "dedup.go", + "metrics.go", "watermark.go", ], importpath = "px.dev/pixie/src/vizier/services/adaptive_export/internal/trigger", @@ -28,6 +30,8 @@ go_library( deps = [ "//src/vizier/services/adaptive_export/internal/chhttp", "//src/vizier/services/adaptive_export/internal/kubescape", + "@com_github_prometheus_client_golang//prometheus", + "@com_github_prometheus_client_golang//prometheus/promauto", "@com_github_sirupsen_logrus//:logrus", ], ) @@ -37,10 +41,15 @@ pl_go_test( srcs = [ "clickhouse_internal_test.go", "clickhouse_test.go", + "dedup_test.go", "fingerprint_bench_test.go", + "lookback_test.go", "oracle_test.go", "watermark_test.go", ], embed = [":trigger"], - deps = ["//src/vizier/services/adaptive_export/internal/kubescape"], + deps = [ + "//src/vizier/services/adaptive_export/internal/kubescape", + "@com_github_prometheus_client_golang//prometheus/testutil", + ], ) diff --git a/src/vizier/services/adaptive_export/internal/trigger/clickhouse.go b/src/vizier/services/adaptive_export/internal/trigger/clickhouse.go index 80e03f3b942..1f548e37cc2 100644 --- a/src/vizier/services/adaptive_export/internal/trigger/clickhouse.go +++ b/src/vizier/services/adaptive_export/internal/trigger/clickhouse.go @@ -84,13 +84,48 @@ type Config struct { // hardcoded to 5s, which under any backlog caused every poll to // time out mid-stream → watermark never advanced. HTTPTimeout time.Duration + + // Lookback (#97 / F8 / AE-9): when > 0, each poll re-scans + // [watermark-Lookback, ∞) instead of the strict [watermark, ∞) and + // dedupes re-seen rows by content fingerprint, so an out-of-order / + // clock-skewed / restart-buried row that lands within the window is + // still processed EXACTLY ONCE (no drop, no duplicate). Rows below + // watermark-Lookback stay dropped — the documented bound. 0 keeps + // the legacy strict high-water-mark behavior (anything below the + // watermark is dropped forever). Production default is 300s via + // ADAPTIVE_TRIGGER_LOOKBACK_SEC in cmd/main.go; the zero value here + // is legacy so existing callers/tests are unchanged. + Lookback time.Duration + + // MaxSkew is the wall-clock poison clamp (#97): a row whose + // NORMALIZED event_time is more than MaxSkew past now is still + // emitted once, but never advances the watermark, so a single + // corrupted/oversized timestamp (the 1.78e18 leftover of loadtest + // E8) cannot jump the cursor past all real data and silently halt + // the trigger. Also applied to the persisted watermark at load, so + // an ALREADY-poisoned cursor self-recovers on restart without the + // manual `ALTER TABLE trigger_watermark DELETE`. <=0 → 1h. + MaxSkew time.Duration + + // DedupMaxEntries caps the lookback dedup set (memory bound). An + // in-window fingerprint evicted by capacity may re-emit once, so + // size it >= the max rows expected per lookback window. + // <=0 → 4*PollLimit. + DedupMaxEntries int } +// defaultMaxSkew is the default wall-clock poison-clamp bound (#97): +// an event_time more than this far in the future is implausible. +const defaultMaxSkew = time.Hour + // ClickHouseHTTP polls forensic_db.
over the ClickHouse HTTP // interface, scoped to a single node. type ClickHouseHTTP struct { cfg Config client *http.Client + // now is the wall clock used by the poison clamp (#97). + // Injectable for deterministic tests; time.Now in production. + now func() time.Time } // New validates Config and returns a ready trigger. @@ -143,9 +178,19 @@ func New(cfg Config) (*ClickHouseHTTP, error) { if cfg.HTTPTimeout <= 0 { cfg.HTTPTimeout = 30 * time.Second } + if cfg.Lookback < 0 { + return nil, fmt.Errorf("trigger: Lookback must be >= 0 (got %v)", cfg.Lookback) + } + if cfg.MaxSkew <= 0 { + cfg.MaxSkew = defaultMaxSkew + } + if cfg.DedupMaxEntries <= 0 { + cfg.DedupMaxEntries = 4 * cfg.PollLimit + } return &ClickHouseHTTP{ cfg: cfg, client: &http.Client{Timeout: cfg.HTTPTimeout}, + now: time.Now, }, nil } @@ -169,11 +214,15 @@ func (t *ClickHouseHTTP) Subscribe(ctx context.Context) (<-chan kubescape.Event, func (t *ClickHouseHTTP) run(ctx context.Context, out chan<- kubescape.Event) { defer close(out) // Watermark uses event_time as the cursor PLUS a set of row - // fingerprints already pushed at that exact event_time. This - // closes the race where two kubescape rows share the same - // event_time but the second arrives after our previous poll: the - // query is `event_time >= watermark` (inclusive) and we skip rows - // whose fingerprint we have already seen at the boundary. + // fingerprints already pushed. In legacy strict mode (Lookback==0) + // the query is `event_time >= watermark` (inclusive) and the + // fingerprint set covers only the exact boundary event_time — + // closing the race where two kubescape rows share the same + // event_time but the second arrives after our previous poll. With + // a bounded lookback (#97, the F8/AE-9 fix) the query starts at + // max(0, watermark-Lookback) and the fingerprint set is a bounded + // LRU over the whole re-scanned window, so out-of-order / skewed / + // restart-buried rows inside the window are captured exactly once. // // Cold-start order: persistent store > InitialWatermark > 0. // The persistent store is the production answer to "operator @@ -203,7 +252,39 @@ func (t *ClickHouseHTTP) run(ctx context.Context, out chan<- kubescape.Event) { // pre-fix persisted seconds watermark (or a non-seconds InitialWatermark) // is interpreted on the same scale as chNormEventTimeNanos in the SQL. watermark = normalizeEventTimeNanos(watermark) + maxSkewNS := uint64(t.cfg.MaxSkew.Nanoseconds()) + lookbackNS := uint64(t.cfg.Lookback.Nanoseconds()) + // Self-recovery from an ALREADY-poisoned persisted cursor (#97 T1): + // a pre-fix deployment could have persisted a far-future watermark + // (loadtest E8's leftover 1.78e18-style value). Clamp it to + // wall-clock so fresh rows flow again on restart WITHOUT the manual + // `ALTER TABLE trigger_watermark DELETE WHERE 1=1` + redeploy. + if nowNS := uint64(t.now().UnixNano()); watermark > nowNS+maxSkewNS { + log.WithFields(log.Fields{"watermark": watermark, "clamped_to": nowNS}). + Warn("trigger: persisted watermark is implausibly far in the future — clamping to wall-clock (poison recovery, #97)") + watermark = nowNS + } + wmGauge := metricWatermarkNS.WithLabelValues(t.cfg.Table, t.cfg.Hostname) + wmGauge.Set(float64(watermark)) + // Dedup state. Strict mode (Lookback==0) keeps the legacy exact + // boundary set; lookback mode dedupes the whole re-scanned window + // with a bounded LRU (#97). rejectedSeen exists only in strict mode: + // a clamp-rejected row never falls below the cursor, so without a + // fingerprint record it would re-emit on every poll. seenAtBoundary := map[string]bool{} + var seenInWindow *dedupLRU + var rejectedSeen *dedupLRU + if lookbackNS > 0 { + seenInWindow = newDedupLRU(t.cfg.DedupMaxEntries) + } else { + rejectedSeen = newDedupLRU(t.cfg.DedupMaxEntries) + } + // catchup lifts a poll's lower bound above the sliding lookback + // floor while an in-window backlog is wider than PollLimit: without + // it every poll would re-fetch the same fully-deduped first + // PollLimit rows and never reach deeper into the window. Cleared as + // soon as a poll returns under capacity (back to full-window scans). + var catchup uint64 ticker := time.NewTicker(t.cfg.PollInterval) defer ticker.Stop() @@ -253,7 +334,21 @@ func (t *ClickHouseHTTP) run(ctx context.Context, out chan<- kubescape.Event) { }() pollOnce := func() { - rows, maxSeen, err := t.fetchSince(ctx, watermark) + // Bounded lookback (#97): scan from max(0, watermark-Lookback) + // so rows that landed BELOW the cursor (out-of-order, clock + // skew, restart burial) are still fetched; the dedup LRU makes + // re-seen rows exactly-once. Lookback==0 → legacy strict HWM. + queryFrom := watermark + if lookbackNS > 0 { + queryFrom = 0 + if watermark > lookbackNS { + queryFrom = watermark - lookbackNS + } + if catchup > queryFrom { + queryFrom = catchup + } + } + rows, maxFetched, err := t.fetchSince(ctx, queryFrom) // Partial-read tolerance: when the body read is cut short by // HTTP timeout / connection reset, fetchSince returns the rows // it managed to parse + err. We still process those rows so @@ -267,6 +362,19 @@ func (t *ClickHouseHTTP) run(ctx context.Context, out chan<- kubescape.Event) { log.WithError(err).WithField("partial_rows", len(rows)). Warn("trigger: poll partial — advancing on what parsed") } + // Wall-clock poison clamp (#97): any normalized event_time past + // now+MaxSkew must never advance the cursor. acceptedMax is the + // advancement target — the max normalized event_time among rows + // that PASS the clamp. With no poison rows it equals maxFetched, + // so the monotonic happy path is byte-identical to before. + skewLimit := uint64(t.now().UnixNano()) + maxSkewNS + acceptedMax := uint64(0) + for _, row := range rows { + if evn := normalizeEventTimeNanos(row.EventTime); evn <= skewLimit && evn > acceptedMax { + acceptedMax = evn + } + } + wmAtPollStart := watermark nextSeen := map[string]bool{} // Periodic in-loop save: when pollOnce is draining a large // initial backlog, the watermark advances long before the @@ -276,45 +384,122 @@ func (t *ClickHouseHTTP) run(ctx context.Context, out chan<- kubescape.Event) { // with the time-based throttle inside flushWatermark, this // produces at most one persistent INSERT per WatermarkSaveInterval. const saveEveryN = 256 - skippedAtBoundary := 0 + skippedSeen := 0 + emitted := 0 for i, row := range rows { fp := rowFingerprint(row) // Cursor comparisons are in NORMALIZED nanos (F8): the raw // event_time unit is not enforced, so compare on the same scale - // as the SQL filter (chNormEventTimeNanos) and maxSeen. + // as the SQL filter (chNormEventTimeNanos) and acceptedMax. evn := normalizeEventTimeNanos(row.EventTime) - if evn == watermark && seenAtBoundary[fp] { - skippedAtBoundary++ - continue // already pushed in a prior poll at this exact boundary + if lookbackNS > 0 { + if seenInWindow.Contains(fp) { + skippedSeen++ + continue // already pushed in a prior scan of this window + } + } else { + if evn == watermark && seenAtBoundary[fp] { + skippedSeen++ + continue // already pushed in a prior poll at this exact boundary + } + if rejectedSeen.Contains(fp) { + continue // clamp-rejected row re-fetched (it never sinks below the cursor) + } } - ev, err := kubescape.Extract(row) - if err != nil { - log.WithError(err).Debug("trigger: skip incomplete row") + poison := evn > skewLimit + ev, exErr := kubescape.Extract(row) + if exErr != nil { + log.WithError(exErr).Debug("trigger: skip incomplete row") + // Register the fingerprint anyway (lookback / poison): + // the row can never become extractable, and without a + // record it would be re-fetched + re-logged every poll + // for as long as it stays above the scan floor. + if lookbackNS > 0 { + seenInWindow.Add(fp, evn) + } else if poison { + rejectedSeen.Add(fp, evn) + } continue } - // Promote the per-row (normalized) event_time into the watermark - // immediately so flushWatermark below can persist mid-drain. - if evn > watermark { - watermark = evn - dirty = true + if poison { + // Emit the row once (it may be a real anomaly with a + // mangled timestamp) but do NOT let it advance the + // cursor: one 1.78e18 row must not jump the watermark + // past all real seconds rows (F8 halt). + metricEventTimeRejected.Inc() + log.WithFields(log.Fields{ + "event_time": row.EventTime, + "normalized": evn, + "skew_limit": skewLimit, + }).Warn("trigger: event_time beyond wall-clock skew bound — processing row WITHOUT advancing watermark (poison clamp, #97)") + } else { + if evn < wmAtPollStart { + // A row the legacy strict HWM would have dropped — + // captured via the lookback (T2). Observable proof + // the fix is doing work (T3). + metricBelowWatermark.Inc() + } + // Promote the per-row (normalized) event_time into the watermark + // immediately so flushWatermark below can persist mid-drain. + if evn > watermark { + watermark = evn + dirty = true + wmGauge.Set(float64(watermark)) + } + } + if lookbackNS > 0 { + seenInWindow.Add(fp, evn) + } else if poison { + rejectedSeen.Add(fp, evn) } select { case out <- ev: case <-ctx.Done(): return } - if evn == maxSeen { + emitted++ + if !poison && evn == acceptedMax { nextSeen[fp] = true } if i > 0 && i%saveEveryN == 0 { flushWatermark() } } - if maxSeen > watermark { - watermark = maxSeen + if lookbackNS > 0 { + if acceptedMax > watermark { + watermark = acceptedMax + dirty = true + wmGauge.Set(float64(watermark)) + } + // Paging within the window: a saturated response means the + // window holds more rows than PollLimit — lift the floor so + // the next poll pages FORWARD instead of re-fetching the + // same deduped prefix forever. + if len(rows) >= t.cfg.PollLimit { + if emitted == 0 && skippedSeen == len(rows) { + // Every row in the saturated page was already seen — + // step past the page entirely (lookback analog of the + // legacy 1ns boundary escape). + catchup = maxFetched + 1 + } else if acceptedMax > catchup { + catchup = acceptedMax + } + } else { + catchup = 0 + } + // Entries below the sliding floor can never be re-fetched; + // evict them so the LRU stays at ~window size. + floor := uint64(0) + if watermark > lookbackNS { + floor = watermark - lookbackNS + } + seenInWindow.EvictBelow(floor) + } else if acceptedMax > watermark { + watermark = acceptedMax seenAtBoundary = nextSeen dirty = true - } else if maxSeen == watermark { + wmGauge.Set(float64(watermark)) + } else if acceptedMax == watermark { // no progress this tick — preserve boundary set, optionally extend for fp := range nextSeen { seenAtBoundary[fp] = true @@ -329,10 +514,11 @@ func (t *ClickHouseHTTP) run(ctx context.Context, out chan<- kubescape.Event) { // the next poll, which is acceptable: the fingerprint dedup already // tolerates boundary overlap, and we prefer forward progress over // an infinite loop. - if skippedAtBoundary > 0 && len(nextSeen) == 0 && len(rows) >= t.cfg.PollLimit { + if skippedSeen > 0 && len(nextSeen) == 0 && len(rows) >= t.cfg.PollLimit { watermark++ seenAtBoundary = map[string]bool{} dirty = true + wmGauge.Set(float64(watermark)) log.WithField("watermark", watermark). Warn("trigger: boundary paging escape — advanced watermark by 1ns to unblock poll") } diff --git a/src/vizier/services/adaptive_export/internal/trigger/dedup.go b/src/vizier/services/adaptive_export/internal/trigger/dedup.go new file mode 100644 index 00000000000..ca1c5dda9a0 --- /dev/null +++ b/src/vizier/services/adaptive_export/internal/trigger/dedup.go @@ -0,0 +1,98 @@ +// Copyright 2018- The Pixie Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package trigger + +import "container/list" + +// dedupLRU is a bounded, insertion-ordered set of row fingerprints, +// each tagged with the row's normalized event_time (nanos). It is the +// #97 (F8/AE-9) extension of the old single-boundary `seenAtBoundary` +// map: with a bounded lookback the trigger re-fetches every row in +// [watermark-Lookback, watermark] on each poll, so dedup must cover the +// whole window, not just the exact watermark boundary. +// +// Eviction is two-fold: +// - EvictBelow(floor): entries whose event_time has slid below the +// lookback floor can never be returned by the SELECT again, so they +// are dropped eagerly to keep the set at ~window size. +// - capacity: Add evicts the OLDEST INSERTION when over max, bounding +// memory even if the window holds more rows than expected. An +// in-window entry evicted by capacity may cause one duplicate emit — +// the documented trade-off for bounded memory (size it >= the max +// rows per window; default 4*PollLimit). +// +// Not goroutine-safe; owned by the single poll loop. +type dedupLRU struct { + max int + ll *list.List // front = oldest insertion + items map[string]*list.Element +} + +type dedupEntry struct { + fp string + evn uint64 // normalized event_time (nanos) +} + +func newDedupLRU(capacity int) *dedupLRU { + if capacity <= 0 { + capacity = 1 + } + return &dedupLRU{max: capacity, ll: list.New(), items: map[string]*list.Element{}} +} + +// Contains reports whether fp was Added and not yet evicted. +func (d *dedupLRU) Contains(fp string) bool { + _, ok := d.items[fp] + return ok +} + +// Add records fp with its normalized event_time. No-op if already +// present. Evicts oldest insertions while over capacity. +func (d *dedupLRU) Add(fp string, evn uint64) { + if _, ok := d.items[fp]; ok { + return + } + d.items[fp] = d.ll.PushBack(dedupEntry{fp: fp, evn: evn}) + for d.ll.Len() > d.max { + d.removeElement(d.ll.Front()) + } +} + +// EvictBelow drops entries with evn < floor, popping from the oldest +// insertion. Insertion order tracks the poll's ORDER BY event_time, so +// in the common case this removes exactly the expired prefix. A late +// arrival (low evn inserted after a higher one) may survive behind a +// newer entry until capacity eviction — harmless: Contains on an +// expired fp only suppresses a row the SELECT can no longer return. +func (d *dedupLRU) EvictBelow(floor uint64) { + for e := d.ll.Front(); e != nil; { + if e.Value.(dedupEntry).evn >= floor { + return + } + next := e.Next() + d.removeElement(e) + e = next + } +} + +// Len returns the number of live entries. +func (d *dedupLRU) Len() int { return d.ll.Len() } + +func (d *dedupLRU) removeElement(e *list.Element) { + delete(d.items, e.Value.(dedupEntry).fp) + d.ll.Remove(e) +} diff --git a/src/vizier/services/adaptive_export/internal/trigger/dedup_test.go b/src/vizier/services/adaptive_export/internal/trigger/dedup_test.go new file mode 100644 index 00000000000..c139b21831c --- /dev/null +++ b/src/vizier/services/adaptive_export/internal/trigger/dedup_test.go @@ -0,0 +1,92 @@ +// Copyright 2018- The Pixie Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package trigger + +import "testing" + +func TestDedupLRU_AddContains(t *testing.T) { + d := newDedupLRU(8) + if d.Contains("a") { + t.Fatalf("empty LRU claims to contain a") + } + d.Add("a", 100) + d.Add("b", 200) + if !d.Contains("a") || !d.Contains("b") { + t.Fatalf("added fingerprints not found") + } + if d.Len() != 2 { + t.Fatalf("Len = %d, want 2", d.Len()) + } + // Duplicate Add is a no-op (no double-entry, no reorder). + d.Add("a", 100) + if d.Len() != 2 { + t.Fatalf("duplicate Add changed Len to %d", d.Len()) + } +} + +func TestDedupLRU_CapacityEvictsOldestInsertion(t *testing.T) { + d := newDedupLRU(3) + d.Add("a", 1) + d.Add("b", 2) + d.Add("c", 3) + d.Add("d", 4) // over capacity → "a" (oldest insertion) evicted + if d.Contains("a") { + t.Fatalf("oldest entry not evicted at capacity") + } + for _, fp := range []string{"b", "c", "d"} { + if !d.Contains(fp) { + t.Fatalf("entry %q evicted unexpectedly", fp) + } + } + if d.Len() != 3 { + t.Fatalf("Len = %d, want 3", d.Len()) + } +} + +func TestDedupLRU_EvictBelow(t *testing.T) { + d := newDedupLRU(8) + d.Add("a", 100) + d.Add("b", 200) + d.Add("c", 300) + d.EvictBelow(250) + if d.Contains("a") || d.Contains("b") { + t.Fatalf("entries below floor survived EvictBelow") + } + if !d.Contains("c") { + t.Fatalf("entry at/above floor was evicted") + } + // EvictBelow stops at the first entry >= floor (prefix semantics): + // a late arrival (low evn inserted AFTER a higher one) survives — + // documented as harmless. + d.Add("late", 50) + d.EvictBelow(250) + if !d.Contains("late") { + t.Fatalf("late-arrival entry behind a newer one should survive prefix eviction") + } +} + +func TestDedupLRU_ZeroCapacityIsSafe(t *testing.T) { + d := newDedupLRU(0) // clamped to 1 + d.Add("a", 1) + if !d.Contains("a") { + t.Fatalf("single entry not retained") + } + d.Add("b", 2) + if d.Contains("a") || !d.Contains("b") { + t.Fatalf("capacity-1 eviction wrong: a=%v b=%v", d.Contains("a"), d.Contains("b")) + } +} diff --git a/src/vizier/services/adaptive_export/internal/trigger/lookback_test.go b/src/vizier/services/adaptive_export/internal/trigger/lookback_test.go new file mode 100644 index 00000000000..8fa153001fa --- /dev/null +++ b/src/vizier/services/adaptive_export/internal/trigger/lookback_test.go @@ -0,0 +1,321 @@ +// Copyright 2018- The Pixie Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +// Bounded-lookback + wall-clock poison-clamp tests (#97 / F8 / AE-9). +// No live ClickHouse: a stub HTTP server implements the trigger's +// JSONEachRow contract INCLUDING the `>= ` watermark predicate, +// so re-poll semantics (the essence of lookback) are exercised for real. + +package trigger + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "regexp" + "sort" + "strconv" + "sync" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus/testutil" +) + +// fakeCH is a stub ClickHouse HTTP endpoint that stores rows and, like +// the real server, only returns rows whose NORMALIZED event_time is >= +// the bound parsed out of the trigger's SELECT. +type fakeCH struct { + mu sync.Mutex + rows []fakeRow + srv *httptest.Server +} + +type fakeRow struct { + eventTime uint64 // raw, unit-ambiguous — exactly like production + ruleID string + pid int +} + +var boundRE = regexp.MustCompile(`>= (\d+) ORDER`) + +func newFakeCH(t *testing.T) *fakeCH { + f := &fakeCH{} + f.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query().Get("query") + m := boundRE.FindStringSubmatch(q) + if m == nil { + t.Errorf("query without >= bound: %q", q) + w.WriteHeader(400) + return + } + bound, err := strconv.ParseUint(m[1], 10, 64) + if err != nil { + t.Errorf("unparseable bound in query %q: %v", q, err) + w.WriteHeader(400) + return + } + f.mu.Lock() + var out []fakeRow + for _, row := range f.rows { + if normalizeEventTimeNanos(row.eventTime) >= bound { + out = append(out, row) + } + } + f.mu.Unlock() + sort.Slice(out, func(i, j int) bool { + return normalizeEventTimeNanos(out[i].eventTime) < normalizeEventTimeNanos(out[j].eventTime) + }) + for _, row := range out { + fmt.Fprintf(w, + `{"RuleID":%q,"RuntimeK8sDetails":"{\"podName\":\"p-1\",\"podNamespace\":\"ns\"}","RuntimeProcessDetails":"{\"processTree\":{\"pid\":%d,\"comm\":\"c\"}}","event_time":"%d","hostname":"node-1"}`+"\n", + row.ruleID, row.pid, row.eventTime) + } + })) + return f +} + +func (f *fakeCH) add(r fakeRow) { + f.mu.Lock() + f.rows = append(f.rows, r) + f.mu.Unlock() +} + +func (f *fakeCH) close() { f.srv.Close() } + +// testBase is a fixed "now" for deterministic clamp behavior: +// 2026-05-29T… ≈ 1.7805e9 seconds. +const testBase = uint64(1_780_500_000) + +func fixedNow() time.Time { return time.Unix(int64(testBase), 0) } + +// newLookbackTrigger builds a trigger against the fake server with the +// #97 config (300s lookback) and a pinned wall clock. +func newLookbackTrigger(t *testing.T, f *fakeCH, hostname string, lookback time.Duration) *ClickHouseHTTP { + t.Helper() + tr, err := New(Config{ + Endpoint: f.srv.URL, + Hostname: hostname, + PollInterval: 20 * time.Millisecond, + Lookback: lookback, + }) + if err != nil { + t.Fatalf("New: %v", err) + } + tr.now = fixedNow // deterministic poison clamp + return tr +} + +// TestTrigger_LookbackCapturesLateArrivalExactlyOnce — T2: a row that +// lands BELOW the watermark but inside the lookback window is processed +// exactly once (no drop, no duplicate over many re-polls), and a row +// below watermark-lookback stays dropped (the documented bound). Also +// asserts ae_trigger_below_watermark_total increments (T3). +func TestTrigger_LookbackCapturesLateArrivalExactlyOnce(t *testing.T) { + f := newFakeCH(t) + defer f.close() + f.add(fakeRow{eventTime: testBase, ruleID: "R1", pid: 111}) // head row → watermark = testBase + + belowBefore := testutil.ToFloat64(metricBelowWatermark) + + tr := newLookbackTrigger(t, f, "node-lb", 300*time.Second) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ch, _ := tr.Subscribe(ctx) + + // Wait for the head row so the watermark is at testBase. + select { + case ev := <-ch: + if ev.Target.PID != 111 { + t.Fatalf("first event PID = %d, want 111", ev.Target.PID) + } + case <-time.After(500 * time.Millisecond): + t.Fatalf("timeout waiting for head row") + } + + // Late arrival 60s below the watermark (inside the 300s window) and + // one 400s below (outside the window). + f.add(fakeRow{eventTime: testBase - 60, ruleID: "R2", pid: 222}) + f.add(fakeRow{eventTime: testBase - 400, ruleID: "R3", pid: 333}) + + got := map[uint64]int{} + deadline := time.Now().Add(400 * time.Millisecond) // ~20 re-polls of the same window + for time.Now().Before(deadline) { + select { + case ev := <-ch: + got[ev.Target.PID]++ + case <-time.After(20 * time.Millisecond): + } + } + if got[222] != 1 { + t.Errorf("late-arrival row emitted %d times, want exactly 1 (T2)", got[222]) + } + if got[333] != 0 { + t.Errorf("row below watermark-lookback emitted %d times, want 0 (documented bound)", got[333]) + } + if got[111] != 0 { + t.Errorf("head row re-emitted %d times after initial delivery (window dedup failed)", got[111]) + } + if delta := testutil.ToFloat64(metricBelowWatermark) - belowBefore; delta < 1 { + t.Errorf("ae_trigger_below_watermark_total delta = %v, want >= 1", delta) + } +} + +// TestTrigger_PoisonRowDoesNotHalt — T1 (the F8 non-halt guarantee): a +// row carrying the real E8 poison timestamp (1.78e18-style far-future +// vs the pinned clock) is clamp-rejected from advancing the watermark, +// the reject metric increments, the watermark gauge stays wall-clock- +// bounded, and SUBSEQUENT seconds rows are still processed — no manual +// watermark reset needed. +func TestTrigger_PoisonRowDoesNotHalt(t *testing.T) { + // The exact leftover value from loadtest E8's poisoned watermark. + // Normalized it stays 1.781559e18 ns ≈ 12 days past the pinned + // clock (1.7805e9 s) — beyond the 1h MaxSkew. + const poisonET = uint64(1781559619170395824) + + f := newFakeCH(t) + defer f.close() + f.add(fakeRow{eventTime: testBase - 10, ruleID: "R1", pid: 111}) + + rejBefore := testutil.ToFloat64(metricEventTimeRejected) + + tr := newLookbackTrigger(t, f, "node-poison", 300*time.Second) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ch, _ := tr.Subscribe(ctx) + + select { + case ev := <-ch: + if ev.Target.PID != 111 { + t.Fatalf("first event PID = %d, want 111", ev.Target.PID) + } + case <-time.After(500 * time.Millisecond): + t.Fatalf("timeout waiting for head row") + } + + // Inject the poison row; it is emitted once (real anomaly, mangled + // timestamp) but must not advance the cursor. + f.add(fakeRow{eventTime: poisonET, ruleID: "RPOISON", pid: 666}) + select { + case ev := <-ch: + if ev.Target.PID != 666 { + t.Fatalf("expected poison row emission, got PID %d", ev.Target.PID) + } + case <-time.After(500 * time.Millisecond): + t.Fatalf("poison row was dropped entirely; want emitted-once-without-advance") + } + if delta := testutil.ToFloat64(metricEventTimeRejected) - rejBefore; delta < 1 { + t.Errorf("ae_trigger_event_time_rejected_total delta = %v, want >= 1", delta) + } + + // THE F8 guarantee: a fresh seconds row AFTER the poison must flow. + // Under the old strict HWM the cursor sat at 1.78e18 and this row + // was below it forever (25/25 ticks at n_anomalies=0 in E8). + f.add(fakeRow{eventTime: testBase + 5, ruleID: "R2", pid: 222}) + var got222 int + deadline := time.Now().Add(600 * time.Millisecond) + for time.Now().Before(deadline) && got222 == 0 { + select { + case ev := <-ch: + if ev.Target.PID == 222 { + got222++ + } + case <-time.After(20 * time.Millisecond): + } + } + if got222 != 1 { + t.Fatalf("post-poison seconds row emitted %d times, want 1 (T1 non-halt)", got222) + } + + // Watermark gauge stays wall-clock-bounded: it advanced to the real + // row (testBase+5 s), NOT to the poison value. + wantWM := float64(normalizeEventTimeNanos(testBase + 5)) + if got := testutil.ToFloat64(metricWatermarkNS.WithLabelValues("kubescape_logs", "node-poison")); got != wantWM { + t.Errorf("ae_trigger_watermark_ns = %v, want %v (wall-clock-bounded, not poison)", got, wantWM) + } +} + +// TestTrigger_PoisonPersistedWatermarkSelfRecovers — the E8 recovery +// scenario without the manual ALTER TABLE … DELETE: a pre-fix deployment +// left a far-future watermark behind; on start the trigger clamps it to +// wall-clock and fresh rows flow again. +func TestTrigger_PoisonPersistedWatermarkSelfRecovers(t *testing.T) { + const poisonWM = uint64(1781559619170395824) + + f := newFakeCH(t) + defer f.close() + f.add(fakeRow{eventTime: testBase, ruleID: "R1", pid: 111}) + + tr := newLookbackTrigger(t, f, "node-recover", 300*time.Second) + tr.cfg.InitialWatermark = poisonWM // simulates the poisoned persisted cursor + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ch, _ := tr.Subscribe(ctx) + + select { + case ev := <-ch: + if ev.Target.PID != 111 { + t.Fatalf("recovered event PID = %d, want 111", ev.Target.PID) + } + case <-time.After(500 * time.Millisecond): + t.Fatalf("fresh row not delivered — poisoned persisted watermark was not clamped (still halted)") + } +} + +// TestTrigger_LookbackZeroIsStrictHWM — T4: LOOKBACK=0 preserves the +// legacy strict high-water-mark exactly — the poll bound IS the +// watermark (no window subtraction) and a below-watermark row stays +// dropped. (The monotonic happy path itself is pinned by the existing +// clickhouse_test.go suite, which runs with the zero-value Lookback.) +func TestTrigger_LookbackZeroIsStrictHWM(t *testing.T) { + f := newFakeCH(t) + defer f.close() + f.add(fakeRow{eventTime: testBase, ruleID: "R1", pid: 111}) + + tr := newLookbackTrigger(t, f, "node-strict", 0) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + ch, _ := tr.Subscribe(ctx) + + select { + case ev := <-ch: + if ev.Target.PID != 111 { + t.Fatalf("first event PID = %d, want 111", ev.Target.PID) + } + case <-time.After(500 * time.Millisecond): + t.Fatalf("timeout waiting for head row") + } + + // A late arrival below the watermark: with strict HWM the SELECT + // bound equals the watermark, so it is never fetched again → dropped. + f.add(fakeRow{eventTime: testBase - 60, ruleID: "R2", pid: 222}) + got := map[uint64]int{} + deadline := time.Now().Add(300 * time.Millisecond) + for time.Now().Before(deadline) { + select { + case ev := <-ch: + got[ev.Target.PID]++ + case <-time.After(20 * time.Millisecond): + } + } + if got[222] != 0 { + t.Errorf("strict mode emitted a below-watermark row %d times; want 0 (legacy behavior)", got[222]) + } + if got[111] != 0 { + t.Errorf("strict mode re-emitted the boundary row %d times; want 0", got[111]) + } +} diff --git a/src/vizier/services/adaptive_export/internal/trigger/metrics.go b/src/vizier/services/adaptive_export/internal/trigger/metrics.go new file mode 100644 index 00000000000..4fc9dee9eda --- /dev/null +++ b/src/vizier/services/adaptive_export/internal/trigger/metrics.go @@ -0,0 +1,57 @@ +// Copyright 2018- The Pixie Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// SPDX-License-Identifier: Apache-2.0 + +package trigger + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +// Watermark observability (#97 / F8 / AE-9). Registered on the DEFAULT +// prometheus registry via promauto — the same pattern the rest of pixie +// uses (e.g. query_broker's queryExec* summaries) — and served by the +// shared services/metrics /metrics handler wired up in cmd/main.go. +// Before these existed a watermark halt was completely invisible: writes +// stopped, no error, no signal (loadtest E8). +var ( + // metricWatermarkNS tracks the trigger's current cursor in + // normalized unix NANOS, per (table, hostname). A flat gauge while + // kubescape rows keep arriving is the F8 silent-halt signature — + // alert on it. + metricWatermarkNS = promauto.NewGaugeVec(prometheus.GaugeOpts{ + Name: "ae_trigger_watermark_ns", + Help: "Current trigger high-water-mark cursor in normalized unix nanoseconds, per (table, hostname).", + }, []string{"table", "hostname"}) + + // metricBelowWatermark counts rows processed with a normalized + // event_time BELOW the poll-start watermark — i.e. out-of-order / + // clock-skewed / restart-buried rows the legacy strict HWM silently + // dropped and the bounded lookback now captures. + metricBelowWatermark = promauto.NewCounter(prometheus.CounterOpts{ + Name: "ae_trigger_below_watermark_total", + Help: "Rows seen with event_time below the prior watermark that the bounded lookback captured (strict HWM would have dropped them).", + }) + + // metricEventTimeRejected counts poison clamps: rows whose + // normalized event_time was implausibly far in the future + // (> now + MaxSkew) and were therefore barred from advancing the + // watermark. + metricEventTimeRejected = promauto.NewCounter(prometheus.CounterOpts{ + Name: "ae_trigger_event_time_rejected_total", + Help: "Rows whose normalized event_time exceeded now+max-skew and were rejected from advancing the watermark (poison clamp).", + }) +)