adaptive_export: reliable dx-steered pem-direct capture (chunk/end_time, breaker, dc_snoop filter, DaemonSet)

bazel/repository_locations.bzl

@@ -456,7 +456,10 @@ REPOSITORY_LOCATIONS = dict(
     org_libc_musl = dict(
         sha256 = "7d5b0b6062521e4627e099e4c9dc8248d32a30285e959b7eecaa780cf8cfd4a4",
         strip_prefix = "musl-1.2.3",
-        urls = ["http://musl.libc.org/releases/musl-1.2.3.tar.gz"],
+        urls = [
+            "https://sources.openwrt.org/musl-1.2.3.tar.gz",
+            "http://musl.libc.org/releases/musl-1.2.3.tar.gz",
+        ],
         manual_license_name = "libc/musl",
     ),
     rules_cc = dict(

ci/artifact_utils.sh

@@ -107,7 +107,15 @@ create_manifest_update() {
   tag_name="release/${component}/v${version}"
   # actions/checkout doesn't get the tag annotation properly.
   git fetch origin tag "${tag_name}" -f
-  timestamp="$(git tag -l --format "%(taggerdate:raw)" "${tag_name}"  | awk '{print $1}' | jq '. | todate')"
+  # taggerdate is empty for a LIGHTWEIGHT tag → produces `timestamp: ,` → jq syntax
+  # error → release-metadata step fails even though the image built fine. Fall back to
+  # the tagged commit's committer date so the manifest is well-formed regardless of how
+  # the release tag was cut (annotated vs lightweight).
+  raw_ts="$(git tag -l --format "%(taggerdate:raw)" "${tag_name}" | awk '{print $1}')"
+  if [ -z "${raw_ts}" ]; then
+    raw_ts="$(git log -1 --format="%ct" "${tag_name}")"
+  fi
+  timestamp="$(printf '%s' "${raw_ts}" | jq '. | todate')"
 
   jq -s \
     "[{name: \"${component}\", artifact: [{timestamp: ${timestamp}, commitHash: \"${commit_hash}\", versionStr: \"${version}\", availableArtifactMirrors: .}]}]" \

k8s/vizier/adaptive_export/kustomization.yaml

@@ -0,0 +1,10 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: pl
+resources:
+  - ../bootstrap/adaptive_export_role.yaml
+  - ../bootstrap/adaptive_export_deployment.yaml
+images:
+  - name: vizier-adaptive_export_image
+    newName: ghcr.io/k8sstormcenter/vizier-adaptive_export_image
+    newTag: 0.14.19-aeprod76

k8s/vizier/bootstrap/adaptive_export_deployment.yaml

@@ -1,115 +1,97 @@
 ---
+# adaptive-export: node-local forensic capture operator. DaemonSet so each pod
+# queries its own node's vizier-pem (pem-direct). Secret seeded per-cluster.
 apiVersion: apps/v1
-kind: Deployment
+kind: DaemonSet
 metadata:
   name: adaptive-export
+  labels: { name: adaptive-export, plane: control }
 spec:
-  replicas: 0
   selector:
-    matchLabels:
-      name: adaptive-export
+    matchLabels: { name: adaptive-export }
   template:
     metadata:
-      labels:
-        name: adaptive-export
-        plane: control
+      labels: { name: adaptive-export, plane: control }
     spec:
       affinity:
         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
             nodeSelectorTerms:
-            # The beta.kubernetes.io/os label has been deprecated since
-            # k8s v1.14; every modern kubelet sets kubernetes.io/os. The
-            # single term below is enough — kept both ORed terms in the
-            # past for pre-1.14 compatibility.
             - matchExpressions:
-              - key: kubernetes.io/os
-                operator: In
-                values:
-                - linux
+              - { key: kubernetes.io/os, operator: In, values: [linux] }
       serviceAccountName: pl-adaptive-export-service-account
       containers:
       - name: adaptive-export
         image: vizier-adaptive_export_image:latest
-        # Bounded so AE can never memory-pressure a node (measured: AE uses
-        # only ~16-38Mi steady; passthrough with the raised 1M-row cap can
-        # spike, so 1Gi caps the worst case). CPU was pinned at the old 300m
-        # limit under concurrent passthrough → raised to 1 core.
+        ports:
+        - { name: control, containerPort: 9100, hostPort: 9100 }
         resources:
-          requests:
-            cpu: 200m
-            memory: 128Mi
-          limits:
-            cpu: "1"
-            memory: 1Gi
+          requests: { cpu: 100m, memory: 128Mi }
+          limits:   { cpu: "1",  memory: 1Gi }
         env:
+        - name: HOST_IP
+          valueFrom: { fieldRef: { fieldPath: status.hostIP } }
+        - name: ADAPTIVE_VIZIER_DIRECT_ADDR
+          value: "$(HOST_IP):50305"
+        - name: PL_JWT_SIGNING_KEY
+          valueFrom: { secretKeyRef: { name: pl-cluster-secrets, key: jwt-signing-key } }
+        - name: PX_DISABLE_TLS
+          value: "1"
         - name: PL_NAMESPACE
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.namespace
+          valueFrom: { fieldRef: { fieldPath: metadata.namespace } }
+        - name: NODE_NAME
+          valueFrom: { fieldRef: { fieldPath: spec.nodeName } }
         - name: PIXIE_API_KEY
-          valueFrom:
-            secretKeyRef:
-              name: pl-adaptive-export-secrets
-              key: pixie-api-key
+          valueFrom: { secretKeyRef: { name: pl-adaptive-export-secrets, key: pixie-api-key } }
         - name: CLICKHOUSE_DSN
-          valueFrom:
-            secretKeyRef:
-              name: pl-adaptive-export-secrets
-              key: clickhouse-dsn
-        - name: VERBOSE
-          value: "true"
-        - name: DETECTION_INTERVAL_SEC
-          value: "10"
-        - name: DETECTION_LOOKBACK_SEC
-          value: "30"
-        # EXPORT_MODE controls the reconcile behaviour:
-        #   auto    - detection drives on/off (default)
-        #   always  - plugin always enabled (bypass detection)
-        #   never   - plugin always disabled and ch-* scripts purged
-        - name: EXPORT_MODE
-          value: "auto"
-        # Number of consecutive empty detection ticks before auto-disable fires.
-        - name: EXPORT_QUIET_TICKS
-          value: "6"
-        # Optional overrides for the ClickHouse PxL scripts. When unset they are
-        # parsed from CLICKHOUSE_DSN. Individual fields win over the parsed DSN.
-        # Defaults below match soc/tree/clickhouse-lab (forensic-soc-db CHI,
-        # ingest_writer user, forensic_db database).
+          valueFrom: { secretKeyRef: { name: pl-adaptive-export-secrets, key: clickhouse-dsn } }
         - name: KUBESCAPE_TABLE
           value: "kubescape_logs"
-        # - name: CLICKHOUSE_HOST
-        #   value: "clickhouse-forensic-soc-db.clickhouse.svc.cluster.local"
-        # - name: CLICKHOUSE_PORT
-        #   value: "9000"
-        # - name: CLICKHOUSE_USER
-        #   value: "ingest_writer"
-        # - name: CLICKHOUSE_PASSWORD
-        #   value: "changeme-ingest"
-        # - name: CLICKHOUSE_DATABASE
-        #   value: "forensic_db"
-        # TLS for the control surface (CONTROL_TLS=true). server.crt/key from the
-        # same service-tls-certs secret the broker/PEM use; without this the dx
-        # bearer JWT crosses the CNI in cleartext. Harmless when control is off.
+        - name: EXPORT_MODE
+          value: "never"
+        # Control surface is secure-by-default (#96): TLS + bearer-JWT auth are ON
+        # out of the box. The service-tls-certs keypair mounted at /certs below is
+        # used for TLS (else AE self-generates an ephemeral in-memory cert), and
+        # PL_JWT_SIGNING_KEY above turns on auth. CONTROL_TLS / CONTROL_REQUIRE_AUTH
+        # are deprecated no-ops; set CONTROL_INSECURE=true only to opt out (dev).
+        - name: CONTROL_ADDR
+          value: ":9100"
+        - name: ADAPTIVE_PUSH_PIXIE_ROWS
+          value: "true"
+        - name: ADAPTIVE_RECONCILE
+          value: "true"
+        - name: DEPLOY_TRACEPOINTS
+          value: "true"
+        - name: INSTALL_PRESET_SCRIPTS
+          value: "false"
+        - name: ADAPTIVE_MAX_INFLIGHT_QUERIES_GLOBAL
+          value: "4"
+        - name: ADAPTIVE_ORDER_CHUNK_SEC
+          value: "600"
+        - name: VERBOSE
+          value: "true"
         volumeMounts:
-        - name: certs
-          mountPath: /certs
-          readOnly: true
+        - { name: certs, mountPath: /certs, readOnly: true }
         securityContext:
           allowPrivilegeEscalation: false
-          capabilities:
-            drop:
-            - ALL
-          seccompProfile:
-            type: RuntimeDefault
+          capabilities: { drop: [ALL] }
+          seccompProfile: { type: RuntimeDefault }
       volumes:
       - name: certs
-        secret:
-          secretName: service-tls-certs
+        secret: { secretName: service-tls-certs }
       securityContext:
         runAsUser: 10100
         runAsGroup: 10100
         fsGroup: 10100
         runAsNonRoot: true
-        seccompProfile:
-          type: RuntimeDefault
+        seccompProfile: { type: RuntimeDefault }
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: adaptive-export-control
+spec:
+  selector: { name: adaptive-export }
+  internalTrafficPolicy: Local   # dx reaches its co-located (same-node) AE
+  ports:
+  - { name: control, port: 9100, targetPort: 9100 }

k8s/vizier/dx/dx-daemon.yaml

@@ -0,0 +1,79 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata: { name: dx-daemon, namespace: honey }
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: dx-daemon
+  namespace: honey
+  labels: { app: dx-daemon }
+spec:
+  selector: { matchLabels: { app: dx-daemon } }
+  template:
+    metadata:
+      labels: { app: dx-daemon }
+    spec:
+      serviceAccountName: dx-daemon
+      tolerations: [{ operator: Exists }]
+      terminationGracePeriodSeconds: 35
+      containers:
+      - name: dx-daemon
+        # OBFUSCATED release rc13 (entlein/dx#138 fault 2 RESOLVED): garble -literals
+        # (WITHOUT -tiny — -tiny's pclntab stripping SIGSEGV'd under load). Passes the
+        # obfuscation gate AND survives the kill-chain (restarts=0, 4 rounds). Carries the
+        # evidence-manifest + DX_FOREST_PUSHDOWN code.
+        image: docker.io/entlein/dx-daemon:0.5.0-keepset-rc5
+        ports:
+        - { name: findings, containerPort: 9099, hostPort: 9099 }
+        env:
+        - { name: NODE_NAME, valueFrom: { fieldRef: { fieldPath: spec.nodeName } } }
+        - { name: HOST_IP, valueFrom: { fieldRef: { fieldPath: status.hostIP } } }
+        - { name: DX_RECEIVER_TLS, value: "1" }
+        # AE control surface is TLS-by-default (#96); the dx client TLS-skip-verifies
+        # the in-cluster (self-signed/shared) cert and attaches its bearer JWT.
+        - { name: AE_CONTROL_ADDR, value: "https://adaptive-export-control.pl.svc.cluster.local:9100" }
+        - { name: PX_API_KEY, valueFrom: { secretKeyRef: { name: dx-pixie-auth, key: api-key, optional: true } } }
+        - { name: PX_CLUSTER_ID, valueFrom: { secretKeyRef: { name: dx-pixie-auth, key: cluster-id, optional: true } } }
+        - { name: PX_CLOUD_ADDR, valueFrom: { secretKeyRef: { name: dx-pixie-auth, key: cloud-addr, optional: true } } }
+        - { name: DX_BENCH, value: "pemdirect" }
+        - { name: PL_JWT_SIGNING_KEY, valueFrom: { secretKeyRef: { name: dx-vizier-direct, key: jwt-signing-key, optional: true } } }
+        - { name: DX_VIZIER_DIRECT_ADDR, value: "vizier-query-broker-svc.pl.svc.cluster.local:50300" }
+        - { name: PX_DISABLE_TLS, value: "1" }
+        - { name: DX_CLUSTER_MALIGNANT_HTTP, valueFrom: { secretKeyRef: { name: dx-metastasis-ch, key: http-url, optional: true } } }
+        - { name: DX_PX_TIMEOUT_S, value: "90" }
+        - { name: DX_TELEMETRY_CACHE, value: "1" }
+        - { name: DX_WORKERS, value: "4" }
+        # evidence-graph: forest-scope the evidence, write the per-anomaly edge set,
+        # sink it straight to forensic_db.dx_evidence_graph (soc ingest_writer).
+        - { name: DX_FOREST_SCOPE, value: "1" }
+        # FOREST_PUSHDOWN (entlein/dx#138 fault 3): push the dc_snoop ppid-lineage filter
+        # INTO the PxL so dx pulls only the alert pod's subtree, not the whole node —
+        # frees the node-local PEM so AE can export dc_snoop under load (validated: 0→1777).
+        - { name: DX_FOREST_PUSHDOWN, value: "1" }
+        - { name: DX_FOREST_PUSHDOWN_DEPTH, value: "4" }
+        - { name: DX_PRECORRELATE_GRAPH, value: "1" }
+        - { name: DX_EVIDENCE_GRAPH_CH, value: "http://ingest_writer:changeme-ingest@clickhouse-forensic-soc-db.clickhouse.svc.cluster.local:8123/forensic_db" }
+        readinessProbe:
+          httpGet: { path: /healthz, port: 9099, scheme: HTTPS }
+          initialDelaySeconds: 3
+          periodSeconds: 10
+        resources:
+          # memory: the precorrelate/full-evidence workup (DX_PRECORRELATE_GRAPH) + pemdirect
+          # gRPC result streams pull the per-anomaly evidence set into memory; at 1Gi dx is
+          # OOM-killed mid-workup (exit 137) BEFORE it writes the graph → crash-loop, empty
+          # graph. Measured peak ~1.3GB/round under the redis kill-chain (entlein/dx#138
+          # fault 1); 3Gi clears it reliably on an 8GiB node (validated: restarts=0 over 6+ rounds).
+          requests: { cpu: 50m, memory: 1Gi }
+          limits:   { cpu: "2", memory: 3Gi }
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: dx-daemon
+  namespace: honey
+spec:
+  selector: { app: dx-daemon }
+  internalTrafficPolicy: Local
+  ports:
+  - { name: findings, port: 9099, targetPort: 9099 }

k8s/vizier/dx/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: honey
+resources:
+  - dx-daemon.yaml

skaffold/skaffold_adaptive_export.yaml

@@ -0,0 +1,44 @@
+---
+# Deploy-only Skaffold for the adaptive_export DaemonSet using a prebuilt image
+# (lab / review), overlaying an already-running vizier. Run from the repo root:
+#   skaffold deploy -f skaffold/skaffold_adaptive_export.yaml
+# Bump the image via newTag in k8s/vizier/adaptive_export/kustomization.yaml.
+apiVersion: skaffold/v4beta11
+kind: Config
+metadata:
+  name: adaptive-export
+manifests:
+  kustomize:
+    paths:
+      - k8s/vizier/adaptive_export
+    buildArgs:
+      - --load-restrictor=LoadRestrictionsNone
+deploy:
+  kubectl:
+    defaultNamespace: pl
+    hooks:
+      before:
+        - host:
+            command:
+              - bash
+              - -c
+              - |
+                set -e
+                # PL_CLOUD_ADDR must carry an explicit :443 or the AE cloud client crashloops.
+                CA=$(kubectl -n pl get cm pl-cloud-config -o jsonpath='{.data.PL_CLOUD_ADDR}' 2>/dev/null || true)
+                case "$CA" in ""|*:*) ;; *) kubectl -n pl patch cm pl-cloud-config --type merge -p "{\"data\":{\"PL_CLOUD_ADDR\":\"$CA:443\"}}";; esac
+                # Seed pl-adaptive-export-secrets ONLY when a key is supplied; never clobber a good secret with an empty one.
+                API="${PIXIE_API_KEY:-${PX_API_KEY:-}}"
+                CH_DSN="${AE_CH_DSN:-ingest_writer:changeme-ingest@clickhouse-forensic-soc-db.clickhouse.svc.cluster.local:9000/forensic_db}"
+                if [ -n "$API" ]; then
+                  kubectl -n pl create secret generic pl-adaptive-export-secrets \
+                    --from-literal=pixie-api-key="$API" \
+                    --from-literal=clickhouse-dsn="$CH_DSN" \
+                    --dry-run=client -o yaml | kubectl apply -f -
+                elif ! kubectl -n pl get secret pl-adaptive-export-secrets >/dev/null 2>&1; then
+                  echo "ERROR: set PIXIE_API_KEY (or source keys.env) to seed pl-adaptive-export-secrets" >&2
+                  exit 1
+                else
+                  echo "pl-adaptive-export-secrets exists; PIXIE_API_KEY unset -> leaving it untouched"
+                fi
+            os: [linux, darwin]

skaffold/skaffold_dx.yaml

@@ -0,0 +1,42 @@
+---
+# Deploy-only Skaffold for the dx-daemon DaemonSet (prebuilt image), overlaying an
+# already-running vizier + soc stack. Run from the repo root, AFTER adaptive_export
+# (the hook mirrors pl-adaptive-export-secrets into honey):
+#   skaffold deploy -f skaffold/skaffold_dx.yaml
+apiVersion: skaffold/v4beta11
+kind: Config
+metadata:
+  name: dx-daemon
+manifests:
+  kustomize:
+    paths:
+      - k8s/vizier/dx
+deploy:
+  kubectl:
+    defaultNamespace: honey
+    hooks:
+      before:
+        - host:
+            command:
+              - bash
+              - -c
+              - |
+                set -e
+                kubectl create namespace honey --dry-run=client -o yaml | kubectl apply -f -
+                JWT=$(kubectl -n pl get secret pl-cluster-secrets -o jsonpath='{.data.jwt-signing-key}' | base64 -d)
+                CID=$(kubectl -n pl get secret pl-cluster-secrets -o jsonpath='{.data.cluster-id}' | base64 -d)
+                CA=$(kubectl -n pl get cm pl-cloud-config -o jsonpath='{.data.PL_CLOUD_ADDR}')
+                API=$(kubectl -n pl get secret pl-adaptive-export-secrets -o jsonpath='{.data.pixie-api-key}' 2>/dev/null | base64 -d)
+                CH_URL="${DX_CH_HTTP_URL:-http://ingest_writer:changeme-ingest@clickhouse-forensic-soc-db.clickhouse.svc.cluster.local:8123/?database=forensic_db}"
+                kubectl -n honey create secret generic dx-vizier-direct \
+                  --from-literal=jwt-signing-key="$JWT" \
+                  --dry-run=client -o yaml | kubectl apply -f -
+                kubectl -n honey create secret generic dx-pixie-auth \
+                  --from-literal=api-key="$API" \
+                  --from-literal=cluster-id="$CID" \
+                  --from-literal=cloud-addr="$CA" \
+                  --dry-run=client -o yaml | kubectl apply -f -
+                kubectl -n honey create secret generic dx-metastasis-ch \
+                  --from-literal=http-url="$CH_URL" \
+                  --dry-run=client -o yaml | kubectl apply -f -
+            os: [linux, darwin]