From 14a3283ae61b57d988b061bd36c5065696eaae6c Mon Sep 17 00:00:00 2001 From: entlein Date: Tue, 18 Aug 2026 17:37:39 +0200 Subject: [PATCH 01/19] allow networkpolicy to be a cel selector for internal/external traffic allowlisting Signed-off-by: entlein --- .../containerprofilecache/projection_apply.go | 30 +++++ .../projection_golden_test.go | 7 ++ .../testdata/golden/network_all.json | 32 ++++++ .../testdata/golden/rich_filtered.json | 2 + .../testdata/golden/rich_passthrough.json | 2 + pkg/objectcache/projection_types.go | 25 +++++ .../containerprofilenetwork.go | 29 +++++ .../containerprofilenetwork/network.go | 105 ++++++++++++++++++ .../containerprofilenetwork/selector_test.go | 63 +++++++++++ pkg/rulemanager/cel/selector_compile_test.go | 41 +++++++ pkg/utils/cel.go | 29 +++++ 11 files changed, 365 insertions(+) create mode 100644 pkg/rulemanager/cel/libraries/containerprofilenetwork/selector_test.go create mode 100644 pkg/rulemanager/cel/selector_compile_test.go diff --git a/pkg/objectcache/containerprofilecache/projection_apply.go b/pkg/objectcache/containerprofilecache/projection_apply.go index f0de872a52..a513680164 100644 --- a/pkg/objectcache/containerprofilecache/projection_apply.go +++ b/pkg/objectcache/containerprofilecache/projection_apply.go @@ -51,6 +51,10 @@ func Apply(spec *objectcache.RuleProjectionSpec, cp *v1beta1.ContainerProfile, c pcp.Execs = projectField(s.Execs, execsPaths, true) pcp.ExecsByPath = extractExecsByPath(cp) + pcp.Namespace = cp.Namespace + pcp.IngressPeers = extractIngressPeers(cp) + pcp.EgressPeers = extractEgressPeers(cp) + endpointPaths := extractEndpointPaths(cp) pcp.Endpoints = projectField(s.Endpoints, endpointPaths, true) @@ -260,3 +264,29 @@ func extractIngressAddresses(cp *v1beta1.ContainerProfile) []string { } return addrs } + +// extractIngressPeers / extractEgressPeers carry the label selectors of each +// network-neighbor entry so cp.was_selector_in_{ingress,egress} can match a +// peer by identity. Only entries that actually declare a podSelector are kept. +func extractIngressPeers(cp *v1beta1.ContainerProfile) []objectcache.PeerSelector { + return extractPeers(cp.Spec.Ingress) +} + +func extractEgressPeers(cp *v1beta1.ContainerProfile) []objectcache.PeerSelector { + return extractPeers(cp.Spec.Egress) +} + +func extractPeers(neighbors []v1beta1.NetworkNeighbor) []objectcache.PeerSelector { + var peers []objectcache.PeerSelector + for i := range neighbors { + n := &neighbors[i] + if n.PodSelector == nil { + continue + } + peers = append(peers, objectcache.PeerSelector{ + PodSelector: n.PodSelector, + NamespaceSelector: n.NamespaceSelector, + }) + } + return peers +} diff --git a/pkg/objectcache/containerprofilecache/projection_golden_test.go b/pkg/objectcache/containerprofilecache/projection_golden_test.go index 750d120bcb..1450907e29 100644 --- a/pkg/objectcache/containerprofilecache/projection_golden_test.go +++ b/pkg/objectcache/containerprofilecache/projection_golden_test.go @@ -65,6 +65,8 @@ type projectionGolden struct { EgressAddresses objectcache.ProjectedField `json:"egressAddresses"` IngressDomains objectcache.ProjectedField `json:"ingressDomains"` IngressAddresses objectcache.ProjectedField `json:"ingressAddresses"` + IngressPeers []objectcache.PeerSelector `json:"ingressPeers"` + EgressPeers []objectcache.PeerSelector `json:"egressPeers"` ExecsByPath map[string][][]string `json:"execsByPath"` PolicyByRuleId map[string]v1beta1.RulePolicy `json:"policyByRuleId"` CallStacks []callStackSummary `json:"callStacks"` @@ -93,6 +95,8 @@ func toGolden(pcp *objectcache.ProjectedContainerProfile, tree *callstackcache.C EgressAddresses: pcp.EgressAddresses, IngressDomains: pcp.IngressDomains, IngressAddresses: pcp.IngressAddresses, + IngressPeers: pcp.IngressPeers, + EgressPeers: pcp.EgressPeers, ExecsByPath: pcp.ExecsByPath, PolicyByRuleId: pcp.PolicyByRuleId, } @@ -236,9 +240,12 @@ func networkProfile() *v1beta1.ContainerProfile { Ingress: []v1beta1.NetworkNeighbor{ {Identifier: "in-a", DNS: "old.internal", DNSNames: []string{"a.internal", "b.internal"}, IPAddresses: []string{"192.168.1.10", "192.168.0.0/16"}}, {Identifier: "in-b", IPAddresses: []string{wild}}, + {Identifier: "in-c", PodSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "redis-client"}}}, + {Identifier: "in-d", PodSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "probe"}}, NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/metadata.name": "monitoring"}}}, }, Egress: []v1beta1.NetworkNeighbor{ {Identifier: "eg-a", DNSNames: []string{"c.example.com"}, IPAddress: "203.0.113.7", IPAddresses: []string{"203.0.113.0/24", wild}}, + {Identifier: "eg-b", PodSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "upstream"}}}, }, }, } diff --git a/pkg/objectcache/containerprofilecache/testdata/golden/network_all.json b/pkg/objectcache/containerprofilecache/testdata/golden/network_all.json index 3b833c5cfd..8607a33268 100644 --- a/pkg/objectcache/containerprofilecache/testdata/golden/network_all.json +++ b/pkg/objectcache/containerprofilecache/testdata/golden/network_all.json @@ -78,6 +78,38 @@ "PrefixHits": {}, "SuffixHits": {} }, + "ingressPeers": [ + { + "PodSelector": { + "matchLabels": { + "app": "redis-client" + } + }, + "NamespaceSelector": null + }, + { + "PodSelector": { + "matchLabels": { + "app": "probe" + } + }, + "NamespaceSelector": { + "matchLabels": { + "kubernetes.io/metadata.name": "monitoring" + } + } + } + ], + "egressPeers": [ + { + "PodSelector": { + "matchLabels": { + "app": "upstream" + } + }, + "NamespaceSelector": null + } + ], "execsByPath": null, "policyByRuleId": null, "callStacks": null diff --git a/pkg/objectcache/containerprofilecache/testdata/golden/rich_filtered.json b/pkg/objectcache/containerprofilecache/testdata/golden/rich_filtered.json index dccf5126c1..71fb76abb4 100644 --- a/pkg/objectcache/containerprofilecache/testdata/golden/rich_filtered.json +++ b/pkg/objectcache/containerprofilecache/testdata/golden/rich_filtered.json @@ -103,6 +103,8 @@ "PrefixHits": {}, "SuffixHits": {} }, + "ingressPeers": null, + "egressPeers": null, "execsByPath": { "/bin/curl": [ [ diff --git a/pkg/objectcache/containerprofilecache/testdata/golden/rich_passthrough.json b/pkg/objectcache/containerprofilecache/testdata/golden/rich_passthrough.json index a343f73b67..13eef20a98 100644 --- a/pkg/objectcache/containerprofilecache/testdata/golden/rich_passthrough.json +++ b/pkg/objectcache/containerprofilecache/testdata/golden/rich_passthrough.json @@ -97,6 +97,8 @@ "PrefixHits": {}, "SuffixHits": {} }, + "ingressPeers": null, + "egressPeers": null, "execsByPath": { "/bin/curl": [ [ diff --git a/pkg/objectcache/projection_types.go b/pkg/objectcache/projection_types.go index 8d452c7a36..07ea313ebc 100644 --- a/pkg/objectcache/projection_types.go +++ b/pkg/objectcache/projection_types.go @@ -3,8 +3,20 @@ package objectcache import ( "github.com/kubescape/node-agent/pkg/objectcache/callstackcache" "github.com/kubescape/storage/pkg/apis/softwarecomposition/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +// PeerSelector carries a single network-neighbor entry's identity selectors +// (podSelector + namespaceSelector) through the projection so the +// cp.was_selector_in_{ingress,egress} CEL helpers can resolve a runtime peer +// IP to a pod and match it by LABEL rather than by (volatile) IP. The address +// surfaces (Ingress/EgressAddresses) still carry the ipAddress/CIDR form for +// the was_address_in_* helpers; these are complementary. +type PeerSelector struct { + PodSelector *metav1.LabelSelector + NamespaceSelector *metav1.LabelSelector +} + // PathMatcher is implemented by the trie-based matchers in containerprofilecache. type PathMatcher interface { HasMatch(s string) bool @@ -44,6 +56,11 @@ type FieldSpec struct { // ProjectedContainerProfile is the cache-resident compact form. Pure node-agent // internal type; never serialized. Replaces *v1beta1.ContainerProfile in the cache. type ProjectedContainerProfile struct { + // Namespace is the profiled workload's own namespace; a peer entry whose + // NamespaceSelector is nil matches only peers in this namespace (the learned + // encoding and the NetworkPolicyPeer semantic for an absent namespaceSelector). + Namespace string + Opens ProjectedField Execs ProjectedField Endpoints ProjectedField @@ -54,6 +71,14 @@ type ProjectedContainerProfile struct { IngressDomains ProjectedField IngressAddresses ProjectedField + // IngressPeers / EgressPeers carry the podSelector+namespaceSelector of each + // network-neighbor entry (dropped by the address/domain projection) so the + // cp.was_selector_in_{ingress,egress} helpers can match a runtime peer by + // label. Always projected in full (not gated by a rule surface) since they + // are small and only populated when the profile actually declares selectors. + IngressPeers []PeerSelector + EgressPeers []PeerSelector + // ExecsByPath carries the per-Path Args slices from cp.Spec.Execs so // downstream consumers (e.g. dynamicpathdetector.CompareExecArgs used // by R0040 in node-agent#807) can run wildcard-aware argv matching diff --git a/pkg/rulemanager/cel/libraries/containerprofilenetwork/containerprofilenetwork.go b/pkg/rulemanager/cel/libraries/containerprofilenetwork/containerprofilenetwork.go index 58058c2aed..d9c7a2938d 100644 --- a/pkg/rulemanager/cel/libraries/containerprofilenetwork/containerprofilenetwork.go +++ b/pkg/rulemanager/cel/libraries/containerprofilenetwork/containerprofilenetwork.go @@ -83,6 +83,9 @@ type containerProfileNetworkFuncSpec struct { arity int // call invokes the shared implementation method on l. call func(l *containerProfileNetworkLibrary, args []ref.Val) ref.Val + // noCache bypasses the functionCache: a map argument has no stable scalar + // cache key, and the selector match is cheap (O(selectors)). + noCache bool } var containerProfileNetworkFuncSpecs = []containerProfileNetworkFuncSpec{ @@ -140,6 +143,26 @@ var containerProfileNetworkFuncSpecs = []containerProfileNetworkFuncSpec{ return l.wasAddressPortProtocolInIngress(a[0], a[1], a[2], a[3]) }, }, + { + name: "was_selector_in_egress", + argTypes: []*cel.Type{cel.StringType, cel.StringType, cel.MapType(cel.StringType, cel.StringType)}, + resultType: cel.BoolType, + arity: 3, + call: func(l *containerProfileNetworkLibrary, a []ref.Val) ref.Val { + return l.wasSelectorInEgress(a[0], a[1], a[2]) + }, + noCache: true, + }, + { + name: "was_selector_in_ingress", + argTypes: []*cel.Type{cel.StringType, cel.StringType, cel.MapType(cel.StringType, cel.StringType)}, + resultType: cel.BoolType, + arity: 3, + call: func(l *containerProfileNetworkLibrary, a []ref.Val) ref.Val { + return l.wasSelectorInIngress(a[0], a[1], a[2]) + }, + noCache: true, + }, } // declarationsWithPrefix builds the cel.FunctionOpt map for every function in @@ -165,6 +188,9 @@ func (l *containerProfileNetworkLibrary) declarationsWithPrefix(namePrefix, over if l.detailedMetrics && l.metrics != nil { l.metrics.IncHelperCall(fullName) } + if spec.noCache { + return cache.ConvertProfileNotAvailableErrToBool(spec.call(l, values), false) + } wrapperFunc := func(args ...ref.Val) ref.Val { return spec.call(l, args) } @@ -270,6 +296,9 @@ func (e *containerProfileNetworkCostEstimator) EstimateCallCost(function, overlo case "cp.is_domain_in_egress", "cp.is_domain_in_ingress": // Cache lookup + O(n) list iteration + O(m) slice.Contains on DNS names per entry cost = 35 + case "cp.was_selector_in_egress", "cp.was_selector_in_ingress": + // O(selectors) label-set match per peer entry + cost = 30 case "cp.was_address_port_protocol_in_egress", "cp.was_address_port_protocol_in_ingress": // Cache lookup + O(n) address search + O(p) nested port/protocol matching cost = 45 diff --git a/pkg/rulemanager/cel/libraries/containerprofilenetwork/network.go b/pkg/rulemanager/cel/libraries/containerprofilenetwork/network.go index 3d97e85f2e..f653c1b8c3 100644 --- a/pkg/rulemanager/cel/libraries/containerprofilenetwork/network.go +++ b/pkg/rulemanager/cel/libraries/containerprofilenetwork/network.go @@ -2,6 +2,7 @@ package containerprofilenetwork import ( "net" + "reflect" "strings" "github.com/google/cel-go/common/types" @@ -10,6 +11,8 @@ import ( "github.com/kubescape/node-agent/pkg/rulemanager/cel/libraries/cache" "github.com/kubescape/node-agent/pkg/rulemanager/profilehelper" "github.com/kubescape/storage/pkg/registry/file/networkmatch" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" ) // matchIPField is the wildcard-aware adapter from the projection layer's @@ -219,3 +222,105 @@ func (l *containerProfileNetworkLibrary) wasAddressPortProtocolInIngress(contain } return types.Bool(matchIPField(&cp.IngressAddresses, addressStr)) } + +// namespaceSelectorMatches matches a namespaceSelector against the peer's +// namespace via the implicit kubernetes.io/metadata.name label every namespace +// carries (the form these profiles use). A nil selector matches only the +// profiled workload's own namespace: the learned generator omits the selector +// exactly for same-namespace peers, and NetworkPolicyPeer gives an absent +// namespaceSelector the same meaning. Selectors keyed on other namespace +// labels are not resolved here. +func namespaceSelectorMatches(sel *metav1.LabelSelector, ns, profileNs string) bool { + if sel == nil { + return ns == profileNs + } + s, err := metav1.LabelSelectorAsSelector(sel) + if err != nil { + return false + } + return s.Matches(labels.Set{"kubernetes.io/metadata.name": ns}) +} + +// wasSelectorInPeers reports whether the peer identified by (podLabels, ns) +// matches any peer entry's podSelector AND its namespaceSelector. +func wasSelectorInPeers(peers []objectcache.PeerSelector, podLabels labels.Set, ns, profileNs string) bool { + for i := range peers { + peer := &peers[i] + if peer.PodSelector == nil { + continue + } + ps, err := metav1.LabelSelectorAsSelector(peer.PodSelector) + if err != nil { + continue + } + if ps.Matches(podLabels) && namespaceSelectorMatches(peer.NamespaceSelector, ns, profileNs) { + return true + } + } + return false +} + +func (l *containerProfileNetworkLibrary) wasSelectorInIngress(containerID, namespace, podLabels ref.Val) ref.Val { + return l.wasSelectorIn(containerID, namespace, podLabels, true) +} + +func (l *containerProfileNetworkLibrary) wasSelectorInEgress(containerID, namespace, podLabels ref.Val) ref.Val { + return l.wasSelectorIn(containerID, namespace, podLabels, false) +} + +// wasSelectorIn reports whether the runtime peer — identified by the namespace +// and pod labels that Inspektor Gadget's kubeipresolver stamps onto the network +// event — matches any of the profile's ingress-or-egress peer selectors. +// +// Matching on the peer's identity (namespace + labels) rather than its IP is the +// whole point: it is stable across pod IP churn AND works across nodes, because +// kubeipresolver resolves the peer against a cluster-wide pod inventory before +// the event ever reaches CEL. There is deliberately no IP→pod lookup here — that +// would reintroduce a dependency on node-agent's node-local pod cache, which is +// exactly what breaks cross-node peers. +func (l *containerProfileNetworkLibrary) wasSelectorIn(containerID, namespace, podLabels ref.Val, ingress bool) ref.Val { + if l.objectCache == nil { + return types.NewErr("objectCache is nil") + } + containerIDStr, ok := containerID.Value().(string) + if !ok { + return types.MaybeNoSuchOverloadErr(containerID) + } + nsStr, ok := namespace.Value().(string) + if !ok { + return types.MaybeNoSuchOverloadErr(namespace) + } + if nsStr == "" { + // The peer did not resolve to a pod (external IP, or the resolver had no + // inventory entry): it cannot satisfy any selector. A resolved pod with + // zero labels is NOT this case - an empty podSelector may still match it. + return types.Bool(false) + } + peerLabels := refValToStringMap(podLabels) + cp, _, err := profilehelper.GetProjectedContainerProfile(l.objectCache, containerIDStr) + if err != nil { + return cache.NewProfileNotAvailableErr("%v", err) + } + peers := cp.EgressPeers + if ingress { + peers = cp.IngressPeers + } + if len(peers) == 0 { + return types.Bool(false) + } + return types.Bool(wasSelectorInPeers(peers, labels.Set(peerLabels), nsStr, cp.Namespace)) +} + +// refValToStringMap converts a CEL map argument to a Go map[string]string. A nil +// or non-map value yields nil (treated as "peer has no labels"). +func refValToStringMap(v ref.Val) map[string]string { + if v == nil { + return nil + } + native, err := v.ConvertToNative(reflect.TypeOf(map[string]string(nil))) + if err != nil { + return nil + } + m, _ := native.(map[string]string) + return m +} diff --git a/pkg/rulemanager/cel/libraries/containerprofilenetwork/selector_test.go b/pkg/rulemanager/cel/libraries/containerprofilenetwork/selector_test.go new file mode 100644 index 0000000000..e80efce383 --- /dev/null +++ b/pkg/rulemanager/cel/libraries/containerprofilenetwork/selector_test.go @@ -0,0 +1,63 @@ +package containerprofilenetwork + +import ( + "testing" + + "github.com/kubescape/node-agent/pkg/objectcache" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" +) + +func peer(pod, ns map[string]string) objectcache.PeerSelector { + p := objectcache.PeerSelector{PodSelector: &metav1.LabelSelector{MatchLabels: pod}} + if ns != nil { + p.NamespaceSelector = &metav1.LabelSelector{MatchLabels: ns} + } + return p +} + +func TestWasSelectorInPeers(t *testing.T) { + // Peer identity as IG's kubeipresolver stamps it onto the event: a namespace + // and pod labels, resolved cluster-wide. No IP, no local pod lookup. + podLabels := labels.Set{"app": "redis-client"} + ns := "redis" + profileNs := "redis" + nsRedis := map[string]string{"kubernetes.io/metadata.name": "redis"} + + cases := []struct { + name string + peers []objectcache.PeerSelector + peerNs string + want bool + }{ + {"label+ns match", []objectcache.PeerSelector{peer(map[string]string{"app": "redis-client"}, nsRedis)}, ns, true}, + {"label mismatch", []objectcache.PeerSelector{peer(map[string]string{"app": "other"}, nsRedis)}, ns, false}, + {"ns mismatch", []objectcache.PeerSelector{peer(map[string]string{"app": "redis-client"}, map[string]string{"kubernetes.io/metadata.name": "other"})}, ns, false}, + {"nil ns selector matches the profile namespace", []objectcache.PeerSelector{peer(map[string]string{"app": "redis-client"}, nil)}, ns, true}, + {"nil ns selector rejects a foreign namespace", []objectcache.PeerSelector{peer(map[string]string{"app": "redis-client"}, nil)}, "attacker", false}, + {"empty peers", nil, ns, false}, + {"one of several matches", []objectcache.PeerSelector{peer(map[string]string{"app": "x"}, nil), peer(map[string]string{"app": "redis-client"}, nil)}, ns, true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := wasSelectorInPeers(tc.peers, podLabels, tc.peerNs, profileNs); got != tc.want { + t.Fatalf("wasSelectorInPeers = %v, want %v", got, tc.want) + } + }) + } +} + +func TestWasSelectorInPeers_EmptySelectorAndEmptyLabels(t *testing.T) { + profileNs := "redis" + emptySelector := []objectcache.PeerSelector{{PodSelector: &metav1.LabelSelector{}}} + + if !wasSelectorInPeers(emptySelector, labels.Set{}, "redis", profileNs) { + t.Fatal("an empty podSelector must match a resolved label-less pod in the profile namespace (NetworkPolicyPeer semantics)") + } + if wasSelectorInPeers(emptySelector, labels.Set{}, "attacker", profileNs) { + t.Fatal("an empty podSelector with nil namespaceSelector must not match a pod outside the profile namespace") + } + if !wasSelectorInPeers(emptySelector, labels.Set{"app": "anything"}, "redis", profileNs) { + t.Fatal("an empty podSelector selects all pods in the namespace, labelled or not") + } +} diff --git a/pkg/rulemanager/cel/selector_compile_test.go b/pkg/rulemanager/cel/selector_compile_test.go new file mode 100644 index 0000000000..fbcb225d5b --- /dev/null +++ b/pkg/rulemanager/cel/selector_compile_test.go @@ -0,0 +1,41 @@ +package cel + +import ( + "testing" + "time" + + "github.com/goradd/maps" + "github.com/kubescape/node-agent/pkg/config" + "github.com/kubescape/node-agent/pkg/objectcache" + objectcachev1 "github.com/kubescape/node-agent/pkg/objectcache/v1" + "github.com/kubescape/node-agent/pkg/rulemanager/cel/libraries/cache" +) + +// TestCompileSelectorRules pins that the selector rules type-check against the +// real event object type: event.dstPodLabels is declared as a generic CEL map +// and must remain assignable to the was_selector_in_{ingress,egress} map param. +// Regression guard for the R0012 ingress rule that consumes the IG-enriched +// peer namespace + labels. +func TestCompileSelectorRules(t *testing.T) { + objCache := &objectcachev1.RuleObjectCacheMock{ + ContainerIDToSharedData: maps.NewSafeMap[string, *objectcache.WatchedContainerData](), + } + c, err := NewCEL(objCache, config.Config{ + CelConfigCache: cache.FunctionCacheConfig{MaxSize: 1000, TTL: time.Minute}, + }) + if err != nil { + t.Fatalf("NewCEL: %v", err) + } + + exprs := []string{ + `cp.was_selector_in_ingress(event.containerId, event.dstNamespace, event.dstPodLabels)`, + `cp.was_selector_in_egress(event.containerId, event.dstNamespace, event.dstPodLabels)`, + // The full R0012 ingress expression as bound in default-rules.yaml. + `event.pktType == 'HOST' && !event.dstAddr.startsWith('127.') && !cp.was_address_in_ingress(event.containerId, event.dstAddr) && !cp.was_selector_in_ingress(event.containerId, event.dstNamespace, event.dstPodLabels)`, + } + for _, e := range exprs { + if err := c.registerExpression(e); err != nil { + t.Fatalf("expression failed to compile: %q\n%v", e, err) + } + } +} diff --git a/pkg/utils/cel.go b/pkg/utils/cel.go index 39c3ad40f2..b95cc1c115 100644 --- a/pkg/utils/cel.go +++ b/pkg/utils/cel.go @@ -202,6 +202,35 @@ var CelFields = map[string]*celtypes.FieldType{ return celtypes.Int(x.Raw.GetDstPort()), nil }), }, + // dstNamespace / dstPodLabels carry the peer identity that IG's + // kubeipresolver resolves cluster-wide (independent of node-agent's + // node-local pod cache), so selector rules can match a peer on any node. + "dstNamespace": { + Type: celtypes.StringType, + IsSet: isSet, + GetFrom: ref.FieldGetter(func(target any) (any, error) { + x := target.(*xcel.Object[CelEvent]) + if x.Raw == nil { + return nil, errCelObjectNil + } + return celtypes.String(x.Raw.GetDstEndpoint().Namespace), nil + }), + }, + "dstPodLabels": { + Type: celtypes.MapType, + IsSet: isSet, + GetFrom: ref.FieldGetter(func(target any) (any, error) { + x := target.(*xcel.Object[CelEvent]) + if x.Raw == nil { + return nil, errCelObjectNil + } + pl := x.Raw.GetDstEndpoint().PodLabels + if pl == nil { + pl = map[string]string{} + } + return pl, nil + }), + }, "exepath": { Type: celtypes.StringType, IsSet: isSet, From e6b7fabacdd3d3da122aca3515466c192f659859 Mon Sep 17 00:00:00 2001 From: entlein Date: Tue, 18 Aug 2026 19:06:01 +0200 Subject: [PATCH 02/19] Allow alert from unexpected Ports, allow Port=0 as intentional wildcard Signed-off-by: entlein --- .../containerprofilecache/projection_apply.go | 33 ++++ .../projection_golden_test.go | 7 + .../testdata/golden/network_all.json | 32 ++++ .../testdata/golden/rich_filtered.json | 2 + .../testdata/golden/rich_passthrough.json | 2 + pkg/objectcache/projection_types.go | 72 +++++++++ pkg/objectcache/v1/mock.go | 3 + .../containerprofilenetwork.go | 29 ++++ .../integration_test.go | 15 +- .../containerprofilenetwork/network.go | 141 +++++++++++++++++- .../containerprofilenetwork/network_test.go | 14 +- .../port_protocol_test.go | 47 ++++++ .../containerprofilenetwork/selector_test.go | 63 ++++++++ .../containerprofilenetwork/wildcard_test.go | 32 ++-- pkg/rulemanager/cel/selector_compile_test.go | 41 +++++ pkg/utils/cel.go | 29 ++++ .../templates/node-agent/default-rules.yaml | 2 +- tests/component_test.go | 60 ++++++++ ...containerprofile-user-defined-network.yaml | 33 ++++ tests/resources/network_fixture_lint_test.go | 5 +- tests/scripts/issue79-eol-ladder.sh | 107 +++++++++++++ 21 files changed, 717 insertions(+), 52 deletions(-) create mode 100644 pkg/rulemanager/cel/libraries/containerprofilenetwork/port_protocol_test.go create mode 100644 pkg/rulemanager/cel/libraries/containerprofilenetwork/selector_test.go create mode 100644 pkg/rulemanager/cel/selector_compile_test.go create mode 100755 tests/scripts/issue79-eol-ladder.sh diff --git a/pkg/objectcache/containerprofilecache/projection_apply.go b/pkg/objectcache/containerprofilecache/projection_apply.go index f0de872a52..60e7fb32bf 100644 --- a/pkg/objectcache/containerprofilecache/projection_apply.go +++ b/pkg/objectcache/containerprofilecache/projection_apply.go @@ -51,6 +51,10 @@ func Apply(spec *objectcache.RuleProjectionSpec, cp *v1beta1.ContainerProfile, c pcp.Execs = projectField(s.Execs, execsPaths, true) pcp.ExecsByPath = extractExecsByPath(cp) + pcp.Namespace = cp.Namespace + pcp.IngressPeers = extractIngressPeers(cp) + pcp.EgressPeers = extractEgressPeers(cp) + endpointPaths := extractEndpointPaths(cp) pcp.Endpoints = projectField(s.Endpoints, endpointPaths, true) @@ -63,6 +67,9 @@ func Apply(spec *objectcache.RuleProjectionSpec, cp *v1beta1.ContainerProfile, c pcp.IngressDomains = projectField(s.IngressDomains, extractIngressDomains(cp), false) pcp.IngressAddresses = projectField(s.IngressAddresses, extractIngressAddresses(cp), false) + pcp.EgressAddrPorts = objectcache.ExtractAddrPorts(cp.Spec.Egress) + pcp.IngressAddrPorts = objectcache.ExtractAddrPorts(cp.Spec.Ingress) + return pcp } @@ -260,3 +267,29 @@ func extractIngressAddresses(cp *v1beta1.ContainerProfile) []string { } return addrs } + +// extractIngressPeers / extractEgressPeers carry the label selectors of each +// network-neighbor entry so cp.was_selector_in_{ingress,egress} can match a +// peer by identity. Only entries that actually declare a podSelector are kept. +func extractIngressPeers(cp *v1beta1.ContainerProfile) []objectcache.PeerSelector { + return extractPeers(cp.Spec.Ingress) +} + +func extractEgressPeers(cp *v1beta1.ContainerProfile) []objectcache.PeerSelector { + return extractPeers(cp.Spec.Egress) +} + +func extractPeers(neighbors []v1beta1.NetworkNeighbor) []objectcache.PeerSelector { + var peers []objectcache.PeerSelector + for i := range neighbors { + n := &neighbors[i] + if n.PodSelector == nil { + continue + } + peers = append(peers, objectcache.PeerSelector{ + PodSelector: n.PodSelector, + NamespaceSelector: n.NamespaceSelector, + }) + } + return peers +} diff --git a/pkg/objectcache/containerprofilecache/projection_golden_test.go b/pkg/objectcache/containerprofilecache/projection_golden_test.go index 750d120bcb..1450907e29 100644 --- a/pkg/objectcache/containerprofilecache/projection_golden_test.go +++ b/pkg/objectcache/containerprofilecache/projection_golden_test.go @@ -65,6 +65,8 @@ type projectionGolden struct { EgressAddresses objectcache.ProjectedField `json:"egressAddresses"` IngressDomains objectcache.ProjectedField `json:"ingressDomains"` IngressAddresses objectcache.ProjectedField `json:"ingressAddresses"` + IngressPeers []objectcache.PeerSelector `json:"ingressPeers"` + EgressPeers []objectcache.PeerSelector `json:"egressPeers"` ExecsByPath map[string][][]string `json:"execsByPath"` PolicyByRuleId map[string]v1beta1.RulePolicy `json:"policyByRuleId"` CallStacks []callStackSummary `json:"callStacks"` @@ -93,6 +95,8 @@ func toGolden(pcp *objectcache.ProjectedContainerProfile, tree *callstackcache.C EgressAddresses: pcp.EgressAddresses, IngressDomains: pcp.IngressDomains, IngressAddresses: pcp.IngressAddresses, + IngressPeers: pcp.IngressPeers, + EgressPeers: pcp.EgressPeers, ExecsByPath: pcp.ExecsByPath, PolicyByRuleId: pcp.PolicyByRuleId, } @@ -236,9 +240,12 @@ func networkProfile() *v1beta1.ContainerProfile { Ingress: []v1beta1.NetworkNeighbor{ {Identifier: "in-a", DNS: "old.internal", DNSNames: []string{"a.internal", "b.internal"}, IPAddresses: []string{"192.168.1.10", "192.168.0.0/16"}}, {Identifier: "in-b", IPAddresses: []string{wild}}, + {Identifier: "in-c", PodSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "redis-client"}}}, + {Identifier: "in-d", PodSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "probe"}}, NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/metadata.name": "monitoring"}}}, }, Egress: []v1beta1.NetworkNeighbor{ {Identifier: "eg-a", DNSNames: []string{"c.example.com"}, IPAddress: "203.0.113.7", IPAddresses: []string{"203.0.113.0/24", wild}}, + {Identifier: "eg-b", PodSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "upstream"}}}, }, }, } diff --git a/pkg/objectcache/containerprofilecache/testdata/golden/network_all.json b/pkg/objectcache/containerprofilecache/testdata/golden/network_all.json index 3b833c5cfd..8607a33268 100644 --- a/pkg/objectcache/containerprofilecache/testdata/golden/network_all.json +++ b/pkg/objectcache/containerprofilecache/testdata/golden/network_all.json @@ -78,6 +78,38 @@ "PrefixHits": {}, "SuffixHits": {} }, + "ingressPeers": [ + { + "PodSelector": { + "matchLabels": { + "app": "redis-client" + } + }, + "NamespaceSelector": null + }, + { + "PodSelector": { + "matchLabels": { + "app": "probe" + } + }, + "NamespaceSelector": { + "matchLabels": { + "kubernetes.io/metadata.name": "monitoring" + } + } + } + ], + "egressPeers": [ + { + "PodSelector": { + "matchLabels": { + "app": "upstream" + } + }, + "NamespaceSelector": null + } + ], "execsByPath": null, "policyByRuleId": null, "callStacks": null diff --git a/pkg/objectcache/containerprofilecache/testdata/golden/rich_filtered.json b/pkg/objectcache/containerprofilecache/testdata/golden/rich_filtered.json index dccf5126c1..71fb76abb4 100644 --- a/pkg/objectcache/containerprofilecache/testdata/golden/rich_filtered.json +++ b/pkg/objectcache/containerprofilecache/testdata/golden/rich_filtered.json @@ -103,6 +103,8 @@ "PrefixHits": {}, "SuffixHits": {} }, + "ingressPeers": null, + "egressPeers": null, "execsByPath": { "/bin/curl": [ [ diff --git a/pkg/objectcache/containerprofilecache/testdata/golden/rich_passthrough.json b/pkg/objectcache/containerprofilecache/testdata/golden/rich_passthrough.json index a343f73b67..13eef20a98 100644 --- a/pkg/objectcache/containerprofilecache/testdata/golden/rich_passthrough.json +++ b/pkg/objectcache/containerprofilecache/testdata/golden/rich_passthrough.json @@ -97,6 +97,8 @@ "PrefixHits": {}, "SuffixHits": {} }, + "ingressPeers": null, + "egressPeers": null, "execsByPath": { "/bin/curl": [ [ diff --git a/pkg/objectcache/projection_types.go b/pkg/objectcache/projection_types.go index 8d452c7a36..a8f7e4944a 100644 --- a/pkg/objectcache/projection_types.go +++ b/pkg/objectcache/projection_types.go @@ -1,10 +1,25 @@ package objectcache import ( + "strconv" + "strings" + "github.com/kubescape/node-agent/pkg/objectcache/callstackcache" "github.com/kubescape/storage/pkg/apis/softwarecomposition/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +// PeerSelector carries a single network-neighbor entry's identity selectors +// (podSelector + namespaceSelector) through the projection so the +// cp.was_selector_in_{ingress,egress} CEL helpers can resolve a runtime peer +// IP to a pod and match it by LABEL rather than by (volatile) IP. The address +// surfaces (Ingress/EgressAddresses) still carry the ipAddress/CIDR form for +// the was_address_in_* helpers; these are complementary. +type PeerSelector struct { + PodSelector *metav1.LabelSelector + NamespaceSelector *metav1.LabelSelector +} + // PathMatcher is implemented by the trie-based matchers in containerprofilecache. type PathMatcher interface { HasMatch(s string) bool @@ -41,9 +56,54 @@ type FieldSpec struct { SuffixMatcher PathMatcher } +// AddrPortGroup pairs one neighbor entry's addresses with its allowed ports. +// Empty Ports means any port (port 0 or no ports declared = wildcard). +type AddrPortGroup struct { + Addrs []string + Ports map[string]struct{} +} + +func PortKey(protocol string, port int32) string { + return strings.ToUpper(protocol) + "-" + strconv.Itoa(int(port)) +} + +func ExtractAddrPorts(neighbors []v1beta1.NetworkNeighbor) []AddrPortGroup { + var groups []AddrPortGroup + for i := range neighbors { + n := &neighbors[i] + var addrs []string + if n.IPAddress != "" { + addrs = append(addrs, n.IPAddress) + } + addrs = append(addrs, n.IPAddresses...) + if len(addrs) == 0 { + continue + } + ports := make(map[string]struct{}, len(n.Ports)) + wildcard := len(n.Ports) == 0 + for _, p := range n.Ports { + if p.Port == nil || *p.Port == 0 { + wildcard = true + continue + } + ports[PortKey(string(p.Protocol), *p.Port)] = struct{}{} + } + if wildcard { + ports = nil + } + groups = append(groups, AddrPortGroup{Addrs: addrs, Ports: ports}) + } + return groups +} + // ProjectedContainerProfile is the cache-resident compact form. Pure node-agent // internal type; never serialized. Replaces *v1beta1.ContainerProfile in the cache. type ProjectedContainerProfile struct { + // Namespace is the profiled workload's own namespace; a peer entry whose + // NamespaceSelector is nil matches only peers in this namespace (the learned + // encoding and the NetworkPolicyPeer semantic for an absent namespaceSelector). + Namespace string + Opens ProjectedField Execs ProjectedField Endpoints ProjectedField @@ -54,6 +114,18 @@ type ProjectedContainerProfile struct { IngressDomains ProjectedField IngressAddresses ProjectedField + // IngressPeers / EgressPeers carry the podSelector+namespaceSelector of each + // network-neighbor entry (dropped by the address/domain projection) so the + // cp.was_selector_in_{ingress,egress} helpers can match a runtime peer by + // label. Always projected in full (not gated by a rule surface) since they + // are small and only populated when the profile actually declares selectors. + IngressPeers []PeerSelector + EgressPeers []PeerSelector + + // IngressAddrPorts / EgressAddrPorts group each neighbor's addresses with its ports for was_address_port_protocol_in_*. + IngressAddrPorts []AddrPortGroup + EgressAddrPorts []AddrPortGroup + // ExecsByPath carries the per-Path Args slices from cp.Spec.Execs so // downstream consumers (e.g. dynamicpathdetector.CompareExecArgs used // by R0040 in node-agent#807) can run wildcard-aware argv matching diff --git a/pkg/objectcache/v1/mock.go b/pkg/objectcache/v1/mock.go index 789eccb9ec..5066b933cc 100644 --- a/pkg/objectcache/v1/mock.go +++ b/pkg/objectcache/v1/mock.go @@ -193,6 +193,9 @@ func (r *RuleObjectCacheMock) GetProjectedContainerProfile(containerID string) * } } + pcp.EgressAddrPorts = objectcache.ExtractAddrPorts(cp.Spec.Egress) + pcp.IngressAddrPorts = objectcache.ExtractAddrPorts(cp.Spec.Ingress) + return pcp } diff --git a/pkg/rulemanager/cel/libraries/containerprofilenetwork/containerprofilenetwork.go b/pkg/rulemanager/cel/libraries/containerprofilenetwork/containerprofilenetwork.go index 58058c2aed..d9c7a2938d 100644 --- a/pkg/rulemanager/cel/libraries/containerprofilenetwork/containerprofilenetwork.go +++ b/pkg/rulemanager/cel/libraries/containerprofilenetwork/containerprofilenetwork.go @@ -83,6 +83,9 @@ type containerProfileNetworkFuncSpec struct { arity int // call invokes the shared implementation method on l. call func(l *containerProfileNetworkLibrary, args []ref.Val) ref.Val + // noCache bypasses the functionCache: a map argument has no stable scalar + // cache key, and the selector match is cheap (O(selectors)). + noCache bool } var containerProfileNetworkFuncSpecs = []containerProfileNetworkFuncSpec{ @@ -140,6 +143,26 @@ var containerProfileNetworkFuncSpecs = []containerProfileNetworkFuncSpec{ return l.wasAddressPortProtocolInIngress(a[0], a[1], a[2], a[3]) }, }, + { + name: "was_selector_in_egress", + argTypes: []*cel.Type{cel.StringType, cel.StringType, cel.MapType(cel.StringType, cel.StringType)}, + resultType: cel.BoolType, + arity: 3, + call: func(l *containerProfileNetworkLibrary, a []ref.Val) ref.Val { + return l.wasSelectorInEgress(a[0], a[1], a[2]) + }, + noCache: true, + }, + { + name: "was_selector_in_ingress", + argTypes: []*cel.Type{cel.StringType, cel.StringType, cel.MapType(cel.StringType, cel.StringType)}, + resultType: cel.BoolType, + arity: 3, + call: func(l *containerProfileNetworkLibrary, a []ref.Val) ref.Val { + return l.wasSelectorInIngress(a[0], a[1], a[2]) + }, + noCache: true, + }, } // declarationsWithPrefix builds the cel.FunctionOpt map for every function in @@ -165,6 +188,9 @@ func (l *containerProfileNetworkLibrary) declarationsWithPrefix(namePrefix, over if l.detailedMetrics && l.metrics != nil { l.metrics.IncHelperCall(fullName) } + if spec.noCache { + return cache.ConvertProfileNotAvailableErrToBool(spec.call(l, values), false) + } wrapperFunc := func(args ...ref.Val) ref.Val { return spec.call(l, args) } @@ -270,6 +296,9 @@ func (e *containerProfileNetworkCostEstimator) EstimateCallCost(function, overlo case "cp.is_domain_in_egress", "cp.is_domain_in_ingress": // Cache lookup + O(n) list iteration + O(m) slice.Contains on DNS names per entry cost = 35 + case "cp.was_selector_in_egress", "cp.was_selector_in_ingress": + // O(selectors) label-set match per peer entry + cost = 30 case "cp.was_address_port_protocol_in_egress", "cp.was_address_port_protocol_in_ingress": // Cache lookup + O(n) address search + O(p) nested port/protocol matching cost = 45 diff --git a/pkg/rulemanager/cel/libraries/containerprofilenetwork/integration_test.go b/pkg/rulemanager/cel/libraries/containerprofilenetwork/integration_test.go index e515a5fd73..81143c0133 100644 --- a/pkg/rulemanager/cel/libraries/containerprofilenetwork/integration_test.go +++ b/pkg/rulemanager/cel/libraries/containerprofilenetwork/integration_test.go @@ -209,28 +209,24 @@ func TestIntegrationWithAllNetworkFunctions(t *testing.T) { expectedResult: true, }, { - // v1 degradation: port/protocol projection is out of scope; address IS in profile → true. name: "Check non-existent egress address with port and protocol", expression: `cp.was_address_port_protocol_in_egress(containerID, "192.168.1.100", 9999, "TCP")`, - expectedResult: true, + expectedResult: false, }, { - // v1 degradation: port/protocol projection is out of scope; address IS in profile → true. name: "Check non-existent ingress address with port and protocol", expression: `cp.was_address_port_protocol_in_ingress(containerID, "172.16.0.10", 9999, "TCP")`, - expectedResult: true, + expectedResult: false, }, { - // v1 degradation: port/protocol projection is out of scope; address IS in profile → true. name: "Check wrong protocol for existing address and port", expression: `cp.was_address_port_protocol_in_egress(containerID, "192.168.1.100", 80, "UDP")`, - expectedResult: true, + expectedResult: false, }, { - // v1 degradation: port/protocol projection is out of scope; address IS in profile → true. name: "Check wrong protocol for existing ingress address and port", expression: `cp.was_address_port_protocol_in_ingress(containerID, "172.16.0.10", 8080, "UDP")`, - expectedResult: true, + expectedResult: false, }, { name: "Complex network check with port and protocol - egress", @@ -243,10 +239,9 @@ func TestIntegrationWithAllNetworkFunctions(t *testing.T) { expectedResult: true, }, { - // v1 degradation: both sides match on address only → true. name: "Mixed valid and invalid port protocol checks", expression: `cp.was_address_port_protocol_in_egress(containerID, "192.168.1.100", 80, "TCP") && cp.was_address_port_protocol_in_egress(containerID, "192.168.1.100", 9999, "TCP")`, - expectedResult: true, + expectedResult: false, }, } diff --git a/pkg/rulemanager/cel/libraries/containerprofilenetwork/network.go b/pkg/rulemanager/cel/libraries/containerprofilenetwork/network.go index 3d97e85f2e..ff1bbed68e 100644 --- a/pkg/rulemanager/cel/libraries/containerprofilenetwork/network.go +++ b/pkg/rulemanager/cel/libraries/containerprofilenetwork/network.go @@ -2,6 +2,7 @@ package containerprofilenetwork import ( "net" + "reflect" "strings" "github.com/google/cel-go/common/types" @@ -10,6 +11,8 @@ import ( "github.com/kubescape/node-agent/pkg/rulemanager/cel/libraries/cache" "github.com/kubescape/node-agent/pkg/rulemanager/profilehelper" "github.com/kubescape/storage/pkg/registry/file/networkmatch" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" ) // matchIPField is the wildcard-aware adapter from the projection layer's @@ -59,6 +62,29 @@ func matchIPField(field *objectcache.ProjectedField, observed string) bool { return networkmatch.MatchIP(entries, observed) } +// matchAddrPort reports whether observed (address, protocol, port) falls within +// any single neighbor entry: its addresses match AND the entry allows the port +// (empty Ports = any port). Address-only entries thus stay wildcard on ports. +func matchAddrPort(groups []objectcache.AddrPortGroup, address, protocol string, port int32) bool { + if address == "" { + return false + } + key := objectcache.PortKey(protocol, port) + for i := range groups { + g := &groups[i] + if !networkmatch.MatchIP(g.Addrs, address) { + continue + } + if len(g.Ports) == 0 { + return true + } + if _, ok := g.Ports[key]; ok { + return true + } + } + return false +} + func matchDNSField(field *objectcache.ProjectedField, observed string) bool { if observed == "" || field == nil { return false @@ -171,9 +197,6 @@ func (l *containerProfileNetworkLibrary) wasAddressPortProtocolInEgress(containe if !ok { return types.MaybeNoSuchOverloadErr(address) } - // port/protocol projection (AddressPortsByAddr) is out of scope for the - // projection-v1 layer upstream landed; matchers degrade to address-only. - // Wildcards remain enforced via matchIPField. portInt, ok := port.Value().(int64) if !ok { return types.MaybeNoSuchOverloadErr(port) @@ -181,14 +204,15 @@ func (l *containerProfileNetworkLibrary) wasAddressPortProtocolInEgress(containe if portInt < 0 || portInt > 65535 { return types.Bool(false) } - if _, ok := protocol.Value().(string); !ok { + protocolStr, ok := protocol.Value().(string) + if !ok { return types.MaybeNoSuchOverloadErr(protocol) } cp, _, err := profilehelper.GetProjectedContainerProfile(l.objectCache, containerIDStr) if err != nil { return cache.NewProfileNotAvailableErr("%v", err) } - return types.Bool(matchIPField(&cp.EgressAddresses, addressStr)) + return types.Bool(matchAddrPort(cp.EgressAddrPorts, addressStr, protocolStr, int32(portInt))) } func (l *containerProfileNetworkLibrary) wasAddressPortProtocolInIngress(containerID, address, port, protocol ref.Val) ref.Val { @@ -210,12 +234,115 @@ func (l *containerProfileNetworkLibrary) wasAddressPortProtocolInIngress(contain if portInt < 0 || portInt > 65535 { return types.Bool(false) } - if _, ok := protocol.Value().(string); !ok { + protocolStr, ok := protocol.Value().(string) + if !ok { return types.MaybeNoSuchOverloadErr(protocol) } cp, _, err := profilehelper.GetProjectedContainerProfile(l.objectCache, containerIDStr) if err != nil { return cache.NewProfileNotAvailableErr("%v", err) } - return types.Bool(matchIPField(&cp.IngressAddresses, addressStr)) + return types.Bool(matchAddrPort(cp.IngressAddrPorts, addressStr, protocolStr, int32(portInt))) +} + +// namespaceSelectorMatches matches a namespaceSelector against the peer's +// namespace via the implicit kubernetes.io/metadata.name label every namespace +// carries (the form these profiles use). A nil selector matches only the +// profiled workload's own namespace: the learned generator omits the selector +// exactly for same-namespace peers, and NetworkPolicyPeer gives an absent +// namespaceSelector the same meaning. Selectors keyed on other namespace +// labels are not resolved here. +func namespaceSelectorMatches(sel *metav1.LabelSelector, ns, profileNs string) bool { + if sel == nil { + return ns == profileNs + } + s, err := metav1.LabelSelectorAsSelector(sel) + if err != nil { + return false + } + return s.Matches(labels.Set{"kubernetes.io/metadata.name": ns}) +} + +// wasSelectorInPeers reports whether the peer identified by (podLabels, ns) +// matches any peer entry's podSelector AND its namespaceSelector. +func wasSelectorInPeers(peers []objectcache.PeerSelector, podLabels labels.Set, ns, profileNs string) bool { + for i := range peers { + peer := &peers[i] + if peer.PodSelector == nil { + continue + } + ps, err := metav1.LabelSelectorAsSelector(peer.PodSelector) + if err != nil { + continue + } + if ps.Matches(podLabels) && namespaceSelectorMatches(peer.NamespaceSelector, ns, profileNs) { + return true + } + } + return false +} + +func (l *containerProfileNetworkLibrary) wasSelectorInIngress(containerID, namespace, podLabels ref.Val) ref.Val { + return l.wasSelectorIn(containerID, namespace, podLabels, true) +} + +func (l *containerProfileNetworkLibrary) wasSelectorInEgress(containerID, namespace, podLabels ref.Val) ref.Val { + return l.wasSelectorIn(containerID, namespace, podLabels, false) +} + +// wasSelectorIn reports whether the runtime peer — identified by the namespace +// and pod labels that Inspektor Gadget's kubeipresolver stamps onto the network +// event — matches any of the profile's ingress-or-egress peer selectors. +// +// Matching on the peer's identity (namespace + labels) rather than its IP is the +// whole point: it is stable across pod IP churn AND works across nodes, because +// kubeipresolver resolves the peer against a cluster-wide pod inventory before +// the event ever reaches CEL. There is deliberately no IP→pod lookup here — that +// would reintroduce a dependency on node-agent's node-local pod cache, which is +// exactly what breaks cross-node peers. +func (l *containerProfileNetworkLibrary) wasSelectorIn(containerID, namespace, podLabels ref.Val, ingress bool) ref.Val { + if l.objectCache == nil { + return types.NewErr("objectCache is nil") + } + containerIDStr, ok := containerID.Value().(string) + if !ok { + return types.MaybeNoSuchOverloadErr(containerID) + } + nsStr, ok := namespace.Value().(string) + if !ok { + return types.MaybeNoSuchOverloadErr(namespace) + } + if nsStr == "" { + // The peer did not resolve to a pod (external IP, or the resolver had no + // inventory entry): it cannot satisfy any selector. A resolved pod with + // zero labels is NOT this case - an empty podSelector may still match it. + return types.Bool(false) + } + peerLabels := refValToStringMap(podLabels) + cp, _, err := profilehelper.GetProjectedContainerProfile(l.objectCache, containerIDStr) + if err != nil { + return cache.NewProfileNotAvailableErr("%v", err) + } + peers := cp.EgressPeers + if ingress { + peers = cp.IngressPeers + } + if len(peers) == 0 { + return types.Bool(false) + } + return types.Bool(wasSelectorInPeers(peers, labels.Set(peerLabels), nsStr, cp.Namespace)) +} + +// refValToStringMap converts a CEL map argument to a Go map[string]string. A nil +// or non-map value yields nil (treated as "peer has no labels"). +func refValToStringMap(v ref.Val) map[string]string { + if v == nil { + return nil + } + native, err := v.ConvertToNative(reflect.TypeOf(map[string]string(nil))) + if err != nil { + return nil + } + m, _ := native.(map[string]string) + return m } diff --git a/pkg/rulemanager/cel/libraries/containerprofilenetwork/network_test.go b/pkg/rulemanager/cel/libraries/containerprofilenetwork/network_test.go index 10321073cc..5446f63a88 100644 --- a/pkg/rulemanager/cel/libraries/containerprofilenetwork/network_test.go +++ b/pkg/rulemanager/cel/libraries/containerprofilenetwork/network_test.go @@ -99,22 +99,20 @@ func TestWasAddressPortProtocolInEgress(t *testing.T) { expectedResult: true, }, { - // v1 degradation: port/protocol projection is out of scope; address-only matching. name: "Invalid port", containerID: "test-container-id", address: "192.168.1.100", port: 9999, protocol: "TCP", - expectedResult: true, + expectedResult: false, }, { - // v1 degradation: port/protocol projection is out of scope; address-only matching. name: "Invalid protocol", containerID: "test-container-id", address: "192.168.1.100", port: 80, protocol: "UDP", - expectedResult: true, + expectedResult: false, }, { name: "Invalid address", @@ -235,22 +233,20 @@ func TestWasAddressPortProtocolInIngress(t *testing.T) { expectedResult: true, }, { - // v1 degradation: port/protocol projection is out of scope; address-only matching. name: "Invalid port", containerID: "test-container-id", address: "172.16.0.10", port: 9999, protocol: "TCP", - expectedResult: true, + expectedResult: false, }, { - // v1 degradation: port/protocol projection is out of scope; address-only matching. name: "Invalid protocol", containerID: "test-container-id", address: "172.16.0.10", port: 8080, protocol: "UDP", - expectedResult: true, + expectedResult: false, }, { name: "Invalid address", @@ -405,7 +401,7 @@ func TestWasAddressPortProtocolWithNilPort(t *testing.T) { functionCache: cache.NewFunctionCache(cache.DefaultFunctionCacheConfig()), } - // v1 degradation: address-only matching; nil port in profile no longer checked. + // nil port in a profile entry = any-port wildcard for that entry's addresses. result := lib.wasAddressPortProtocolInEgress( types.String("test-container-id"), types.String("192.168.1.100"), diff --git a/pkg/rulemanager/cel/libraries/containerprofilenetwork/port_protocol_test.go b/pkg/rulemanager/cel/libraries/containerprofilenetwork/port_protocol_test.go new file mode 100644 index 0000000000..1750ac385c --- /dev/null +++ b/pkg/rulemanager/cel/libraries/containerprofilenetwork/port_protocol_test.go @@ -0,0 +1,47 @@ +package containerprofilenetwork + +import ( + "testing" + + "github.com/google/cel-go/common/types" + "github.com/kubescape/node-agent/pkg/rulemanager/cel/libraries/cache" + "github.com/kubescape/storage/pkg/apis/softwarecomposition/v1beta1" + "github.com/stretchr/testify/assert" + "k8s.io/utils/ptr" +) + +func port(proto string, p int32) v1beta1.NetworkPort { + return v1beta1.NetworkPort{Protocol: v1beta1.Protocol(proto), Port: ptr.To(p)} +} + +func evalEgressPort(lib *containerProfileNetworkLibrary, addr string, p int64, proto string) types.Bool { + res := lib.wasAddressPortProtocolInEgress(types.String("cid"), types.String(addr), types.Int(p), types.String(proto)) + return cache.ConvertProfileNotAvailableErrToBool(res, false).(types.Bool) +} + +func evalIngressPort(lib *containerProfileNetworkLibrary, addr string, p int64, proto string) types.Bool { + res := lib.wasAddressPortProtocolInIngress(types.String("cid"), types.String(addr), types.Int(p), types.String(proto)) + return cache.ConvertProfileNotAvailableErrToBool(res, false).(types.Bool) +} + +func TestWasAddressPortProtocolInEgress_PortWildcard(t *testing.T) { + noPorts := buildLibWithContainer(t, []v1beta1.NetworkNeighbor{ + {IPAddresses: []string{"93.184.216.34"}}, + }, nil) + assert.Equal(t, types.Bool(true), evalEgressPort(noPorts, "93.184.216.34", 8080, "TCP")) + assert.Equal(t, types.Bool(false), evalEgressPort(noPorts, "1.1.1.1", 8080, "TCP")) + + zeroPort := buildLibWithContainer(t, []v1beta1.NetworkNeighbor{ + {IPAddresses: []string{"93.184.216.34"}, Ports: []v1beta1.NetworkPort{port("TCP", 0)}}, + }, nil) + assert.Equal(t, types.Bool(true), evalEgressPort(zeroPort, "93.184.216.34", 8080, "TCP")) +} + +func TestWasAddressPortProtocolInIngress_Symmetric(t *testing.T) { + lib := buildLibWithContainer(t, nil, []v1beta1.NetworkNeighbor{ + {IPAddresses: []string{"172.16.0.0/12"}, Ports: []v1beta1.NetworkPort{port("TCP", 6379)}}, + }) + assert.Equal(t, types.Bool(true), evalIngressPort(lib, "172.16.5.9", 6379, "TCP")) + assert.Equal(t, types.Bool(false), evalIngressPort(lib, "172.16.5.9", 5432, "TCP")) + assert.Equal(t, types.Bool(false), evalIngressPort(lib, "10.0.0.1", 6379, "TCP")) +} diff --git a/pkg/rulemanager/cel/libraries/containerprofilenetwork/selector_test.go b/pkg/rulemanager/cel/libraries/containerprofilenetwork/selector_test.go new file mode 100644 index 0000000000..e80efce383 --- /dev/null +++ b/pkg/rulemanager/cel/libraries/containerprofilenetwork/selector_test.go @@ -0,0 +1,63 @@ +package containerprofilenetwork + +import ( + "testing" + + "github.com/kubescape/node-agent/pkg/objectcache" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" +) + +func peer(pod, ns map[string]string) objectcache.PeerSelector { + p := objectcache.PeerSelector{PodSelector: &metav1.LabelSelector{MatchLabels: pod}} + if ns != nil { + p.NamespaceSelector = &metav1.LabelSelector{MatchLabels: ns} + } + return p +} + +func TestWasSelectorInPeers(t *testing.T) { + // Peer identity as IG's kubeipresolver stamps it onto the event: a namespace + // and pod labels, resolved cluster-wide. No IP, no local pod lookup. + podLabels := labels.Set{"app": "redis-client"} + ns := "redis" + profileNs := "redis" + nsRedis := map[string]string{"kubernetes.io/metadata.name": "redis"} + + cases := []struct { + name string + peers []objectcache.PeerSelector + peerNs string + want bool + }{ + {"label+ns match", []objectcache.PeerSelector{peer(map[string]string{"app": "redis-client"}, nsRedis)}, ns, true}, + {"label mismatch", []objectcache.PeerSelector{peer(map[string]string{"app": "other"}, nsRedis)}, ns, false}, + {"ns mismatch", []objectcache.PeerSelector{peer(map[string]string{"app": "redis-client"}, map[string]string{"kubernetes.io/metadata.name": "other"})}, ns, false}, + {"nil ns selector matches the profile namespace", []objectcache.PeerSelector{peer(map[string]string{"app": "redis-client"}, nil)}, ns, true}, + {"nil ns selector rejects a foreign namespace", []objectcache.PeerSelector{peer(map[string]string{"app": "redis-client"}, nil)}, "attacker", false}, + {"empty peers", nil, ns, false}, + {"one of several matches", []objectcache.PeerSelector{peer(map[string]string{"app": "x"}, nil), peer(map[string]string{"app": "redis-client"}, nil)}, ns, true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := wasSelectorInPeers(tc.peers, podLabels, tc.peerNs, profileNs); got != tc.want { + t.Fatalf("wasSelectorInPeers = %v, want %v", got, tc.want) + } + }) + } +} + +func TestWasSelectorInPeers_EmptySelectorAndEmptyLabels(t *testing.T) { + profileNs := "redis" + emptySelector := []objectcache.PeerSelector{{PodSelector: &metav1.LabelSelector{}}} + + if !wasSelectorInPeers(emptySelector, labels.Set{}, "redis", profileNs) { + t.Fatal("an empty podSelector must match a resolved label-less pod in the profile namespace (NetworkPolicyPeer semantics)") + } + if wasSelectorInPeers(emptySelector, labels.Set{}, "attacker", profileNs) { + t.Fatal("an empty podSelector with nil namespaceSelector must not match a pod outside the profile namespace") + } + if !wasSelectorInPeers(emptySelector, labels.Set{"app": "anything"}, "redis", profileNs) { + t.Fatal("an empty podSelector selects all pods in the namespace, labelled or not") + } +} diff --git a/pkg/rulemanager/cel/libraries/containerprofilenetwork/wildcard_test.go b/pkg/rulemanager/cel/libraries/containerprofilenetwork/wildcard_test.go index e0a16c2299..bca5171f51 100644 --- a/pkg/rulemanager/cel/libraries/containerprofilenetwork/wildcard_test.go +++ b/pkg/rulemanager/cel/libraries/containerprofilenetwork/wildcard_test.go @@ -272,20 +272,16 @@ func TestWasAddressPortProtocolInEgress_PortWrapRejected(t *testing.T) { }, }, nil) - // See TestWasAddressPortProtocolInEgress_WithCIDR for the - // port/protocol regression note. The port-range guard ([0, 65535]) - // still applies — what's gone is port-specific matching: any in-range - // port matches if the address matches. cases := []struct { name string port int64 want bool }{ {"in-range hit", 443, true}, - {"in-range miss", 444, true}, // was: false (port mismatch). Now matches: address-only after projection-v1. - {"wrap-to-443 rejected", 4294967739, false}, // (1<<32)+443 — range guard fires - {"negative rejected", -1, false}, // range guard fires - {"too-large rejected", 65536, false}, // range guard fires + {"in-range miss", 444, false}, + {"wrap-to-443 rejected", 4294967739, false}, + {"negative rejected", -1, false}, + {"too-large rejected", 65536, false}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { @@ -350,27 +346,17 @@ func TestWasAddressPortProtocolInEgress_WithCIDR(t *testing.T) { }, }, nil) - // NOTE: upstream's projection-v1 (PR #799) explicitly drops port/protocol - // granularity from the address surface — the comment in network.go reads - // "port/protocol projection (AddressPortsByAddr) is out of scope for v1; - // degrade to address-only matching". So the matcher now only checks IP. - // - // Spec §4.7 still says ports[] is per-neighbor; the runtime gap is a - // known limitation flagged in the rebase commit. Test expectations - // updated to match runtime reality. Bringing port/protocol back is a - // follow-up: would need projection_apply to surface a per-address - // (port, protocol) set into ProjectedContainerProfile and the CEL - // helper to consult it. cases := []struct { observed string port int64 proto string want bool }{ - {"10.1.2.3", 443, "TCP", true}, // CIDR match (port/proto not enforced) - {"10.1.2.3", 80, "TCP", true}, // was: wrong port — now matches address-only - {"10.1.2.3", 443, "UDP", true}, // was: wrong protocol — now matches address-only - {"11.0.0.1", 443, "TCP", false}, // outside CIDR — still rejected + {"10.1.2.3", 443, "TCP", true}, + {"10.1.2.3", 80, "TCP", false}, + {"10.1.2.3", 443, "UDP", false}, + {"11.0.0.1", 443, "TCP", false}, + {"10.1.2.3", 443, "tcp", true}, } for _, tc := range cases { t.Run(tc.observed, func(t *testing.T) { diff --git a/pkg/rulemanager/cel/selector_compile_test.go b/pkg/rulemanager/cel/selector_compile_test.go new file mode 100644 index 0000000000..fbcb225d5b --- /dev/null +++ b/pkg/rulemanager/cel/selector_compile_test.go @@ -0,0 +1,41 @@ +package cel + +import ( + "testing" + "time" + + "github.com/goradd/maps" + "github.com/kubescape/node-agent/pkg/config" + "github.com/kubescape/node-agent/pkg/objectcache" + objectcachev1 "github.com/kubescape/node-agent/pkg/objectcache/v1" + "github.com/kubescape/node-agent/pkg/rulemanager/cel/libraries/cache" +) + +// TestCompileSelectorRules pins that the selector rules type-check against the +// real event object type: event.dstPodLabels is declared as a generic CEL map +// and must remain assignable to the was_selector_in_{ingress,egress} map param. +// Regression guard for the R0012 ingress rule that consumes the IG-enriched +// peer namespace + labels. +func TestCompileSelectorRules(t *testing.T) { + objCache := &objectcachev1.RuleObjectCacheMock{ + ContainerIDToSharedData: maps.NewSafeMap[string, *objectcache.WatchedContainerData](), + } + c, err := NewCEL(objCache, config.Config{ + CelConfigCache: cache.FunctionCacheConfig{MaxSize: 1000, TTL: time.Minute}, + }) + if err != nil { + t.Fatalf("NewCEL: %v", err) + } + + exprs := []string{ + `cp.was_selector_in_ingress(event.containerId, event.dstNamespace, event.dstPodLabels)`, + `cp.was_selector_in_egress(event.containerId, event.dstNamespace, event.dstPodLabels)`, + // The full R0012 ingress expression as bound in default-rules.yaml. + `event.pktType == 'HOST' && !event.dstAddr.startsWith('127.') && !cp.was_address_in_ingress(event.containerId, event.dstAddr) && !cp.was_selector_in_ingress(event.containerId, event.dstNamespace, event.dstPodLabels)`, + } + for _, e := range exprs { + if err := c.registerExpression(e); err != nil { + t.Fatalf("expression failed to compile: %q\n%v", e, err) + } + } +} diff --git a/pkg/utils/cel.go b/pkg/utils/cel.go index 39c3ad40f2..b95cc1c115 100644 --- a/pkg/utils/cel.go +++ b/pkg/utils/cel.go @@ -202,6 +202,35 @@ var CelFields = map[string]*celtypes.FieldType{ return celtypes.Int(x.Raw.GetDstPort()), nil }), }, + // dstNamespace / dstPodLabels carry the peer identity that IG's + // kubeipresolver resolves cluster-wide (independent of node-agent's + // node-local pod cache), so selector rules can match a peer on any node. + "dstNamespace": { + Type: celtypes.StringType, + IsSet: isSet, + GetFrom: ref.FieldGetter(func(target any) (any, error) { + x := target.(*xcel.Object[CelEvent]) + if x.Raw == nil { + return nil, errCelObjectNil + } + return celtypes.String(x.Raw.GetDstEndpoint().Namespace), nil + }), + }, + "dstPodLabels": { + Type: celtypes.MapType, + IsSet: isSet, + GetFrom: ref.FieldGetter(func(target any) (any, error) { + x := target.(*xcel.Object[CelEvent]) + if x.Raw == nil { + return nil, errCelObjectNil + } + pl := x.Raw.GetDstEndpoint().PodLabels + if pl == nil { + pl = map[string]string{} + } + return pl, nil + }), + }, "exepath": { Type: celtypes.StringType, IsSet: isSet, diff --git a/tests/chart/templates/node-agent/default-rules.yaml b/tests/chart/templates/node-agent/default-rules.yaml index 512b4d9ec8..d64cc0c044 100644 --- a/tests/chart/templates/node-agent/default-rules.yaml +++ b/tests/chart/templates/node-agent/default-rules.yaml @@ -313,7 +313,7 @@ spec: uniqueId: "event.dstAddr + '_' + string(event.dstPort) + '_' + event.proto" ruleExpression: - eventType: "network" - expression: "event.pktType == 'OUTGOING' && !net.is_private_ip(event.dstAddr) && !cp.was_address_in_egress(event.containerId, event.dstAddr)" + expression: "event.pktType == 'OUTGOING' && !event.dstAddr.startsWith('127.') && !cp.was_address_port_protocol_in_egress(event.containerId, event.dstAddr, event.dstPort, event.proto)" profileDependency: 0 profileDataRequired: egressAddresses: all diff --git a/tests/component_test.go b/tests/component_test.go index 154e441e8b..7514071faf 100644 --- a/tests/component_test.go +++ b/tests/component_test.go @@ -1192,6 +1192,19 @@ func Test_21_AlertOnPartialThenLearnNetworkTest(t *testing.T) { fusioncoreIP = "162.0.217.171" ) port80 := int32(80) + port53 := int32(53) + // R0011 excludes only loopback (maximally noisy by design), so authored + // profiles must allow the pod's own DNS egress to cluster DNS or every + // nslookup mints an R0011 that skews the before/after counts. + clusterDNS := v1beta1.NetworkNeighbor{ + Identifier: "cluster-dns", + Type: v1beta1.CommunicationTypeEgress, + IPAddresses: []string{"10.96.0.0/12"}, + Ports: []v1beta1.NetworkPort{ + {Name: "UDP-53", Protocol: v1beta1.ProtocolUDP, Port: &port53}, + {Name: "TCP-53", Protocol: v1beta1.ProtocolTCP, Port: &port53}, + }, + } ns := testutils.NewRandomNamespace() k8sClient := k8sinterface.NewKubernetesApi() @@ -1220,6 +1233,7 @@ func Test_21_AlertOnPartialThenLearnNetworkTest(t *testing.T) { IPAddress: fusioncoreIP, Ports: []v1beta1.NetworkPort{{Name: "TCP-80", Protocol: v1beta1.ProtocolTCP, Port: &port80}}, }, + clusterDNS, }, }, } @@ -1291,6 +1305,7 @@ func Test_21_AlertOnPartialThenLearnNetworkTest(t *testing.T) { IPAddress: subjectIP, Ports: []v1beta1.NetworkPort{{Name: "TCP-80", Protocol: v1beta1.ProtocolTCP, Port: &port80}}, }, + clusterDNS, } _, err = storageClient.ContainerProfiles(ns.Name).Update(context.Background(), cur, metav1.UpdateOptions{}) require.NoError(t, err, "update CP: add subject IP, remove canary domain") @@ -2718,6 +2733,51 @@ func Test_28_UserDefinedNetworkNeighborhood(t *testing.T) { "fusioncore.ai IP is in NN — should NOT fire R0011") }) + // 162.0.217.171 is allowed on TCP/80 only; :443 is a port violation → R0011. + t.Run("port_violation_different_port_R0011", func(t *testing.T) { + wl := setup(t) + stdout, stderr, err := wl.ExecIntoPod([]string{"curl", "-sm5", "-k", "https://162.0.217.171"}, "curl") + t.Logf("curl https://162.0.217.171 → err=%v stdout=%q stderr=%q", err, stdout, stderr) + alerts := waitAlerts(t, wl.Namespace) + logAlerts(t, alerts) + assert.GreaterOrEqual(t, countByRule(alerts, "R0011"), 1, + "egress to allowed IP 162.0.217.171 on non-allowed port 443 must fire R0011") + }) + + // 9.9.9.9 is allowlisted with port 0 (ANY); no port fires R0011. + t.Run("port_wildcard_zero_allows_any", func(t *testing.T) { + wl := setup(t) + wl.ExecIntoPod([]string{"curl", "-sm5", "http://9.9.9.9"}, "curl") + wl.ExecIntoPod([]string{"curl", "-sm5", "-k", "https://9.9.9.9"}, "curl") + alerts := waitAlerts(t, wl.Namespace) + logAlerts(t, alerts) + assert.Equal(t, 0, countByRule(alerts, "R0011"), + "9.9.9.9 allowlisted on port 0 (any) must not fire R0011 on any port") + }) + + // 208.67.222.222 is allowlisted with no ports stanza (ANY); no port fires R0011. + t.Run("port_wildcard_empty_stanza_allows_any", func(t *testing.T) { + wl := setup(t) + wl.ExecIntoPod([]string{"curl", "-sm5", "http://208.67.222.222"}, "curl") + wl.ExecIntoPod([]string{"curl", "-sm5", "-k", "https://208.67.222.222"}, "curl") + alerts := waitAlerts(t, wl.Namespace) + logAlerts(t, alerts) + assert.Equal(t, 0, countByRule(alerts, "R0011"), + "208.67.222.222 allowlisted with empty ports stanza (any) must not fire R0011 on any port") + }) + + // Internal peer 10.96.0.1 (kube-api) is allowlisted on TCP/443 only; :80 is a port violation → R0011. + t.Run("internal_port_violation_R0011", func(t *testing.T) { + wl := setup(t) + wl.ExecIntoPod([]string{"curl", "-sm5", "-k", "https://10.96.0.1"}, "curl") + stdout, stderr, err := wl.ExecIntoPod([]string{"curl", "-sm5", "http://10.96.0.1"}, "curl") + t.Logf("curl http://10.96.0.1:80 → err=%v stdout=%q stderr=%q", err, stdout, stderr) + alerts := waitAlerts(t, wl.Namespace) + logAlerts(t, alerts) + assert.GreaterOrEqual(t, countByRule(alerts, "R0011"), 1, + "egress to internal IP 10.96.0.1 on non-allowed port 80 must fire R0011") + }) + // --------------------------------------------------------------- // 28b. Unknown domains — domains NOT in the NN → R0005. // Uses both nslookup (pure DNS) and curl (DNS + TCP). diff --git a/tests/resources/containerprofile-user-defined-network.yaml b/tests/resources/containerprofile-user-defined-network.yaml index f2f6edda1c..94e7928940 100644 --- a/tests/resources/containerprofile-user-defined-network.yaml +++ b/tests/resources/containerprofile-user-defined-network.yaml @@ -58,3 +58,36 @@ spec: - name: TCP-80 protocol: TCP port: 80 + # R0011 excludes only loopback (maximally noisy by design): allow the pod's + # own DNS egress to cluster DNS or every nslookup/curl resolution mints R0011. + - identifier: cluster-dns + type: internal + ipAddresses: + - 10.96.0.0/12 + ports: + - name: UDP-53 + protocol: UDP + port: 53 + - name: TCP-53 + protocol: TCP + port: 53 + - identifier: wildcard-zero-port + type: external + ipAddress: 9.9.9.9 + ports: + - name: TCP-any + protocol: TCP + port: 0 + - identifier: wildcard-empty-ports + type: external + ipAddress: 208.67.222.222 + - identifier: cluster-dns + type: internal + ipAddress: 10.96.0.10 + - identifier: kube-api + type: internal + ipAddress: 10.96.0.1 + ports: + - name: TCP-443 + protocol: TCP + port: 443 diff --git a/tests/resources/network_fixture_lint_test.go b/tests/resources/network_fixture_lint_test.go index a7ec6cd0e2..5efacdcf5a 100644 --- a/tests/resources/network_fixture_lint_test.go +++ b/tests/resources/network_fixture_lint_test.go @@ -205,8 +205,9 @@ func lintEndpoint(dir string, e netEndpoint, add func(rule, msg string)) { if p.Protocol != "TCP" && p.Protocol != "UDP" { add("R-NN-20", where(fmt.Sprintf("port %q protocol %q is not TCP|UDP", p.Name, p.Protocol))) } - if p.Port < 1 || p.Port > 65535 { - add("R-NN-20", where(fmt.Sprintf("port %q value %d out of range 1..65535", p.Name, p.Port))) + // Port 0 is the any-port wildcard (matches R0011/R0012 port semantics). + if p.Port != 0 && (p.Port < 1 || p.Port > 65535) { + add("R-NN-20", where(fmt.Sprintf("port %q value %d out of range 1..65535 (0 = any)", p.Name, p.Port))) } } } diff --git a/tests/scripts/issue79-eol-ladder.sh b/tests/scripts/issue79-eol-ladder.sh new file mode 100755 index 0000000000..66c3ec9ea3 --- /dev/null +++ b/tests/scripts/issue79-eol-ladder.sh @@ -0,0 +1,107 @@ +#!/usr/bin/env bash +# issue79-eol-ladder.sh — E2E ladder for exec-event delivery at container +# end-of-life (issue #79, acceptance tests T4/T5). +# +# Measures, over N iterations on a live cluster running the kubescape stack: +# T4: the forbidden terminal exec of the init container `setup` +# (sh -c "sleep ; /usr/bin/id") produces an R0001 alert — N/N. +# T5: the forbidden terminal execs of an ephemeral container `debug` +# (sh -c "sleep ; /usr/bin/whoami; /usr/bin/id") produce R0001 — +# N/N. +# +# Prerequisites: +# - kubectl context pointing at the test cluster +# - kubescape stack deployed (node-agent image under test), namespace +# `kubescape` +# - fixtures from tests/resources: mc37-cp-doc.yaml (grouped profile: +# app allows id; setup forbids id; debug forbids id/whoami) +# +# Usage: issue79-eol-ladder.sh [ITERATIONS] [RUNWAY_SECONDS] +set -euo pipefail + +ITERATIONS="${1:-5}" +RUNWAY="${2:-30}" +NS="node-agent-test-eol" +KS_NS="kubescape" +FIXTURE_DIR="$(cd "$(dirname "$0")/../resources" && pwd)" + +t4_pass=0 +t5_pass=0 + +log() { echo "[$(date -u +%H:%M:%S)] $*"; } + +node_agent_pod() { + kubectl -n "$KS_NS" get pods -l app.kubernetes.io/name=node-agent \ + -o jsonpath='{.items[0].metadata.name}' +} + +# Count R0001 alerts for a container name in node-agent logs since a given +# RFC3339 timestamp. +count_r0001() { + # Read EVERY node-agent pod (DaemonSet - the workload may land on any node) + # and match the alert JSON's containerName field explicitly. + local container="$1" since="$2" total=0 n + for pod in $(kubectl -n "$KS_NS" get pods -o name | grep node-agent); do + n=$(kubectl -n "$KS_NS" logs "${pod#pod/}" -c node-agent --since-time="$since" 2>/dev/null \ + | grep '"RuleID":"R0001"' | grep -c "\"containerName\":\"${container}\"" || true) + total=$((total + n)) + done + echo "$total" +} + +kubectl get ns "$NS" >/dev/null 2>&1 || kubectl create ns "$NS" +kubectl -n "$NS" apply -f "$FIXTURE_DIR/mc37-cp-doc.yaml" + +for i in $(seq 1 "$ITERATIONS"); do + log "=== iteration $i/$ITERATIONS (runway ${RUNWAY}s) ===" + kubectl -n "$NS" delete deployment mc37-deployment --ignore-not-found --wait + sleep 3 + iter_start="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + + # Deploy with the requested init runway. + sed "s/sleep 100/sleep ${RUNWAY}/" \ + "$FIXTURE_DIR/mc37-multi-subtype-userdefined-deployment.yaml" \ + | kubectl -n "$NS" apply -f - + + # Wait for the pod: init phase (runway) + margin. + log "waiting for pod Ready (init runway ${RUNWAY}s)..." + kubectl -n "$NS" rollout status deploy/mc37-deployment --timeout="$((RUNWAY + 150))s" + pod="$(kubectl -n "$NS" get pod -l app=mc37 -o jsonpath='{.items[0].metadata.name}')" + + # T4: the init terminal exec happened just before the pod became Ready. + # Give the pipeline a moment, then count. + sleep 10 + init_r0001="$(count_r0001 setup "$iter_start")" + if [ "${init_r0001:-0}" -gt 0 ]; then + t4_pass=$((t4_pass + 1)); log "T4 init: PASS (R0001 setup=${init_r0001})" + else + log "T4 init: FAIL (R0001 setup=0)" + fi + + # T5: attach ephemeral container with a terminal forbidden exec. + eph_start="$(date -u +%Y-%m-%dT%H:%M:%SZ)" + eph_runway=20 + kubectl -n "$NS" debug "$pod" --image=debian:12-slim --container=debug \ + --profile=general -- sh -c "sleep ${eph_runway}; /usr/bin/whoami; /usr/bin/id" \ + >/dev/null + log "waiting for ephemeral container debug to terminate..." + for _ in $(seq 1 $((eph_runway + 60))); do + state="$(kubectl -n "$NS" get pod "$pod" \ + -o jsonpath='{.status.ephemeralContainerStatuses[?(@.name=="debug")].state.terminated.exitCode}' 2>/dev/null || true)" + [ -n "$state" ] && break + sleep 2 + done + sleep 10 + eph_r0001="$(count_r0001 debug "$eph_start")" + if [ "${eph_r0001:-0}" -gt 0 ]; then + t5_pass=$((t5_pass + 1)); log "T5 ephemeral: PASS (R0001 debug=${eph_r0001})" + else + log "T5 ephemeral: FAIL (R0001 debug=0)" + fi +done + +echo +echo "==== issue #79 EOL ladder result ====" +echo "T4 (init terminal exec R0001): ${t4_pass}/${ITERATIONS}" +echo "T5 (ephemeral terminal exec R0001): ${t5_pass}/${ITERATIONS}" +[ "$t4_pass" -eq "$ITERATIONS" ] && [ "$t5_pass" -eq "$ITERATIONS" ] From 456d267f5892cf4686514a8769c25d731ca3e1e3 Mon Sep 17 00:00:00 2001 From: entlein Date: Tue, 18 Aug 2026 19:08:27 +0200 Subject: [PATCH 03/19] Allow alert from unexpected Ports, allow Port=0 as intentional wildcard Signed-off-by: entlein --- tests/scripts/issue79-eol-ladder.sh | 107 ---------------------------- 1 file changed, 107 deletions(-) delete mode 100755 tests/scripts/issue79-eol-ladder.sh diff --git a/tests/scripts/issue79-eol-ladder.sh b/tests/scripts/issue79-eol-ladder.sh deleted file mode 100755 index 66c3ec9ea3..0000000000 --- a/tests/scripts/issue79-eol-ladder.sh +++ /dev/null @@ -1,107 +0,0 @@ -#!/usr/bin/env bash -# issue79-eol-ladder.sh — E2E ladder for exec-event delivery at container -# end-of-life (issue #79, acceptance tests T4/T5). -# -# Measures, over N iterations on a live cluster running the kubescape stack: -# T4: the forbidden terminal exec of the init container `setup` -# (sh -c "sleep ; /usr/bin/id") produces an R0001 alert — N/N. -# T5: the forbidden terminal execs of an ephemeral container `debug` -# (sh -c "sleep ; /usr/bin/whoami; /usr/bin/id") produce R0001 — -# N/N. -# -# Prerequisites: -# - kubectl context pointing at the test cluster -# - kubescape stack deployed (node-agent image under test), namespace -# `kubescape` -# - fixtures from tests/resources: mc37-cp-doc.yaml (grouped profile: -# app allows id; setup forbids id; debug forbids id/whoami) -# -# Usage: issue79-eol-ladder.sh [ITERATIONS] [RUNWAY_SECONDS] -set -euo pipefail - -ITERATIONS="${1:-5}" -RUNWAY="${2:-30}" -NS="node-agent-test-eol" -KS_NS="kubescape" -FIXTURE_DIR="$(cd "$(dirname "$0")/../resources" && pwd)" - -t4_pass=0 -t5_pass=0 - -log() { echo "[$(date -u +%H:%M:%S)] $*"; } - -node_agent_pod() { - kubectl -n "$KS_NS" get pods -l app.kubernetes.io/name=node-agent \ - -o jsonpath='{.items[0].metadata.name}' -} - -# Count R0001 alerts for a container name in node-agent logs since a given -# RFC3339 timestamp. -count_r0001() { - # Read EVERY node-agent pod (DaemonSet - the workload may land on any node) - # and match the alert JSON's containerName field explicitly. - local container="$1" since="$2" total=0 n - for pod in $(kubectl -n "$KS_NS" get pods -o name | grep node-agent); do - n=$(kubectl -n "$KS_NS" logs "${pod#pod/}" -c node-agent --since-time="$since" 2>/dev/null \ - | grep '"RuleID":"R0001"' | grep -c "\"containerName\":\"${container}\"" || true) - total=$((total + n)) - done - echo "$total" -} - -kubectl get ns "$NS" >/dev/null 2>&1 || kubectl create ns "$NS" -kubectl -n "$NS" apply -f "$FIXTURE_DIR/mc37-cp-doc.yaml" - -for i in $(seq 1 "$ITERATIONS"); do - log "=== iteration $i/$ITERATIONS (runway ${RUNWAY}s) ===" - kubectl -n "$NS" delete deployment mc37-deployment --ignore-not-found --wait - sleep 3 - iter_start="$(date -u +%Y-%m-%dT%H:%M:%SZ)" - - # Deploy with the requested init runway. - sed "s/sleep 100/sleep ${RUNWAY}/" \ - "$FIXTURE_DIR/mc37-multi-subtype-userdefined-deployment.yaml" \ - | kubectl -n "$NS" apply -f - - - # Wait for the pod: init phase (runway) + margin. - log "waiting for pod Ready (init runway ${RUNWAY}s)..." - kubectl -n "$NS" rollout status deploy/mc37-deployment --timeout="$((RUNWAY + 150))s" - pod="$(kubectl -n "$NS" get pod -l app=mc37 -o jsonpath='{.items[0].metadata.name}')" - - # T4: the init terminal exec happened just before the pod became Ready. - # Give the pipeline a moment, then count. - sleep 10 - init_r0001="$(count_r0001 setup "$iter_start")" - if [ "${init_r0001:-0}" -gt 0 ]; then - t4_pass=$((t4_pass + 1)); log "T4 init: PASS (R0001 setup=${init_r0001})" - else - log "T4 init: FAIL (R0001 setup=0)" - fi - - # T5: attach ephemeral container with a terminal forbidden exec. - eph_start="$(date -u +%Y-%m-%dT%H:%M:%SZ)" - eph_runway=20 - kubectl -n "$NS" debug "$pod" --image=debian:12-slim --container=debug \ - --profile=general -- sh -c "sleep ${eph_runway}; /usr/bin/whoami; /usr/bin/id" \ - >/dev/null - log "waiting for ephemeral container debug to terminate..." - for _ in $(seq 1 $((eph_runway + 60))); do - state="$(kubectl -n "$NS" get pod "$pod" \ - -o jsonpath='{.status.ephemeralContainerStatuses[?(@.name=="debug")].state.terminated.exitCode}' 2>/dev/null || true)" - [ -n "$state" ] && break - sleep 2 - done - sleep 10 - eph_r0001="$(count_r0001 debug "$eph_start")" - if [ "${eph_r0001:-0}" -gt 0 ]; then - t5_pass=$((t5_pass + 1)); log "T5 ephemeral: PASS (R0001 debug=${eph_r0001})" - else - log "T5 ephemeral: FAIL (R0001 debug=0)" - fi -done - -echo -echo "==== issue #79 EOL ladder result ====" -echo "T4 (init terminal exec R0001): ${t4_pass}/${ITERATIONS}" -echo "T5 (ephemeral terminal exec R0001): ${t5_pass}/${ITERATIONS}" -[ "$t4_pass" -eq "$ITERATIONS" ] && [ "$t5_pass" -eq "$ITERATIONS" ] From ffb22be5cbbe35a1059e59b9789f484462cc7612 Mon Sep 17 00:00:00 2001 From: entlein Date: Thu, 20 Aug 2026 10:57:47 +0200 Subject: [PATCH 04/19] remove explicit wildcard, declare port as non-mandatory, keep the alert if delcared and violated Signed-off-by: entlein --- pkg/objectcache/addr_ports_test.go | 43 +++++++++++++++++++ pkg/objectcache/projection_types.go | 13 +++--- .../containerprofilenetwork/network.go | 4 +- .../containerprofilenetwork/network_test.go | 7 +-- .../port_protocol_test.go | 29 ++++++++++++- tests/component_test.go | 9 ++-- 6 files changed, 90 insertions(+), 15 deletions(-) create mode 100644 pkg/objectcache/addr_ports_test.go diff --git a/pkg/objectcache/addr_ports_test.go b/pkg/objectcache/addr_ports_test.go new file mode 100644 index 0000000000..d1dd2ee88d --- /dev/null +++ b/pkg/objectcache/addr_ports_test.go @@ -0,0 +1,43 @@ +package objectcache + +import ( + "testing" + + "github.com/kubescape/storage/pkg/apis/softwarecomposition/v1beta1" + "github.com/stretchr/testify/assert" + "k8s.io/utils/ptr" +) + +func np(proto string, p int32) v1beta1.NetworkPort { + return v1beta1.NetworkPort{Protocol: v1beta1.Protocol(proto), Port: ptr.To(p)} +} + +func TestExtractAddrPorts_ZeroPortIsALiteralNotAWildcard(t *testing.T) { + groups := ExtractAddrPorts([]v1beta1.NetworkNeighbor{ + {IPAddresses: []string{"10.0.5.9"}, Ports: []v1beta1.NetworkPort{np("TCP", 0), np("UDP", 53)}}, + }) + assert.Len(t, groups, 1) + assert.NotNil(t, groups[0].Ports, "a zero-port entry must not collapse the entry to fully open") + assert.Contains(t, groups[0].Ports, PortKey("TCP", 0)) + assert.Contains(t, groups[0].Ports, PortKey("UDP", 53)) + assert.Len(t, groups[0].Ports, 2) +} + +func TestExtractAddrPorts_AbsentStanzaIsTheOnlyWildcard(t *testing.T) { + groups := ExtractAddrPorts([]v1beta1.NetworkNeighbor{ + {IPAddresses: []string{"93.184.216.34"}}, + }) + assert.Len(t, groups, 1) + assert.Nil(t, groups[0].Ports) +} + +func TestExtractAddrPorts_NilPortEntryContributesNothing(t *testing.T) { + groups := ExtractAddrPorts([]v1beta1.NetworkNeighbor{ + {IPAddresses: []string{"10.1.2.3"}, Ports: []v1beta1.NetworkPort{{Protocol: "TCP", Port: nil}, np("UDP", 53)}}, + }) + assert.Len(t, groups, 1) + assert.NotNil(t, groups[0].Ports) + assert.NotContains(t, groups[0].Ports, PortKey("TCP", 0)) + assert.Contains(t, groups[0].Ports, PortKey("UDP", 53)) + assert.Len(t, groups[0].Ports, 1) +} diff --git a/pkg/objectcache/projection_types.go b/pkg/objectcache/projection_types.go index a8f7e4944a..4d81dd3a60 100644 --- a/pkg/objectcache/projection_types.go +++ b/pkg/objectcache/projection_types.go @@ -57,7 +57,9 @@ type FieldSpec struct { } // AddrPortGroup pairs one neighbor entry's addresses with its allowed ports. -// Empty Ports means any port (port 0 or no ports declared = wildcard). +// Ports == nil means the neighbor declared no ports stanza (indistinguishable +// from an empty one after a storage round-trip) and matches any port; a +// non-empty map matches only its literal (protocol, port) keys. type AddrPortGroup struct { Addrs []string Ports map[string]struct{} @@ -79,16 +81,17 @@ func ExtractAddrPorts(neighbors []v1beta1.NetworkNeighbor) []AddrPortGroup { if len(addrs) == 0 { continue } + // The only port wildcard is an absent (or empty — protobuf cannot tell + // them apart) ports stanza. A listed entry always restricts: an explicit + // port (0 included) is a literal, a nil port contributes nothing. ports := make(map[string]struct{}, len(n.Ports)) - wildcard := len(n.Ports) == 0 for _, p := range n.Ports { - if p.Port == nil || *p.Port == 0 { - wildcard = true + if p.Port == nil { continue } ports[PortKey(string(p.Protocol), *p.Port)] = struct{}{} } - if wildcard { + if len(n.Ports) == 0 { ports = nil } groups = append(groups, AddrPortGroup{Addrs: addrs, Ports: ports}) diff --git a/pkg/rulemanager/cel/libraries/containerprofilenetwork/network.go b/pkg/rulemanager/cel/libraries/containerprofilenetwork/network.go index ff1bbed68e..723827e982 100644 --- a/pkg/rulemanager/cel/libraries/containerprofilenetwork/network.go +++ b/pkg/rulemanager/cel/libraries/containerprofilenetwork/network.go @@ -64,7 +64,7 @@ func matchIPField(field *objectcache.ProjectedField, observed string) bool { // matchAddrPort reports whether observed (address, protocol, port) falls within // any single neighbor entry: its addresses match AND the entry allows the port -// (empty Ports = any port). Address-only entries thus stay wildcard on ports. +// (nil Ports = no ports stanza = any port; a populated map matches literal keys only). func matchAddrPort(groups []objectcache.AddrPortGroup, address, protocol string, port int32) bool { if address == "" { return false @@ -75,7 +75,7 @@ func matchAddrPort(groups []objectcache.AddrPortGroup, address, protocol string, if !networkmatch.MatchIP(g.Addrs, address) { continue } - if len(g.Ports) == 0 { + if g.Ports == nil { return true } if _, ok := g.Ports[key]; ok { diff --git a/pkg/rulemanager/cel/libraries/containerprofilenetwork/network_test.go b/pkg/rulemanager/cel/libraries/containerprofilenetwork/network_test.go index 5446f63a88..e94ac732e7 100644 --- a/pkg/rulemanager/cel/libraries/containerprofilenetwork/network_test.go +++ b/pkg/rulemanager/cel/libraries/containerprofilenetwork/network_test.go @@ -401,14 +401,15 @@ func TestWasAddressPortProtocolWithNilPort(t *testing.T) { functionCache: cache.NewFunctionCache(cache.DefaultFunctionCacheConfig()), } - // nil port in a profile entry = any-port wildcard for that entry's addresses. + // A listed entry with a nil port contributes nothing: the only port + // wildcard is an ABSENT ports stanza, so these addresses stay restricted. result := lib.wasAddressPortProtocolInEgress( types.String("test-container-id"), types.String("192.168.1.100"), types.Int(80), types.String("TCP"), ) - assert.Equal(t, types.Bool(true), result) + assert.Equal(t, types.Bool(false), result) result = lib.wasAddressPortProtocolInIngress( types.String("test-container-id"), @@ -416,5 +417,5 @@ func TestWasAddressPortProtocolWithNilPort(t *testing.T) { types.Int(8080), types.String("TCP"), ) - assert.Equal(t, types.Bool(true), result) + assert.Equal(t, types.Bool(false), result) } diff --git a/pkg/rulemanager/cel/libraries/containerprofilenetwork/port_protocol_test.go b/pkg/rulemanager/cel/libraries/containerprofilenetwork/port_protocol_test.go index 1750ac385c..20773b1b83 100644 --- a/pkg/rulemanager/cel/libraries/containerprofilenetwork/port_protocol_test.go +++ b/pkg/rulemanager/cel/libraries/containerprofilenetwork/port_protocol_test.go @@ -34,7 +34,8 @@ func TestWasAddressPortProtocolInEgress_PortWildcard(t *testing.T) { zeroPort := buildLibWithContainer(t, []v1beta1.NetworkNeighbor{ {IPAddresses: []string{"93.184.216.34"}, Ports: []v1beta1.NetworkPort{port("TCP", 0)}}, }, nil) - assert.Equal(t, types.Bool(true), evalEgressPort(zeroPort, "93.184.216.34", 8080, "TCP")) + assert.Equal(t, types.Bool(false), evalEgressPort(zeroPort, "93.184.216.34", 8080, "TCP"), + "an explicit port 0 is a literal, not a wildcard: only an absent ports stanza opens the entry") } func TestWasAddressPortProtocolInIngress_Symmetric(t *testing.T) { @@ -45,3 +46,29 @@ func TestWasAddressPortProtocolInIngress_Symmetric(t *testing.T) { assert.Equal(t, types.Bool(false), evalIngressPort(lib, "172.16.5.9", 5432, "TCP")) assert.Equal(t, types.Bool(false), evalIngressPort(lib, "10.0.0.1", 6379, "TCP")) } + +func TestWasAddressPortProtocolInEgress_ZeroPortIsNotAWildcard(t *testing.T) { + lib := buildLibWithContainer(t, []v1beta1.NetworkNeighbor{ + {IPAddresses: []string{"93.184.216.34"}, Ports: []v1beta1.NetworkPort{port("TCP", 0)}}, + }, nil) + assert.Equal(t, types.Bool(false), evalEgressPort(lib, "93.184.216.34", 8080, "TCP")) + assert.Equal(t, types.Bool(false), evalEgressPort(lib, "93.184.216.34", 53, "UDP")) +} + +func TestWasAddressPortProtocolInEgress_MixedZeroPortKeepsEveryProtocolRestricted(t *testing.T) { + lib := buildLibWithContainer(t, []v1beta1.NetworkNeighbor{ + {IPAddresses: []string{"10.0.5.9"}, Ports: []v1beta1.NetworkPort{port("TCP", 0), port("UDP", 53)}}, + }, nil) + assert.Equal(t, types.Bool(false), evalEgressPort(lib, "10.0.5.9", 9999, "TCP"), + "an explicit {TCP,0} entry no longer opens TCP: wildcard is expressed only by omitting the ports stanza") + assert.Equal(t, types.Bool(true), evalEgressPort(lib, "10.0.5.9", 53, "UDP")) + assert.Equal(t, types.Bool(false), evalEgressPort(lib, "10.0.5.9", 54, "UDP")) +} + +func TestWasAddressPortProtocolInEgress_NilPortEntryContributesNothing(t *testing.T) { + lib := buildLibWithContainer(t, []v1beta1.NetworkNeighbor{ + {IPAddresses: []string{"10.0.5.9"}, Ports: []v1beta1.NetworkPort{{Protocol: "TCP"}, port("UDP", 53)}}, + }, nil) + assert.Equal(t, types.Bool(false), evalEgressPort(lib, "10.0.5.9", 8080, "TCP")) + assert.Equal(t, types.Bool(true), evalEgressPort(lib, "10.0.5.9", 53, "UDP")) +} diff --git a/tests/component_test.go b/tests/component_test.go index 7514071faf..d88e768b33 100644 --- a/tests/component_test.go +++ b/tests/component_test.go @@ -2744,15 +2744,16 @@ func Test_28_UserDefinedNetworkNeighborhood(t *testing.T) { "egress to allowed IP 162.0.217.171 on non-allowed port 443 must fire R0011") }) - // 9.9.9.9 is allowlisted with port 0 (ANY); no port fires R0011. - t.Run("port_wildcard_zero_allows_any", func(t *testing.T) { + // 9.9.9.9 is listed with an explicit port 0 — a literal, NOT a wildcard: + // the only port wildcard is an absent ports stanza, so :80/:443 violate. + t.Run("port_zero_is_literal_not_wildcard", func(t *testing.T) { wl := setup(t) wl.ExecIntoPod([]string{"curl", "-sm5", "http://9.9.9.9"}, "curl") wl.ExecIntoPod([]string{"curl", "-sm5", "-k", "https://9.9.9.9"}, "curl") alerts := waitAlerts(t, wl.Namespace) logAlerts(t, alerts) - assert.Equal(t, 0, countByRule(alerts, "R0011"), - "9.9.9.9 allowlisted on port 0 (any) must not fire R0011 on any port") + assert.GreaterOrEqual(t, countByRule(alerts, "R0011"), 1, + "an explicit port-0 entry must not open 9.9.9.9 on other ports") }) // 208.67.222.222 is allowlisted with no ports stanza (ANY); no port fires R0011. From 283098d8070f9e0fc194c698b8619f32cbfb4816 Mon Sep 17 00:00:00 2001 From: tanzee Date: Sun, 23 Aug 2026 20:35:53 +0200 Subject: [PATCH 05/19] feat(cel/network): serviceRef/serviceSelector/host neighbor resolution Let a ContainerProfile allowlist cluster-infrastructure egress/ingress by Service name, Service label selector, or host entity instead of a broad ipAddresses serviceCIDR that blinds R0011/R0012 to lateral movement. Each serviceRef/serviceSelector/entity neighbor resolves at projection time to the concrete ClusterIP + backing-endpoint (or node/gateway) IPs it stands for, carrying its own ports, and is appended as an ordinary selector-free ipAddresses neighbor. The existing port-sensitive address matcher enforces it unchanged; unresolved selectors contribute nothing (never a match-all). - pkg/networkpeer: Resolve/Matches/ResolveIPs + Lister over Service, EndpointSlice and Node informers, with a generation counter so a profile projected before the informers synced re-projects once the view changes. - objectcache/reconciler: mark profiles that use service resolution and re-project them when the lister generation advances; plain profiles keep the identical old fast-skip path. - cmd/main.go: cluster-wide Service/EndpointSlice informers + a node-scoped Node informer, started non-blocking (no WaitForCacheSync on the hot path). - fail closed on ServiceSelector MatchExpressions / empty matchLabels and on any namespaceSelector other than kubernetes.io/metadata.name=. - Test_50 component test (serviceRef egress allowed, external egress still fires R0011) + resolve/expand/lister unit tests + fixture-lint R-NN-12 extended to accept the new target fields. Depends on the storage schema fields ServiceRefNamespace/ServiceRefName/ ServiceSelector/Entity; go.mod pins the fork's storage until the companion upstream storage PR lands. Signed-off-by: tanzee --- .github/workflows/component-tests.yaml | 3 +- cmd/main.go | 47 ++++ go.mod | 2 + go.sum | 4 +- pkg/networkpeer/expand.go | 146 ++++++++++++ pkg/networkpeer/expand_test.go | 209 ++++++++++++++++++ pkg/networkpeer/lister.go | 167 ++++++++++++++ pkg/networkpeer/lister_test.go | 147 ++++++++++++ pkg/networkpeer/resolve.go | 197 +++++++++++++++++ pkg/networkpeer/resolve_test.go | 204 +++++++++++++++++ .../containerprofilecache.go | 39 +++- .../containerprofilecache/reconciler.go | 40 ++-- tests/component_test.go | 89 ++++++++ .../containerprofile-serviceref-network.yaml | 48 ++++ tests/resources/network_fixture_lint_test.go | 12 +- .../serviceref-client-deployment.yaml | 21 ++ 16 files changed, 1351 insertions(+), 24 deletions(-) create mode 100644 pkg/networkpeer/expand.go create mode 100644 pkg/networkpeer/expand_test.go create mode 100644 pkg/networkpeer/lister.go create mode 100644 pkg/networkpeer/lister_test.go create mode 100644 pkg/networkpeer/resolve.go create mode 100644 pkg/networkpeer/resolve_test.go create mode 100644 tests/resources/containerprofile-serviceref-network.yaml create mode 100644 tests/resources/serviceref-client-deployment.yaml diff --git a/.github/workflows/component-tests.yaml b/.github/workflows/component-tests.yaml index f3325394eb..f89cfb5639 100644 --- a/.github/workflows/component-tests.yaml +++ b/.github/workflows/component-tests.yaml @@ -107,7 +107,8 @@ jobs: Test_36_MultiContainerPerContainerBinding, Test_43_RelativeOpenPathResolution, Test_48_MultiSubtypeGroupedProfileDocument, - Test_49_EphemeralContainerFullTreatment + Test_49_EphemeralContainerFullTreatment, + Test_50_ServiceRefNetworkNeighbor ] steps: - name: Checkout code diff --git a/cmd/main.go b/cmd/main.go index e41b256727..92fd665d24 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -39,11 +39,13 @@ import ( "github.com/kubescape/node-agent/pkg/malwaremanager" malwaremanagerv1 "github.com/kubescape/node-agent/pkg/malwaremanager/v1" otelmetrics "github.com/kubescape/node-agent/pkg/metricsmanager/otel" + "github.com/kubescape/node-agent/pkg/networkpeer" "github.com/kubescape/node-agent/pkg/networkstream" networkstreamv1 "github.com/kubescape/node-agent/pkg/networkstream/v1" "github.com/kubescape/node-agent/pkg/nodeprofilemanager" nodeprofilemanagerv1 "github.com/kubescape/node-agent/pkg/nodeprofilemanager/v1" "github.com/kubescape/node-agent/pkg/objectcache" + "github.com/kubescape/node-agent/pkg/objectcache/containerprofilecache" "github.com/kubescape/node-agent/pkg/objectcache/dnscache" "github.com/kubescape/node-agent/pkg/objectcache/k8scache" @@ -72,6 +74,8 @@ import ( "github.com/kubescape/node-agent/pkg/watcher/seccompprofilewatcher" goruntime "go.opentelemetry.io/contrib/instrumentation/runtime" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/informers" + toolscache "k8s.io/client-go/tools/cache" ) func main() { @@ -321,6 +325,49 @@ func main() { ruleBindingCache.AddNotifier(&ruleBindingNotify) cpc := containerprofilecache.NewContainerProfileCache(cfg, storageClient, k8sObjectCache, metricsProvider) + // Resolve serviceRef/serviceSelector/entity network neighbors against + // live cluster state (Service ClusterIPs + endpoints, Node IPs + CNI + // gateway) at projection time. Services and EndpointSlices are watched + // cluster-wide (a profile may reference any namespace's Service); the + // Node informer is field-selected to this agent's own node — the "host" + // entity is local, and a cluster-wide Node watch on every DaemonSet pod + // is O(nodes^2) traffic for no benefit. + svcInformers := informers.NewSharedInformerFactory(k8sClient.GetKubernetesClient(), 0) + nodeInformers := informers.NewSharedInformerFactoryWithOptions( + k8sClient.GetKubernetesClient(), 0, + informers.WithTweakListOptions(func(o *metav1.ListOptions) { + o.FieldSelector = "metadata.name=" + cfg.NodeName + }), + ) + serviceLister := networkpeer.NewInformerLister( + svcInformers.Core().V1().Services().Lister(), + svcInformers.Discovery().V1().EndpointSlices().Lister(), + nodeInformers.Core().V1().Nodes().Lister(), + cfg.NodeName, + ) + // Advance the lister generation on any Service/EndpointSlice/Node change, + // so the reconciler re-projects serviceRef/entity profiles when the + // cluster view moves (endpoint churn, or caches that fill after startup). + // Per-event cost is a single atomic increment; the re-projection itself + // is coalesced onto the reconcile tick, and only serviceRef-using + // profiles are eligible. + bump := toolscache.ResourceEventHandlerFuncs{ + AddFunc: func(interface{}) { serviceLister.Bump() }, + UpdateFunc: func(_, _ interface{}) { serviceLister.Bump() }, + DeleteFunc: func(interface{}) { serviceLister.Bump() }, + } + _, _ = svcInformers.Core().V1().Services().Informer().AddEventHandler(bump) + _, _ = svcInformers.Discovery().V1().EndpointSlices().Informer().AddEventHandler(bump) + _, _ = nodeInformers.Core().V1().Nodes().Informer().AddEventHandler(bump) + // Start the informers and hand the lister over WITHOUT blocking on cache + // sync: node-agent's core startup (container watcher, profiling) must not + // wait on these, and a bounded wait here previously delayed learning + // enough to trip the tight completion budget of Test_22. The caches fill + // in the background; serviceRef/entity neighbors resolve on the next + // reconcile once populated. nil-until-set is a no-op in projection. + svcInformers.Start(ctx.Done()) + nodeInformers.Start(ctx.Done()) + cpc.SetServiceLister(serviceLister) cpc.Start(ctx) if cpm, ok := containerProfileManager.(*containerprofilemanagerv1.ContainerProfileManager); ok { cpm.SetCompletionNotifier(cpc) diff --git a/go.mod b/go.mod index 808f11127a..4013bc3b28 100644 --- a/go.mod +++ b/go.mod @@ -479,3 +479,5 @@ replace github.com/anchore/syft => github.com/kubescape/syft v1.32.0-ks.2 replace github.com/anchore/stereoscope => github.com/anchore/stereoscope v0.1.9 replace github.com/opencontainers/runtime-spec => github.com/opencontainers/runtime-spec v1.2.1 + +replace github.com/kubescape/storage => github.com/k8sstormcenter/storage v0.0.240-0.20260823123818-6f3a2ee6385d diff --git a/go.sum b/go.sum index b239690414..0c78ed4da6 100644 --- a/go.sum +++ b/go.sum @@ -859,6 +859,8 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/k8sstormcenter/storage v0.0.240-0.20260823123818-6f3a2ee6385d h1:4d7cpcoXpp8ZJXV2W/ubX5WW78gn9diy9qvLFbROvV0= +github.com/k8sstormcenter/storage v0.0.240-0.20260823123818-6f3a2ee6385d/go.mod h1:d/1hqWPda2clsjx2wmQgysnB5dThIo3rDKP7RWx+v+M= github.com/kastenhq/goversion v0.0.0-20230811215019-93b2f8823953 h1:WdAeg/imY2JFPc/9CST4bZ80nNJbiBFCAdSZCSgrS5Y= github.com/kastenhq/goversion v0.0.0-20230811215019-93b2f8823953/go.mod h1:6o+UrvuZWc4UTyBhQf0LGjW9Ld7qJxLz/OqvSOWWlEc= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= @@ -893,8 +895,6 @@ github.com/kubescape/go-logger v0.0.32 h1:4mI+XJOV8VFCMewrEE9VIFEIOhzXokYT3nFpNf github.com/kubescape/go-logger v0.0.32/go.mod h1:Alj7JBQ8/WCxbXe8Ura6ZheSRK45E0p21M3xeqedX90= github.com/kubescape/k8s-interface v0.0.214 h1:j7KP0/5VvYOoQdBGV2+gRM3qnR8PWLAGF8RM/k/DmJ0= github.com/kubescape/k8s-interface v0.0.214/go.mod h1:WNYUG93aZ5kDmuaRKFLtVhp18Yc6EfaHdD1gLYtVTN4= -github.com/kubescape/storage v0.0.303 h1:0nXI6E07lbWsg7iEH04vR4kwiekj//uCQl/La+8j4aM= -github.com/kubescape/storage v0.0.303/go.mod h1:d/1hqWPda2clsjx2wmQgysnB5dThIo3rDKP7RWx+v+M= github.com/kubescape/syft v1.32.0-ks.2 h1:xdUksUmKEyyVKsTfJDYW8Z5HawVJtelsUolPOsWtDx0= github.com/kubescape/syft v1.32.0-ks.2/go.mod h1:E6Kd4iBM2ljUOUQvSt7hVK6vBwaHkMXwcvBZmGMSY5o= github.com/kubescape/workerpool v0.0.0-20250526074519-0e4a4e7f44cf h1:hI0jVwrB6fT4GJWvuUjzObfci1CUknrZdRHfnRVtKM0= diff --git a/pkg/networkpeer/expand.go b/pkg/networkpeer/expand.go new file mode 100644 index 0000000000..2f76c97897 --- /dev/null +++ b/pkg/networkpeer/expand.go @@ -0,0 +1,146 @@ +package networkpeer + +import ( + "strings" + + "github.com/kubescape/storage/pkg/apis/softwarecomposition/v1beta1" +) + +// ExpandServiceNeighbors resolves every serviceRef / serviceSelector / entity +// neighbor in the list against the cluster view and returns equivalent +// synthesized ipAddresses neighbors (one per source neighbor, carrying the +// resolved IPs and the source neighbor's own ports). +// +// The synthesized neighbors are ordinary selector-free ipAddresses entries, so +// the existing port-sensitive address matcher handles them with no further +// change — a serviceRef/host neighbor becomes exactly the narrow, resolved +// ipAddresses entry it stands for. Neighbors that resolve to nothing (unknown +// Service, selector matching nothing, unknown entity) contribute nothing — +// never a match-all. Callers append the result to the same direction (egress +// or ingress) before projecting the profile. +func ExpandServiceNeighbors(neighbors []v1beta1.NetworkNeighbor, l Lister) []v1beta1.NetworkNeighbor { + if l == nil { + return nil + } + var out []v1beta1.NetworkNeighbor + for i := range neighbors { + n := &neighbors[i] + spec, ok := specFromNeighbor(n) + if !ok { + continue + } + ips := ResolveIPs(spec, l) + if len(ips) == 0 { + continue + } + out = append(out, v1beta1.NetworkNeighbor{ + Identifier: n.Identifier + "-resolved", + Type: n.Type, + IPAddresses: ips, + Ports: n.Ports, + }) + } + return out +} + +// WithResolvedServiceNeighbors returns cp with every serviceRef/serviceSelector/ +// entity neighbor expanded into equivalent selector-free ipAddresses neighbors +// (appended to the same direction), so the projection's existing address +// surface enforces them. It is a no-op — returning cp unchanged — when the +// lister is nil or nothing resolves, and it never mutates the input: a copy is +// made only when there is something to add. Call it immediately before +// projecting a ContainerProfile. +func WithResolvedServiceNeighbors(cp *v1beta1.ContainerProfile, l Lister) *v1beta1.ContainerProfile { + if cp == nil || l == nil { + return cp + } + egExtra := ExpandServiceNeighbors(cp.Spec.Egress, l) + inExtra := ExpandServiceNeighbors(cp.Spec.Ingress, l) + if len(egExtra) == 0 && len(inExtra) == 0 { + return cp + } + out := cp.DeepCopy() + out.Spec.Egress = append(out.Spec.Egress, egExtra...) + out.Spec.Ingress = append(out.Spec.Ingress, inExtra...) + return out +} + +// HasServiceNeighbors reports whether any egress/ingress neighbor declares a +// serviceRef / serviceSelector / entity — i.e. whether this profile's +// projection depends on the live cluster view (Service/EndpointSlice/Node) and +// must be re-projected when that view changes. Keyed on the raw fields, not on +// whether they currently resolve, so a profile projected before the informers +// synced is still marked and re-projects once they do. +func HasServiceNeighbors(cp *v1beta1.ContainerProfile) bool { + if cp == nil { + return false + } + for i := range cp.Spec.Egress { + if hasServiceFields(&cp.Spec.Egress[i]) { + return true + } + } + for i := range cp.Spec.Ingress { + if hasServiceFields(&cp.Spec.Ingress[i]) { + return true + } + } + return false +} + +func hasServiceFields(n *v1beta1.NetworkNeighbor) bool { + return n.ServiceRefNamespace != "" || n.ServiceRefName != "" || n.ServiceSelector != nil || n.Entity != "" +} + +// specFromNeighbor extracts a PeerSpec from a NetworkNeighbor, reporting false +// if the neighbor declares none of the service/entity selectors (a plain +// ipAddresses / dnsNames / podSelector neighbor is left untouched). +func specFromNeighbor(n *v1beta1.NetworkNeighbor) (PeerSpec, bool) { + spec := PeerSpec{Ports: portsFromNeighbor(n.Ports)} + switch { + case n.Entity != "": + spec.Entity = n.Entity + case n.ServiceRefName != "": + spec.ServiceRef = &ServiceRef{Namespace: n.ServiceRefNamespace, Name: n.ServiceRefName} + case n.ServiceSelector != nil: + // Only equality (matchLabels) is honored. A MatchExpressions clause or + // an empty matchLabels would either be silently ignored (broadening the + // match) or resolve to every Service — fail closed instead. + if len(n.ServiceSelector.MatchExpressions) > 0 || len(n.ServiceSelector.MatchLabels) == 0 { + return PeerSpec{}, false + } + spec.ServiceSelector = n.ServiceSelector.MatchLabels + // A namespaceSelector is honored only as the single equality + // kubernetes.io/metadata.name= (the only key the lister scopes on). + // Any other form — MatchExpressions, extra keys, or a different key — + // would be silently dropped and broaden the match cluster-wide, so fail + // closed. A nil namespaceSelector is cluster-wide by design. + if n.NamespaceSelector != nil { + nsl := n.NamespaceSelector + if len(nsl.MatchExpressions) > 0 || len(nsl.MatchLabels) != 1 || + nsl.MatchLabels["kubernetes.io/metadata.name"] == "" { + return PeerSpec{}, false + } + spec.NamespaceLabels = nsl.MatchLabels + } + default: + return PeerSpec{}, false + } + return spec, true +} + +func portsFromNeighbor(ports []v1beta1.NetworkPort) []PortProto { + if len(ports) == 0 { + return nil + } + out := make([]PortProto, 0, len(ports)) + for i := range ports { + p := &ports[i] + var port int32 + if p.Port != nil { + port = *p.Port + } + out = append(out, PortProto{Port: port, Protocol: strings.ToUpper(string(p.Protocol))}) + } + return out +} diff --git a/pkg/networkpeer/expand_test.go b/pkg/networkpeer/expand_test.go new file mode 100644 index 0000000000..dfb397b879 --- /dev/null +++ b/pkg/networkpeer/expand_test.go @@ -0,0 +1,209 @@ +package networkpeer + +import ( + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/kubescape/storage/pkg/apis/softwarecomposition/v1beta1" +) + +func port(name string, p int32) v1beta1.NetworkPort { + return v1beta1.NetworkPort{Name: name, Protocol: "TCP", Port: &p} +} + +// TestExpandServiceNeighbors_Egress: a serviceRef (alertmanager) + a host +// entity neighbor expand into selector-free ipAddresses neighbors carrying the +// resolved IPs and the original ports; a plain ipAddresses neighbor and an +// unresolvable serviceRef contribute nothing. +func TestExpandServiceNeighbors_Egress(t *testing.T) { + l := realFluxTopology() + in := []v1beta1.NetworkNeighbor{ + {Identifier: "am", Type: "internal", ServiceRefNamespace: "honey", ServiceRefName: "alertmanager", Ports: []v1beta1.NetworkPort{port("TCP-9093", 9093)}}, + {Identifier: "plain", Type: "internal", IPAddresses: []string{"10.43.0.0/16"}, Ports: []v1beta1.NetworkPort{port("TCP-443", 443)}}, + {Identifier: "ghost", Type: "internal", ServiceRefNamespace: "honey", ServiceRefName: "missing", Ports: []v1beta1.NetworkPort{port("TCP-1", 1)}}, + } + out := ExpandServiceNeighbors(in, l) + + if len(out) != 1 { + t.Fatalf("expected 1 synthesized neighbor (alertmanager only), got %d", len(out)) + } + got := out[0] + if got.Identifier != "am-resolved" { + t.Errorf("identifier: got %q", got.Identifier) + } + // ClusterIP + both endpoints, port carried over. + wantIPs := map[string]bool{"10.43.54.190": false, "10.42.0.44": false, "10.42.0.84": false} + for _, ip := range got.IPAddresses { + if _, ok := wantIPs[ip]; !ok { + t.Errorf("unexpected resolved IP %s", ip) + } + wantIPs[ip] = true + } + for ip, seen := range wantIPs { + if !seen { + t.Errorf("missing resolved IP %s", ip) + } + } + if len(got.Ports) != 1 || got.Ports[0].Port == nil || *got.Ports[0].Port != 9093 { + t.Errorf("ports not carried over: %+v", got.Ports) + } +} + +// TestExpandServiceNeighbors_HostEntity: fromEntity host resolves to node + +// gateway IPs on the health port. +func TestExpandServiceNeighbors_HostEntity(t *testing.T) { + l := realFluxTopology() + in := []v1beta1.NetworkNeighbor{ + {Identifier: "probes", Type: "internal", Entity: "host", Ports: []v1beta1.NetworkPort{port("TCP-9440", 9440)}}, + } + out := ExpandServiceNeighbors(in, l) + if len(out) != 1 { + t.Fatalf("expected 1 synthesized host neighbor, got %d", len(out)) + } + // Feed the synthesized entry through the address matcher the same way the + // projection would, to prove end-to-end intent. + tuples := Resolve(PeerSpec{Entity: "host", Ports: []PortProto{{Port: 9440, Protocol: "TCP"}}}, l) + if !Matches(tuples, "10.42.0.1", 9440, "TCP") { + t.Errorf("gateway kubelet probe should match") + } + if Matches(tuples, "10.42.0.1", 9090, "TCP") { + t.Errorf("wrong port must not match") + } +} + +// TestExpandServiceNeighbors_Selector: serviceSelector expands across all +// matching Services in the scoped namespace. +func TestExpandServiceNeighbors_Selector(t *testing.T) { + l := realFluxTopology() + in := []v1beta1.NetworkNeighbor{{ + Identifier: "guestbooks", + Type: "internal", + ServiceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "guestbook"}}, + NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/metadata.name": "gitops-demo"}}, + Ports: []v1beta1.NetworkPort{port("TCP-80", 80)}, + }} + out := ExpandServiceNeighbors(in, l) + if len(out) != 1 { + t.Fatalf("expected 1 synthesized neighbor, got %d", len(out)) + } + if len(out[0].IPAddresses) != 2 { + t.Errorf("expected 2 guestbook ClusterIPs, got %v", out[0].IPAddresses) + } +} + +// TestExpandServiceNeighbors_NilLister: no cluster view, no expansion. +func TestExpandServiceNeighbors_NilLister(t *testing.T) { + in := []v1beta1.NetworkNeighbor{{Identifier: "am", Entity: "host"}} + if out := ExpandServiceNeighbors(in, nil); out != nil { + t.Errorf("nil lister must expand to nil, got %v", out) + } +} + +// TestExpandServiceNeighbors_SelectorFailClosed: a serviceSelector carrying +// MatchExpressions (unsupported) or an empty matchLabels must NOT broaden the +// allowlist — it resolves to nothing. +func TestExpandServiceNeighbors_SelectorFailClosed(t *testing.T) { + l := realFluxTopology() + cases := []v1beta1.NetworkNeighbor{ + {Identifier: "expr", ServiceSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "guestbook"}, + MatchExpressions: []metav1.LabelSelectorRequirement{{Key: "tier", Operator: metav1.LabelSelectorOpExists}}, + }, Ports: []v1beta1.NetworkPort{port("TCP-80", 80)}}, + {Identifier: "empty", ServiceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{}}, Ports: []v1beta1.NetworkPort{port("TCP-80", 80)}}, + } + for _, n := range cases { + if out := ExpandServiceNeighbors([]v1beta1.NetworkNeighbor{n}, l); len(out) != 0 { + t.Errorf("%s: selector must fail closed, got %d", n.Identifier, len(out)) + } + } +} + +// TestWithResolvedServiceNeighbors: the CP-level wrapper appends resolved +// neighbors without mutating the input, and is a no-op when nothing resolves. +func TestWithResolvedServiceNeighbors(t *testing.T) { + l := realFluxTopology() + cp := &v1beta1.ContainerProfile{} + cp.Spec.Egress = []v1beta1.NetworkNeighbor{ + {Identifier: "am", ServiceRefNamespace: "honey", ServiceRefName: "alertmanager", Ports: []v1beta1.NetworkPort{port("TCP-9093", 9093)}}, + } + cp.Spec.Ingress = []v1beta1.NetworkNeighbor{ + {Identifier: "probes", Entity: "host", Ports: []v1beta1.NetworkPort{port("TCP-9440", 9440)}}, + } + out := WithResolvedServiceNeighbors(cp, l) + + if len(cp.Spec.Egress) != 1 || len(cp.Spec.Ingress) != 1 { + t.Fatalf("input CP must not be mutated: eg=%d in=%d", len(cp.Spec.Egress), len(cp.Spec.Ingress)) + } + if len(out.Spec.Egress) != 2 { + t.Errorf("egress: want original + 1 resolved, got %d", len(out.Spec.Egress)) + } + if len(out.Spec.Ingress) != 2 { + t.Errorf("ingress: want original + 1 resolved, got %d", len(out.Spec.Ingress)) + } + + // No-op cases. + if got := WithResolvedServiceNeighbors(cp, nil); got != cp { + t.Error("nil lister must return the same CP unchanged") + } + plain := &v1beta1.ContainerProfile{} + plain.Spec.Egress = []v1beta1.NetworkNeighbor{{Identifier: "ip", IPAddresses: []string{"10.0.0.0/8"}}} + if got := WithResolvedServiceNeighbors(plain, l); got != plain { + t.Error("a CP with no service/entity neighbors must return unchanged (same pointer)") + } +} + +// TestExpandServiceNeighbors_NamespaceSelectorFailClosed: a namespaceSelector +// is honored only as the single equality kubernetes.io/metadata.name=; any +// other form must fail closed rather than silently broaden cluster-wide. +func TestExpandServiceNeighbors_NamespaceSelectorFailClosed(t *testing.T) { + l := realFluxTopology() + withNS := func(nsSel *metav1.LabelSelector) v1beta1.NetworkNeighbor { + return v1beta1.NetworkNeighbor{ + Identifier: "svc", + ServiceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "guestbook"}}, + NamespaceSelector: nsSel, + Ports: []v1beta1.NetworkPort{port("TCP-80", 80)}, + } + } + bad := []*metav1.LabelSelector{ + {MatchExpressions: []metav1.LabelSelectorRequirement{{Key: "kubernetes.io/metadata.name", Operator: metav1.LabelSelectorOpExists}}}, + {MatchLabels: map[string]string{"env": "prod"}}, // wrong key + {MatchLabels: map[string]string{"kubernetes.io/metadata.name": "gitops-demo", "x": "y"}}, // extra key + {MatchLabels: map[string]string{}}, // empty + } + for i, ns := range bad { + if out := ExpandServiceNeighbors([]v1beta1.NetworkNeighbor{withNS(ns)}, l); len(out) != 0 { + t.Errorf("bad namespaceSelector[%d] must fail closed, got %d", i, len(out)) + } + } + good := withNS(&metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/metadata.name": "gitops-demo"}}) + if out := ExpandServiceNeighbors([]v1beta1.NetworkNeighbor{good}, l); len(out) != 1 { + t.Errorf("metadata.name namespaceSelector should resolve, got %d", len(out)) + } +} + +// TestHasServiceNeighbors: a profile with a serviceRef/serviceSelector/entity +// neighbor is flagged as depending on the live cluster view; a plain +// ipAddresses/dnsNames profile is not. +func TestHasServiceNeighbors(t *testing.T) { + if HasServiceNeighbors(nil) { + t.Error("nil CP must be false") + } + plain := &v1beta1.ContainerProfile{} + plain.Spec.Egress = []v1beta1.NetworkNeighbor{{Identifier: "ip", IPAddresses: []string{"10.0.0.0/8"}}} + if HasServiceNeighbors(plain) { + t.Error("plain ipAddresses profile must not use service resolution") + } + for _, n := range []v1beta1.NetworkNeighbor{ + {ServiceRefName: "alertmanager", ServiceRefNamespace: "honey"}, + {ServiceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "x"}}}, + {Entity: "host"}, + } { + cp := &v1beta1.ContainerProfile{} + cp.Spec.Ingress = []v1beta1.NetworkNeighbor{n} + if !HasServiceNeighbors(cp) { + t.Errorf("neighbor %+v should be flagged", n) + } + } +} diff --git a/pkg/networkpeer/lister.go b/pkg/networkpeer/lister.go new file mode 100644 index 0000000000..eff6d349db --- /dev/null +++ b/pkg/networkpeer/lister.go @@ -0,0 +1,167 @@ +package networkpeer + +import ( + "net" + "sync/atomic" + + corev1 "k8s.io/api/core/v1" + discoveryv1 "k8s.io/api/discovery/v1" + "k8s.io/apimachinery/pkg/labels" + corelisters "k8s.io/client-go/listers/core/v1" + discoverylisters "k8s.io/client-go/listers/discovery/v1" +) + +// InformerLister is the production Lister, backed by Service / EndpointSlice / +// Node informer listers. It resolves a serviceRef/serviceSelector to the +// Service's ClusterIP(s) ∪ its EndpointSlice addresses, and the "host" entity +// to every node's InternalIP(s) plus the CNI gateway derived from its PodCIDR. +type InformerLister struct { + services corelisters.ServiceLister + slices discoverylisters.EndpointSliceLister + nodes corelisters.NodeLister + // nodeName scopes the "host" entity to the local node. Empty means every + // node (used by tests); production passes the agent's own node so kubelet + // probes from this node's gateway match without broadening "host" to the + // whole cluster. + nodeName string + // generation advances on every observed Service/EndpointSlice/Node change + // (bumped from informer event handlers wired in cmd/main.go). + generation atomic.Int64 +} + +// Generation returns the current cluster-view generation. +func (l *InformerLister) Generation() int64 { return l.generation.Load() } + +// Bump advances the generation; wire it to the informer event handlers. +func (l *InformerLister) Bump() { l.generation.Add(1) } + +func NewInformerLister(services corelisters.ServiceLister, slices discoverylisters.EndpointSliceLister, nodes corelisters.NodeLister, nodeName string) *InformerLister { + return &InformerLister{services: services, slices: slices, nodes: nodes, nodeName: nodeName} +} + +var _ Lister = (*InformerLister)(nil) + +func (l *InformerLister) ServiceByName(namespace, name string) (*ServiceInfo, bool) { + svc, err := l.services.Services(namespace).Get(name) + if err != nil { + return nil, false + } + return l.serviceInfo(svc), true +} + +func (l *InformerLister) ServicesByLabels(serviceSelector, namespaceLabels map[string]string) []*ServiceInfo { + // Never resolve an empty selector to labels.Everything() — that would + // allowlist every Service in the cluster. Fail closed. + if len(serviceSelector) == 0 { + return nil + } + svcs, err := l.services.List(labels.SelectorFromSet(serviceSelector)) + if err != nil { + return nil + } + wantNS := "" + if namespaceLabels != nil { + wantNS = namespaceLabels["kubernetes.io/metadata.name"] + } + var out []*ServiceInfo + for _, svc := range svcs { + if wantNS != "" && svc.Namespace != wantNS { + continue + } + out = append(out, l.serviceInfo(svc)) + } + return out +} + +func (l *InformerLister) HostIPs() []string { + nodes, err := l.nodes.List(labels.Everything()) + if err != nil { + return nil + } + var ips []string + for _, n := range nodes { + if l.nodeName != "" && n.Name != l.nodeName { + continue + } + for _, addr := range n.Status.Addresses { + if addr.Type == corev1.NodeInternalIP { + ips = append(ips, addr.Address) + } + } + for _, cidr := range podCIDRs(n) { + if gw := gatewayIP(cidr); gw != "" { + ips = append(ips, gw) + } + } + } + return dedupe(ips) +} + +func (l *InformerLister) serviceInfo(svc *corev1.Service) *ServiceInfo { + info := &ServiceInfo{Namespace: svc.Namespace, Name: svc.Name, Labels: svc.Labels} + for _, ip := range svc.Spec.ClusterIPs { + if ip != "" && ip != corev1.ClusterIPNone { + info.ClusterIPs = append(info.ClusterIPs, ip) + } + } + if len(info.ClusterIPs) == 0 && svc.Spec.ClusterIP != "" && svc.Spec.ClusterIP != corev1.ClusterIPNone { + info.ClusterIPs = append(info.ClusterIPs, svc.Spec.ClusterIP) + } + info.EndpointIPs = l.endpointIPs(svc.Namespace, svc.Name) + return info +} + +func (l *InformerLister) endpointIPs(namespace, service string) []string { + sel := labels.SelectorFromSet(labels.Set{discoveryv1.LabelServiceName: service}) + slices, err := l.slices.EndpointSlices(namespace).List(sel) + if err != nil { + return nil + } + var ips []string + for _, es := range slices { + for i := range es.Endpoints { + ips = append(ips, es.Endpoints[i].Addresses...) + } + } + return dedupe(ips) +} + +func podCIDRs(n *corev1.Node) []string { + if len(n.Spec.PodCIDRs) > 0 { + return n.Spec.PodCIDRs + } + if n.Spec.PodCIDR != "" { + return []string{n.Spec.PodCIDR} + } + return nil +} + +// gatewayIP returns the conventional CNI gateway for a pod CIDR: the network +// address + 1 (e.g. 10.42.0.0/24 -> 10.42.0.1). Masqueraded node-sourced +// traffic (kubelet health probes) appears from this address. IPv6 CIDRs yield +// no gateway (the .1 convention is IPv4). +func gatewayIP(cidr string) string { + _, ipNet, err := net.ParseCIDR(cidr) + if err != nil { + return "" + } + ip := ipNet.IP.To4() + if ip == nil { + return "" + } + gw := make(net.IP, len(ip)) + copy(gw, ip) + for i := len(gw) - 1; i >= 0; i-- { + gw[i]++ + if gw[i] != 0 { + break + } + } + // A /31 or /32 (or a network address ending in .255 that overflows) yields a + // gateway outside the CIDR — never allowlist an IP the pod network doesn't + // actually contain. + if !ipNet.Contains(gw) { + return "" + } + return gw.String() +} diff --git a/pkg/networkpeer/lister_test.go b/pkg/networkpeer/lister_test.go new file mode 100644 index 0000000000..98d0143b10 --- /dev/null +++ b/pkg/networkpeer/lister_test.go @@ -0,0 +1,147 @@ +package networkpeer + +import ( + "testing" + + corev1 "k8s.io/api/core/v1" + discoveryv1 "k8s.io/api/discovery/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes/fake" +) + +// newTestLister builds an InformerLister over a fake cluster seeded from the +// real flux/kubescape topology, exercising the production Service/EndpointSlice/ +// Node -> Lister path (not the hand-written fake used by the resolver tests). +func newTestLister(t *testing.T) *InformerLister { + t.Helper() + client := fake.NewClientset( + &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{Namespace: "honey", Name: "alertmanager", Labels: map[string]string{"app": "alertmanager"}}, + Spec: corev1.ServiceSpec{ClusterIP: "10.43.54.190", ClusterIPs: []string{"10.43.54.190"}}, + }, + &discoveryv1.EndpointSlice{ + ObjectMeta: metav1.ObjectMeta{Namespace: "honey", Name: "alertmanager-x1", Labels: map[string]string{discoveryv1.LabelServiceName: "alertmanager"}}, + Endpoints: []discoveryv1.Endpoint{{Addresses: []string{"10.42.0.44", "10.42.0.84"}}}, + }, + &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{Namespace: "gitops-demo", Name: "guestbook-ui", Labels: map[string]string{"app": "guestbook"}}, + Spec: corev1.ServiceSpec{ClusterIP: "10.43.111.192", ClusterIPs: []string{"10.43.111.192"}}, + }, + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: "tanzee"}, + Spec: corev1.NodeSpec{PodCIDR: "10.42.0.0/24", PodCIDRs: []string{"10.42.0.0/24"}}, + Status: corev1.NodeStatus{Addresses: []corev1.NodeAddress{{Type: corev1.NodeInternalIP, Address: "192.168.0.191"}}}, + }, + &corev1.Node{ // a second node whose IPs must NOT leak into this node's "host" + ObjectMeta: metav1.ObjectMeta{Name: "other"}, + Spec: corev1.NodeSpec{PodCIDR: "10.42.9.0/24", PodCIDRs: []string{"10.42.9.0/24"}}, + Status: corev1.NodeStatus{Addresses: []corev1.NodeAddress{{Type: corev1.NodeInternalIP, Address: "192.168.0.99"}}}, + }, + ) + factory := informers.NewSharedInformerFactory(client, 0) + l := NewInformerLister( + factory.Core().V1().Services().Lister(), + factory.Discovery().V1().EndpointSlices().Lister(), + factory.Core().V1().Nodes().Lister(), + "tanzee", + ) + stop := make(chan struct{}) + t.Cleanup(func() { close(stop) }) + factory.Start(stop) + factory.WaitForCacheSync(stop) + return l +} + +func TestInformerLister_ServiceByName(t *testing.T) { + l := newTestLister(t) + svc, ok := l.ServiceByName("honey", "alertmanager") + if !ok { + t.Fatal("alertmanager Service should resolve") + } + if len(svc.ClusterIPs) != 1 || svc.ClusterIPs[0] != "10.43.54.190" { + t.Errorf("ClusterIPs: got %v", svc.ClusterIPs) + } + if len(svc.EndpointIPs) != 2 { + t.Errorf("EndpointIPs: got %v (want 2 from the EndpointSlice)", svc.EndpointIPs) + } + // End-to-end through the resolver, like the projection will. + tuples := Resolve(PeerSpec{ServiceRef: &ServiceRef{"honey", "alertmanager"}, Ports: []PortProto{{Port: 9093, Protocol: "TCP"}}}, l) + if !Matches(tuples, "10.43.54.190", 9093, "TCP") || !Matches(tuples, "10.42.0.44", 9093, "TCP") { + t.Errorf("resolver over informer lister should match ClusterIP + endpoints") + } + if _, ok := l.ServiceByName("honey", "nope"); ok { + t.Error("unknown Service must not resolve") + } +} + +func TestInformerLister_HostIPs(t *testing.T) { + l := newTestLister(t) + ips := l.HostIPs() + want := map[string]bool{"192.168.0.191": false, "10.42.0.1": false} + for _, ip := range ips { + if _, ok := want[ip]; ok { + want[ip] = true + } + } + for ip, seen := range want { + if !seen { + t.Errorf("HostIPs missing %s (got %v)", ip, ips) + } + } + for _, ip := range ips { + if ip == "192.168.0.99" || ip == "10.42.9.1" { + t.Errorf("HostIPs must be scoped to the local node; leaked other node's %s", ip) + } + } +} + +// TestInformerLister_EmptySelectorFailsClosed: an empty serviceSelector must +// NOT resolve to every Service in the cluster. +func TestInformerLister_EmptySelectorFailsClosed(t *testing.T) { + l := newTestLister(t) + if got := l.ServicesByLabels(map[string]string{}, nil); len(got) != 0 { + t.Errorf("empty selector must fail closed, got %d services", len(got)) + } + if got := Resolve(PeerSpec{ServiceSelector: map[string]string{}, Ports: tcp(80)}, l); len(got) != 0 { + t.Errorf("Resolve with empty selector must yield nothing, got %v", got) + } +} + +func TestInformerLister_ServicesByLabels(t *testing.T) { + l := newTestLister(t) + svcs := l.ServicesByLabels(map[string]string{"app": "guestbook"}, map[string]string{"kubernetes.io/metadata.name": "gitops-demo"}) + if len(svcs) != 1 || svcs[0].Name != "guestbook-ui" { + t.Fatalf("expected guestbook-ui, got %v", svcs) + } + // Namespace scoping excludes a same-label service elsewhere (none here) and + // a wrong-namespace filter yields nothing. + if got := l.ServicesByLabels(map[string]string{"app": "guestbook"}, map[string]string{"kubernetes.io/metadata.name": "other"}); len(got) != 0 { + t.Errorf("namespace filter should exclude, got %v", got) + } +} + +func TestGatewayIP(t *testing.T) { + cases := map[string]string{ + "10.42.0.0/24": "10.42.0.1", + "10.244.5.0/24": "10.244.5.1", + "2001:db8::/64": "", + "10.42.0.5/32": "", // /32 host: incremented gateway is outside the CIDR + "255.255.255.255/32": "", // overflow to 0.0.0.0, out of CIDR + } + for cidr, want := range cases { + if got := gatewayIP(cidr); got != want { + t.Errorf("gatewayIP(%s)=%q want %q", cidr, got, want) + } + } +} + +func TestInformerLister_Generation(t *testing.T) { + l := newTestLister(t) + g0 := l.Generation() + l.Bump() + l.Bump() + if l.Generation() != g0+2 { + t.Errorf("Generation must advance on Bump: got %d want %d", l.Generation(), g0+2) + } +} diff --git a/pkg/networkpeer/resolve.go b/pkg/networkpeer/resolve.go new file mode 100644 index 0000000000..0467c14b50 --- /dev/null +++ b/pkg/networkpeer/resolve.go @@ -0,0 +1,197 @@ +// Package networkpeer resolves Kubernetes-native network-neighbor selectors +// (a Service reference, a Service label selector, or a reserved entity such as +// "host") into the concrete (IP, port, protocol) tuples an egress/ingress +// allowlist should match. +// +// It exists so a ContainerProfile can express cluster-infrastructure peers — +// Service ClusterIPs (alertmanager, kube-apiserver via default/kubernetes, +// kube-dns, ...) and host/kubelet traffic — portably (by name, resolved +// per-cluster) and narrowly (only the referenced Service/entity), instead of a +// broad ipAddresses CIDR over the whole service network. See +// k8sstormcenter/node-agent#92. +// +// Resolution is intentionally decoupled from live matching: callers resolve a +// PeerSpec to []AllowTuple once (e.g. at projection time) via a Lister backed +// by Service/EndpointSlice/Node informers, then match observed connections +// against the tuples with Matches. The Lister interface keeps the resolver +// unit-testable against a fake cluster view. +package networkpeer + +import "strings" + +// EntityHost is the reserved entity naming the local node: its InternalIP(s) +// and the CNI gateway address. It is the one peer class no Service object can +// represent (kubelet health probes, node-sourced / masqueraded traffic). +const EntityHost = "host" + +// PortProto is a single allowed destination port/protocol. Protocol is +// upper-case ("TCP"/"UDP"); an empty Protocol matches any protocol. +type PortProto struct { + Port int32 + Protocol string +} + +// ServiceRef names a single Service by namespace and name. +type ServiceRef struct { + Namespace string + Name string +} + +// PeerSpec is the storage-agnostic form of one serviceRef / serviceSelector / +// entity network-neighbor entry. Exactly one of ServiceRef, ServiceSelector, +// or Entity is expected to be set; Ports scopes the resolved tuples. +type PeerSpec struct { + ServiceRef *ServiceRef + ServiceSelector map[string]string + NamespaceLabels map[string]string + Entity string + Ports []PortProto +} + +// ServiceInfo is the resolver's view of one Service. +type ServiceInfo struct { + Namespace string + Name string + Labels map[string]string + ClusterIPs []string + EndpointIPs []string +} + +// Lister is the read-only cluster view the resolver needs. Production wires it +// to Service/EndpointSlice/Node informer listers; tests use a fake. +type Lister interface { + ServiceByName(namespace, name string) (*ServiceInfo, bool) + ServicesByLabels(serviceSelector, namespaceLabels map[string]string) []*ServiceInfo + HostIPs() []string + // Generation increments whenever the underlying cluster view changes (any + // Service/EndpointSlice/Node event). Callers store it alongside a projected + // profile and re-project when it advances, so resolved IPs don't go stale on + // endpoint churn or caches that filled after projection. + Generation() int64 +} + +// AllowTuple is one concrete (IP, port, protocol) a resolved PeerSpec permits. +type AllowTuple struct { + IP string + Port int32 + Protocol string +} + +// Resolve expands spec into the concrete tuples it authorises, using l for the +// current cluster view. A spec with no resolvable target (unknown Service, +// selector matching nothing, unknown entity) yields no tuples — never a +// match-all. A spec with no Ports yields one tuple per IP with Port 0 / +// Protocol "" (any-port), so callers that ignore ports still work; callers +// that enforce ports should treat Port 0 as "unspecified". +func Resolve(spec PeerSpec, l Lister) []AllowTuple { + if l == nil { + return nil + } + ips := resolveIPs(spec, l) + if len(ips) == 0 { + return nil + } + return expand(ips, spec.Ports) +} + +// ResolveIPs returns just the IPs a spec resolves to, ignoring ports. Used by +// the projection-time expansion, which pairs them with the neighbor's own +// ports. +func ResolveIPs(spec PeerSpec, l Lister) []string { + if l == nil { + return nil + } + return resolveIPs(spec, l) +} + +func resolveIPs(spec PeerSpec, l Lister) []string { + switch { + case spec.Entity != "": + if strings.EqualFold(spec.Entity, EntityHost) { + return dedupe(l.HostIPs()) + } + return nil + case spec.ServiceRef != nil: + svc, ok := l.ServiceByName(spec.ServiceRef.Namespace, spec.ServiceRef.Name) + if !ok || svc == nil { + return nil + } + return serviceIPs(svc) + case spec.ServiceSelector != nil: + // An empty selector is NOT a cluster-wide match-all: fail closed. + if len(spec.ServiceSelector) == 0 { + return nil + } + var ips []string + for _, svc := range l.ServicesByLabels(spec.ServiceSelector, spec.NamespaceLabels) { + ips = append(ips, serviceIPs(svc)...) + } + return dedupe(ips) + default: + return nil + } +} + +func serviceIPs(svc *ServiceInfo) []string { + out := make([]string, 0, len(svc.ClusterIPs)+len(svc.EndpointIPs)) + out = append(out, svc.ClusterIPs...) + out = append(out, svc.EndpointIPs...) + return dedupe(out) +} + +func expand(ips []string, ports []PortProto) []AllowTuple { + if len(ports) == 0 { + out := make([]AllowTuple, 0, len(ips)) + for _, ip := range ips { + out = append(out, AllowTuple{IP: ip}) + } + return out + } + out := make([]AllowTuple, 0, len(ips)*len(ports)) + for _, ip := range ips { + for _, p := range ports { + out = append(out, AllowTuple{IP: ip, Port: p.Port, Protocol: strings.ToUpper(p.Protocol)}) + } + } + return out +} + +// Matches reports whether the observed (ip, port, protocol) connection is +// permitted by any tuple. Matching is port-sensitive: a tuple with Port 0 +// (any-port) matches any observed port; otherwise the port must be equal. An +// empty tuple Protocol matches any protocol. +func Matches(tuples []AllowTuple, ip string, port int32, protocol string) bool { + protocol = strings.ToUpper(protocol) + for _, t := range tuples { + if t.IP != ip { + continue + } + if t.Port != 0 && t.Port != port { + continue + } + if t.Protocol != "" && t.Protocol != protocol { + continue + } + return true + } + return false +} + +func dedupe(in []string) []string { + if len(in) == 0 { + return nil + } + seen := make(map[string]struct{}, len(in)) + out := make([]string, 0, len(in)) + for _, s := range in { + if s == "" { + continue + } + if _, ok := seen[s]; ok { + continue + } + seen[s] = struct{}{} + out = append(out, s) + } + return out +} diff --git a/pkg/networkpeer/resolve_test.go b/pkg/networkpeer/resolve_test.go new file mode 100644 index 0000000000..f131f774fe --- /dev/null +++ b/pkg/networkpeer/resolve_test.go @@ -0,0 +1,204 @@ +package networkpeer + +import ( + "sort" + "testing" +) + +// fakeLister is a static cluster view seeded from the real Flux/kubescape +// topology observed on the k3s dev cluster (issue #92). It lets the resolver +// tests assert against genuine ClusterIPs/endpoints without a live cluster. +type fakeLister struct { + services map[string]*ServiceInfo // key "ns/name" + hostIPs []string +} + +func (f *fakeLister) ServiceByName(ns, name string) (*ServiceInfo, bool) { + s, ok := f.services[ns+"/"+name] + return s, ok +} + +func (f *fakeLister) ServicesByLabels(sel, nsLabels map[string]string) []*ServiceInfo { + var out []*ServiceInfo + for _, s := range f.services { + if nsLabels != nil { + if s.Labels["__ns__"] != nsLabels["kubernetes.io/metadata.name"] { + continue + } + } + if labelsSubset(sel, s.Labels) { + out = append(out, s) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name }) + return out +} + +func (f *fakeLister) HostIPs() []string { return f.hostIPs } + +func (f *fakeLister) Generation() int64 { return 0 } + +func labelsSubset(want, have map[string]string) bool { + for k, v := range want { + if have[k] != v { + return false + } + } + return true +} + +// realFluxTopology mirrors what `kubectl get svc/endpoints` returned live. +func realFluxTopology() *fakeLister { + return &fakeLister{ + hostIPs: []string{"192.168.0.191", "10.42.0.1"}, // node InternalIP + CNI gateway + services: map[string]*ServiceInfo{ + "honey/alertmanager": { + Namespace: "honey", Name: "alertmanager", + Labels: map[string]string{"app": "alertmanager", "__ns__": "honey"}, + ClusterIPs: []string{"10.43.54.190"}, + EndpointIPs: []string{"10.42.0.44", "10.42.0.84"}, + }, + "honey/storage": { + Namespace: "honey", Name: "storage", + Labels: map[string]string{"app": "storage", "__ns__": "honey"}, + ClusterIPs: []string{"10.43.70.156"}, + }, + "default/kubernetes": { // k3s: apiserver endpoint is the node IP (Kind: Host) + Namespace: "default", Name: "kubernetes", + Labels: map[string]string{"__ns__": "default"}, + ClusterIPs: []string{"10.43.0.1"}, + EndpointIPs: []string{"192.168.0.191"}, + }, + "argocd/argocd-server": { + Namespace: "argocd", Name: "argocd-server", + Labels: map[string]string{"app.kubernetes.io/name": "argocd-server", "__ns__": "argocd"}, + ClusterIPs: []string{"10.43.173.14"}, + }, + "gitops-demo/guestbook-ui": { + Namespace: "gitops-demo", Name: "guestbook-ui", + Labels: map[string]string{"app": "guestbook", "__ns__": "gitops-demo"}, + ClusterIPs: []string{"10.43.111.192"}, + }, + "gitops-demo/helm-guestbook": { + Namespace: "gitops-demo", Name: "helm-guestbook", + Labels: map[string]string{"app": "guestbook", "__ns__": "gitops-demo"}, + ClusterIPs: []string{"10.43.3.63"}, + }, + }, + } +} + +func tcp(port int32) []PortProto { return []PortProto{{Port: port, Protocol: "TCP"}} } + +// Test A1 — serviceRef egress (alertmanager) is narrow AND port-sensitive: +// matches its ClusterIP and endpoints on 9093 only; a sibling service on the +// same port stays visible to R0011 (the whole point vs a /16). +func TestServiceRef_Alertmanager(t *testing.T) { + l := realFluxTopology() + tuples := Resolve(PeerSpec{ServiceRef: &ServiceRef{"honey", "alertmanager"}, Ports: tcp(9093)}, l) + + cases := []struct { + ip string + port int32 + proto string + want bool + why string + }{ + {"10.43.54.190", 9093, "TCP", true, "ClusterIP + port match"}, + {"10.42.0.44", 9093, "TCP", true, "backing endpoint IP"}, + {"10.42.0.84", 9093, "TCP", true, "backing endpoint IP"}, + {"10.43.54.190", 8080, "TCP", false, "wrong port (port-sensitive)"}, + {"10.43.54.190", 9093, "UDP", false, "wrong protocol"}, + {"10.43.173.14", 9093, "TCP", false, "argocd-server — different service, detection preserved"}, + {"10.43.70.156", 9093, "TCP", false, "storage — different service, detection preserved"}, + } + for _, c := range cases { + if got := Matches(tuples, c.ip, c.port, c.proto); got != c.want { + t.Errorf("Matches(%s:%d/%s)=%v want %v (%s)", c.ip, c.port, c.proto, got, c.want, c.why) + } + } +} + +// Test A2 — kube-apiserver needs no dedicated entity: toService default/kubernetes +// resolves to the ClusterIP AND the node-IP endpoint (k3s embeds the apiserver). +func TestServiceRef_KubeApiserver(t *testing.T) { + l := realFluxTopology() + tuples := Resolve(PeerSpec{ServiceRef: &ServiceRef{"default", "kubernetes"}, Ports: tcp(443)}, l) + for _, ip := range []string{"10.43.0.1", "192.168.0.191"} { + if !Matches(tuples, ip, 443, "TCP") { + t.Errorf("apiserver egress %s:443 should match via default/kubernetes", ip) + } + } + if Matches(tuples, "10.43.0.1", 6443, "TCP") { + t.Errorf("apiserver :6443 must not match (port 443 only)") + } +} + +// Test A3 — host entity: kubelet probe from the node/gateway matches on the +// health port only; a pod-CIDR source or wrong port does not. +func TestEntityHost(t *testing.T) { + l := realFluxTopology() + tuples := Resolve(PeerSpec{Entity: EntityHost, Ports: tcp(9440)}, l) + if !Matches(tuples, "10.42.0.1", 9440, "TCP") { + t.Errorf("kubelet probe 10.42.0.1:9440 should match fromEntity host") + } + if !Matches(tuples, "192.168.0.191", 9440, "TCP") { + t.Errorf("node InternalIP :9440 should match fromEntity host") + } + if Matches(tuples, "10.42.0.1", 9090, "TCP") { + t.Errorf("host :9090 must not match (port 9440 only)") + } + if Matches(tuples, "10.42.0.55", 9440, "TCP") { + t.Errorf("a pod IP must not match fromEntity host") + } +} + +// Test A4 — serviceSelector fans out across all matching Services (the two +// gitops-demo guestbook services share app=guestbook), scoped by namespace. +func TestServiceSelector_GuestbookFanout(t *testing.T) { + l := realFluxTopology() + tuples := Resolve(PeerSpec{ + ServiceSelector: map[string]string{"app": "guestbook"}, + NamespaceLabels: map[string]string{"kubernetes.io/metadata.name": "gitops-demo"}, + Ports: tcp(80), + }, l) + for _, ip := range []string{"10.43.111.192", "10.43.3.63"} { + if !Matches(tuples, ip, 80, "TCP") { + t.Errorf("guestbook service %s:80 should match app=guestbook selector", ip) + } + } + if Matches(tuples, "10.43.173.14", 80, "TCP") { + t.Errorf("argocd-server must not match app=guestbook selector") + } +} + +// Test A5 — no accidental match-all: unknown service, unknown entity, and a +// selector matching nothing all resolve to zero tuples. +func TestResolve_NoMatchAll(t *testing.T) { + l := realFluxTopology() + specs := []PeerSpec{ + {ServiceRef: &ServiceRef{"honey", "does-not-exist"}, Ports: tcp(443)}, + {Entity: "world", Ports: tcp(443)}, + {ServiceSelector: map[string]string{"app": "nope"}, Ports: tcp(443)}, + } + for i, s := range specs { + if tuples := Resolve(s, l); len(tuples) != 0 { + t.Errorf("spec[%d] should resolve to no tuples, got %d", i, len(tuples)) + } + } + if Matches(nil, "10.43.0.1", 443, "TCP") { + t.Errorf("nil tuples must never match") + } +} + +// Test A6 — a nil Lister and any-port (no Ports) behave safely. +func TestResolve_Edges(t *testing.T) { + if got := Resolve(PeerSpec{Entity: EntityHost}, nil); got != nil { + t.Errorf("nil lister must resolve to nil, got %v", got) + } + l := realFluxTopology() + anyPort := Resolve(PeerSpec{ServiceRef: &ServiceRef{"honey", "storage"}}, l) + if !Matches(anyPort, "10.43.70.156", 443, "TCP") || !Matches(anyPort, "10.43.70.156", 8443, "TCP") { + t.Errorf("a serviceRef with no Ports should match any observed port on its IP") + } +} diff --git a/pkg/objectcache/containerprofilecache/containerprofilecache.go b/pkg/objectcache/containerprofilecache/containerprofilecache.go index 2a5394d18e..410b39ec47 100644 --- a/pkg/objectcache/containerprofilecache/containerprofilecache.go +++ b/pkg/objectcache/containerprofilecache/containerprofilecache.go @@ -16,6 +16,7 @@ import ( helpersv1 "github.com/kubescape/k8s-interface/instanceidhandler/v1/helpers" "github.com/kubescape/node-agent/pkg/config" "github.com/kubescape/node-agent/pkg/metricsmanager" + "github.com/kubescape/node-agent/pkg/networkpeer" "github.com/kubescape/node-agent/pkg/objectcache" "github.com/kubescape/node-agent/pkg/objectcache/callstackcache" "github.com/kubescape/node-agent/pkg/resourcelocks" @@ -58,6 +59,15 @@ type CachedContainerProfile struct { State *objectcache.ProfileState CallStackTree *callstackcache.CallStackSearchTree + // UsesServiceResolution is true when the profile declares serviceRef/ + // serviceSelector/entity neighbors, so its projected addresses depend on the + // live cluster view. ListerGen is the service-lister generation captured at + // projection time; the reconciler re-projects such entries when the + // generation advances (endpoint churn, or caches that filled after the + // initial projection). Non-resolving profiles keep the RV/spec fast-skip. + UsesServiceResolution bool + ListerGen int64 + ContainerName string PodName string Namespace string @@ -111,6 +121,10 @@ type ContainerProfileCacheImpl struct { containerLocks *resourcelocks.ResourceLocks storageClient storage.ProfileClient k8sObjectCache objectcache.K8sObjectCache + // serviceLister resolves serviceRef/serviceSelector/entity network + // neighbors to concrete IPs at projection time. nil = feature off (the + // profile projects unchanged), which is what every unit test leaves it as. + serviceLister networkpeer.Lister metricsManager metricsmanager.MetricsManager reconcileEvery time.Duration @@ -174,6 +188,22 @@ func NewContainerProfileCache(cfg config.Config, storageClient storage.ProfileCl return c } +// SetServiceLister installs the cluster view used to resolve +// serviceRef/serviceSelector/entity network neighbors at projection time. It +// is optional: when unset the projection leaves such neighbors unresolved. +func (c *ContainerProfileCacheImpl) SetServiceLister(l networkpeer.Lister) { + c.serviceLister = l +} + +// listerGen returns the current service-lister generation, or 0 when no lister +// is installed (unit tests, or the feature is off). +func (c *ContainerProfileCacheImpl) listerGen() int64 { + if c.serviceLister == nil { + return 0 + } + return c.serviceLister.Generation() +} + // refreshRPC calls fn with a context bounded by c.rpcBudget, enforcing a // per-call SLO so a slow API server cannot stall a full reconciler burst. func (c *ContainerProfileCacheImpl) refreshRPC(ctx context.Context, fn func(context.Context) error) error { @@ -579,9 +609,14 @@ func (c *ContainerProfileCacheImpl) buildEntry( } entry.CallStackTree = tree - // Project under the current spec. + // Project under the current spec, resolving any serviceRef/entity network + // neighbors to concrete IPs first. Record whether this profile depends on + // the live cluster view and the lister generation it was resolved against, + // so the reconciler re-projects it when that view changes. spec := c.snapshotSpec() - projected := Apply(spec, userMerged, tree) + entry.UsesServiceResolution = networkpeer.HasServiceNeighbors(userMerged) + entry.ListerGen = c.listerGen() + projected := Apply(spec, networkpeer.WithResolvedServiceNeighbors(userMerged, c.serviceLister), tree) entry.Projected = projected entry.SpecHash = projected.SpecHash diff --git a/pkg/objectcache/containerprofilecache/reconciler.go b/pkg/objectcache/containerprofilecache/reconciler.go index bd7517c728..85c4943aaf 100644 --- a/pkg/objectcache/containerprofilecache/reconciler.go +++ b/pkg/objectcache/containerprofilecache/reconciler.go @@ -23,6 +23,7 @@ import ( "github.com/kubescape/go-logger" "github.com/kubescape/go-logger/helpers" helpersv1 "github.com/kubescape/k8s-interface/instanceidhandler/v1/helpers" + "github.com/kubescape/node-agent/pkg/networkpeer" "github.com/kubescape/node-agent/pkg/objectcache" "github.com/kubescape/node-agent/pkg/objectcache/callstackcache" "github.com/kubescape/node-agent/pkg/utils" @@ -418,9 +419,14 @@ func (c *ContainerProfileCacheImpl) refreshOneEntry(ctx context.Context, id stri if spec := c.snapshotSpec(); spec != nil { currentSpecHash = spec.Hash } + // serviceRef/entity profiles must also re-project when the cluster view + // changed since they were resolved (endpoint churn, or caches that filled + // after projection). Non-resolving profiles ignore the lister generation and + // keep the cheap RV/spec fast-skip. if rvsMatchCP(cp, e.RV) && rvsMatchCP(userDefinedCP, e.UserCPRV) && - e.SpecHash == currentSpecHash { + e.SpecHash == currentSpecHash && + (!e.UsesServiceResolution || e.ListerGen == c.listerGen()) { return } @@ -490,27 +496,29 @@ func (c *ContainerProfileCacheImpl) rebuildEntryFromSources( // Project under the current spec. spec := c.snapshotSpec() applyStart := time.Now() - projectedCP := Apply(spec, projected, tree) + projectedCP := Apply(spec, networkpeer.WithResolvedServiceNeighbors(projected, c.serviceLister), tree) if c.cfg.ProfileProjection.DetailedMetricsEnabled { c.metricsManager.ObserveProjectionApplyDuration(time.Since(applyStart)) c.observeMemoryMetrics(projected, projectedCP) } newEntry := &CachedContainerProfile{ - Projected: projectedCP, - SpecHash: projectedCP.SpecHash, - State: &objectcache.ProfileState{Completion: effectiveCP.Annotations[helpersv1.CompletionMetadataKey], Status: effectiveCP.Annotations[helpersv1.StatusMetadataKey], Name: effectiveCP.Name}, - CallStackTree: tree, - ContainerName: prev.ContainerName, - PodName: prev.PodName, - Namespace: prev.Namespace, - PodUID: podUID, - WorkloadID: prev.WorkloadID, - CPName: prev.CPName, - WorkloadName: prev.WorkloadName, - RV: rvOfCP(cp), - UserCPRV: rvOfCP(userDefinedCP), - terminatedSeenAt: prev.terminatedSeenAt, + Projected: projectedCP, + SpecHash: projectedCP.SpecHash, + UsesServiceResolution: networkpeer.HasServiceNeighbors(projected), + ListerGen: c.listerGen(), + State: &objectcache.ProfileState{Completion: effectiveCP.Annotations[helpersv1.CompletionMetadataKey], Status: effectiveCP.Annotations[helpersv1.StatusMetadataKey], Name: effectiveCP.Name}, + CallStackTree: tree, + ContainerName: prev.ContainerName, + PodName: prev.PodName, + Namespace: prev.Namespace, + PodUID: podUID, + WorkloadID: prev.WorkloadID, + CPName: prev.CPName, + WorkloadName: prev.WorkloadName, + RV: rvOfCP(cp), + UserCPRV: rvOfCP(userDefinedCP), + terminatedSeenAt: prev.terminatedSeenAt, } if userDefinedCP != nil { // The user-authored CP is authoritative and complete by definition (no diff --git a/tests/component_test.go b/tests/component_test.go index 154e441e8b..fa37964a44 100644 --- a/tests/component_test.go +++ b/tests/component_test.go @@ -3906,3 +3906,92 @@ func Test_49_EphemeralContainerFullTreatment(t *testing.T) { return countRuleAlerts(t, ns.Name, "R0001", "ephcon", "id") > 0 }, 2*time.Minute, 10*time.Second, "id was not in the ephemeral container's learned profile — it must fire R0001 (detected + alerted like any other container)") } + +// Test_50_ServiceRefNetworkNeighbor validates the serviceRef selector end to +// end (k8sstormcenter/node-agent#92): a workload whose ContainerProfile +// allowlists egress by Service NAME (default/kubernetes — the apiserver, a +// service every workload legitimately reaches) must NOT fire R0011 for that +// egress, while egress to a real but UNLISTED in-cluster Service (kube-dns) +// MUST still fire R0011. That contrast is the whole point of serviceRef over a +// broad serviceCIDR ipAddresses entry: a narrow, portable allowlist that does +// not blind R0011 to lateral movement. No toy target manifests — the peers are +// the cluster's own infrastructure Services. +func Test_50_ServiceRefNetworkNeighbor(t *testing.T) { + start := time.Now() + defer tearDownTest(t, start) + + getClusterIP := func(t *testing.T, ns, name string) string { + t.Helper() + k := k8sinterface.NewKubernetesApi() + svc, err := k.KubernetesClient.CoreV1().Services(ns).Get(context.TODO(), name, metav1.GetOptions{}) + require.NoError(t, err, "must read %s/%s ClusterIP", ns, name) + require.NotEmpty(t, svc.Spec.ClusterIP) + return svc.Spec.ClusterIP + } + countR0011 := func(alerts []testutils.Alert) int { + n := 0 + for _, a := range alerts { + if a.Labels["rule_id"] == "R0011" { + n++ + } + } + return n + } + waitAlerts := func(t *testing.T, ns string) []testutils.Alert { + t.Helper() + var alerts []testutils.Alert + require.Eventually(t, func() bool { + var err error + alerts, err = testutils.GetAlerts(ns) + return err == nil + }, 60*time.Second, 5*time.Second, "must be able to fetch alerts") + time.Sleep(10 * time.Second) + alerts, _ = testutils.GetAlerts(ns) + return alerts + } + + ns := testutils.NewRandomNamespace() + _ = applyUserDefinedContainerProfile(t, ns.Name, "resources/containerprofile-serviceref-network.yaml") + + wl, err := testutils.NewTestWorkload(ns.Name, + path.Join(utils.CurrentDir(), "resources/serviceref-client-deployment.yaml")) + require.NoError(t, err) + require.NoError(t, wl.WaitForReady(80)) + // Let node-agent load the bound profile AND sync its Service informer + // (serviceRef resolves against live cluster state) before generating traffic. + time.Sleep(40 * time.Second) + + apiserverIP := getClusterIP(t, "default", "kubernetes") + t.Logf("apiserver ClusterIP=%s (serviceRef-allowed); unlisted egress target=1.1.1.1:80", apiserverIP) + + // Phase 1 — egress to the apiserver, allowlisted by serviceRef + // default/kubernetes. The TCP connect is what R0011 evaluates; -k so curl + // attempts it despite the self-signed cert. + t.Run("serviceref_allowed_no_r0011", func(t *testing.T) { + for i := 0; i < 3; i++ { + so, se, e := wl.ExecIntoPod([]string{"curl", "-skm", "5", fmt.Sprintf("https://%s:443/healthz", apiserverIP)}, "curl") + t.Logf("curl apiserver → err=%v out=%q stderr=%q", e, so, se) + } + alerts := waitAlerts(t, wl.Namespace) + assert.Equal(t, 0, countR0011(alerts), + "apiserver egress is allowlisted by serviceRef default/kubernetes — R0011 must NOT fire") + }) + + // Phase 2 — egress NOT covered by the serviceRef must still fire R0011, + // proving serviceRef is a NARROW allowlist (only default/kubernetes), not a + // blanket that suppresses everything. Raw-IP egress to 1.1.1.1:80 is the + // proven R0011 trigger in this suite (mirrors Test_28c) and is the faithful + // analog of the flux RCA, where R0011 fired for the external github egress + // the named-service allowlist did not cover. + t.Run("uncovered_egress_fires_r0011", func(t *testing.T) { + before := countR0011(waitAlerts(t, wl.Namespace)) + for i := 0; i < 3; i++ { + so, se, e := wl.ExecIntoPod([]string{"curl", "-sm", "5", "http://1.1.1.1:80"}, "curl") + t.Logf("curl 1.1.1.1 (uncovered) → err=%v out=%q stderr=%q", e, so, se) + } + require.Eventually(t, func() bool { + return countR0011(waitAlerts(t, wl.Namespace)) > before + }, 3*time.Minute, 15*time.Second, + "egress uncovered by serviceRef MUST fire R0011 — serviceRef is narrow, not a blanket allow") + }) +} diff --git a/tests/resources/containerprofile-serviceref-network.yaml b/tests/resources/containerprofile-serviceref-network.yaml new file mode 100644 index 0000000000..4db29ba9b2 --- /dev/null +++ b/tests/resources/containerprofile-serviceref-network.yaml @@ -0,0 +1,48 @@ +apiVersion: spdx.softwarecomposition.kubescape.io/v1beta1 +kind: ContainerProfile +metadata: + name: serviceref-overlay +spec: + execs: + - path: /bin/sleep + - path: /usr/bin/curl + syscalls: + - socket + - connect + - sendto + - recvfrom + - read + - write + - close + - openat + - mmap + - mprotect + - munmap + - fcntl + - ioctl + - poll + - epoll_create1 + - epoll_ctl + - epoll_wait + - bind + - listen + - accept4 + - getsockopt + - setsockopt + - getsockname + - getpid + - fstat + - rt_sigaction + - rt_sigprocmask + - writev + matchLabels: + app: serviceref-client + egress: + - identifier: apiserver-serviceref + type: internal + serviceRefNamespace: default + serviceRefName: kubernetes + ports: + - name: TCP-443 + protocol: TCP + port: 443 diff --git a/tests/resources/network_fixture_lint_test.go b/tests/resources/network_fixture_lint_test.go index a7ec6cd0e2..376af7a878 100644 --- a/tests/resources/network_fixture_lint_test.go +++ b/tests/resources/network_fixture_lint_test.go @@ -74,6 +74,11 @@ type netEndpoint struct { // declared target for R-NN-12. PodSelector json.RawMessage `json:"podSelector"` NamespaceSelector json.RawMessage `json:"namespaceSelector"` + // Service/entity targets resolved to concrete IPs at projection time. + // Presence of any counts as a declared target for R-NN-12. + ServiceRefName string `json:"serviceRefName"` + ServiceSelector json.RawMessage `json:"serviceSelector"` + Entity string `json:"entity"` } // hasSelector reports whether a raw selector field was set to a real object @@ -114,7 +119,7 @@ func (v NetViolation) String() string { // R-NN-02 — at least one endpoint (egress or ingress) declared // R-NN-10 — endpoint identifier non-empty // R-NN-11 — endpoint type in {internal, external} (or unset) -// R-NN-12 — endpoint declares at least one target (dnsNames/ipAddresses/dns/ipAddress) +// R-NN-12 — endpoint declares at least one target (dnsNames/ipAddresses/dns/ipAddress/selector/serviceRef/entity) // R-NN-13 — dnsNames wildcard tokens are whole-label; no recursive "**", no ascii "..." // R-NN-14 — an entry MUST NOT set both singular ipAddress and plural ipAddresses // R-NN-15 — ipAddresses entries are a literal IP, a CIDR, or the "*" sentinel @@ -176,8 +181,9 @@ func lintEndpoint(dir string, e netEndpoint, add func(rule, msg string)) { add("R-NN-11", where(fmt.Sprintf("type %q is not internal|external", e.Type))) } if len(e.DNSNames) == 0 && len(e.IPAddresses) == 0 && e.DNS == "" && e.IPAddress == "" && - !hasSelector(e.PodSelector) && !hasSelector(e.NamespaceSelector) { - add("R-NN-12", where("endpoint declares no target (dnsNames/ipAddresses/dns/ipAddress/selector)")) + !hasSelector(e.PodSelector) && !hasSelector(e.NamespaceSelector) && + e.ServiceRefName == "" && !hasSelector(e.ServiceSelector) && e.Entity == "" { + add("R-NN-12", where("endpoint declares no target (dnsNames/ipAddresses/dns/ipAddress/selector/serviceRef/entity)")) } if e.IPAddress != "" && len(e.IPAddresses) > 0 { add("R-NN-14", where("sets both singular ipAddress and plural ipAddresses — pick one")) diff --git a/tests/resources/serviceref-client-deployment.yaml b/tests/resources/serviceref-client-deployment.yaml new file mode 100644 index 0000000000..7459155909 --- /dev/null +++ b/tests/resources/serviceref-client-deployment.yaml @@ -0,0 +1,21 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: serviceref-client + name: serviceref-client +spec: + replicas: 1 + selector: + matchLabels: + app: serviceref-client + template: + metadata: + labels: + app: serviceref-client + kubescape.io/user-defined-profile: serviceref-overlay + spec: + containers: + - name: curl + image: docker.io/curlimages/curl@sha256:08e466006f0860e54fc299378de998935333e0e130a15f6f98482e9f8dab3058 + command: ["sleep", "infinity"] From 6f30d65a4b78cc21b4e7ea6d2be57862a4c98d0c Mon Sep 17 00:00:00 2001 From: tanzee Date: Sun, 23 Aug 2026 23:11:11 +0200 Subject: [PATCH 06/19] feat(cel/network): real-Flux component test, RBAC + perf fixes for serviceRef MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Component test Test_50 now generates its traffic from a real Flux source-controller reconciling HelmRepository CRs instead of exec'ing curl, and its ContainerProfile is network-only (no syscalls/execs, which only add false-positive surface to a network test). The profile names every peer as a Kubernetes object: serviceRef default/kubernetes for the apiserver, serviceRef kube-system/kube-dns for resolution, and a serviceSelector role=helm-repo fanning across the two repo Services. The negative is the lateral move a serviceCIDR entry hides: the HelmRepository URL is repointed at a sibling Service on the same port that the selector does not cover, and the controller fetches it itself. Verified on kind: 0 alerts for the named peers, R0011 within 15s for the sibling. Fixes found while validating end to end: - ClusterRole was missing discovery.k8s.io/endpointslices, so the informer was forbidden and Service endpoint IPs never resolved — the feature silently degraded to ClusterIP-only. - Service/EndpointSlice informers are now gated behind networkServiceResolutionEnabled and strip managedFields/annotations (and per-endpoint fields beyond Addresses) via SetTransform, so agents that do not use the feature pay no cluster-wide list+watch and the cache stays small on those that do. - serviceRef/serviceSelector now also imply the Service cluster FQDN as a dnsName, so a client dialling the Service by name is allowlisted without a parallel dnsNames entry. - specFromNeighbor no longer allocates a discarded port slice for every plain ipAddresses neighbor. - R0011 no longer excludes private destinations: in-cluster lateral movement is exactly what this feature exists to expose. Signed-off-by: tanzee --- cmd/main.go | 95 +++--- pkg/config/config.go | 1 + pkg/networkpeer/expand.go | 18 +- pkg/networkpeer/expand_test.go | 8 + pkg/networkpeer/lister.go | 25 ++ pkg/networkpeer/perf_bench_test.go | 283 ++++++++++++++++++ pkg/networkpeer/resolve.go | 49 ++- pkg/networkpeer/resolve_test.go | 24 ++ .../templates/node-agent/clusterrole.yaml | 3 + .../chart/templates/node-agent/configmap.yaml | 1 + .../templates/node-agent/default-rules.yaml | 2 +- tests/component_test.go | 204 +++++++++---- .../containerprofile-serviceref-network.yaml | 48 --- .../serviceref-client-deployment.yaml | 21 -- .../serviceref-suite/00-flux-source-crds.yaml | 132 ++++++++ .../serviceref-suite/10-helm-repo.yaml | 108 +++++++ .../20-source-controller.yaml | 117 ++++++++ tests/testutils/k8s.go | 75 +++++ 18 files changed, 1024 insertions(+), 190 deletions(-) create mode 100644 pkg/networkpeer/perf_bench_test.go delete mode 100644 tests/resources/containerprofile-serviceref-network.yaml delete mode 100644 tests/resources/serviceref-client-deployment.yaml create mode 100644 tests/resources/serviceref-suite/00-flux-source-crds.yaml create mode 100644 tests/resources/serviceref-suite/10-helm-repo.yaml create mode 100644 tests/resources/serviceref-suite/20-source-controller.yaml diff --git a/cmd/main.go b/cmd/main.go index 92fd665d24..bfec59e977 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -325,49 +325,60 @@ func main() { ruleBindingCache.AddNotifier(&ruleBindingNotify) cpc := containerprofilecache.NewContainerProfileCache(cfg, storageClient, k8sObjectCache, metricsProvider) - // Resolve serviceRef/serviceSelector/entity network neighbors against - // live cluster state (Service ClusterIPs + endpoints, Node IPs + CNI - // gateway) at projection time. Services and EndpointSlices are watched - // cluster-wide (a profile may reference any namespace's Service); the - // Node informer is field-selected to this agent's own node — the "host" - // entity is local, and a cluster-wide Node watch on every DaemonSet pod - // is O(nodes^2) traffic for no benefit. - svcInformers := informers.NewSharedInformerFactory(k8sClient.GetKubernetesClient(), 0) - nodeInformers := informers.NewSharedInformerFactoryWithOptions( - k8sClient.GetKubernetesClient(), 0, - informers.WithTweakListOptions(func(o *metav1.ListOptions) { - o.FieldSelector = "metadata.name=" + cfg.NodeName - }), - ) - serviceLister := networkpeer.NewInformerLister( - svcInformers.Core().V1().Services().Lister(), - svcInformers.Discovery().V1().EndpointSlices().Lister(), - nodeInformers.Core().V1().Nodes().Lister(), - cfg.NodeName, - ) - // Advance the lister generation on any Service/EndpointSlice/Node change, - // so the reconciler re-projects serviceRef/entity profiles when the - // cluster view moves (endpoint churn, or caches that fill after startup). - // Per-event cost is a single atomic increment; the re-projection itself - // is coalesced onto the reconcile tick, and only serviceRef-using - // profiles are eligible. - bump := toolscache.ResourceEventHandlerFuncs{ - AddFunc: func(interface{}) { serviceLister.Bump() }, - UpdateFunc: func(_, _ interface{}) { serviceLister.Bump() }, - DeleteFunc: func(interface{}) { serviceLister.Bump() }, + // Resolve serviceRef/serviceSelector/entity network neighbors against live + // cluster state (Service ClusterIPs + endpoints, Node IPs + CNI gateway) at + // projection time. Gated behind networkServiceResolutionEnabled: the + // cluster-wide Service+EndpointSlice list+watch (one per DaemonSet node) is + // only paid where profiles actually use the feature. Services and + // EndpointSlices are watched cluster-wide (a profile may reference any + // namespace's Service); the Node informer is field-selected to this agent's + // own node — the "host" entity is local, and a cluster-wide Node watch on + // every DaemonSet pod is O(nodes^2) traffic for no benefit. A TransformFunc + // strips managedFields/annotations (and per-endpoint fields beyond + // Addresses) before objects enter the cache to keep its footprint small. + if cfg.EnableNetworkServiceResolution { + svcInformers := informers.NewSharedInformerFactory(k8sClient.GetKubernetesClient(), 0) + nodeInformers := informers.NewSharedInformerFactoryWithOptions( + k8sClient.GetKubernetesClient(), 0, + informers.WithTweakListOptions(func(o *metav1.ListOptions) { + o.FieldSelector = "metadata.name=" + cfg.NodeName + }), + ) + svcInformer := svcInformers.Core().V1().Services().Informer() + sliceInformer := svcInformers.Discovery().V1().EndpointSlices().Informer() + _ = svcInformer.SetTransform(networkpeer.TrimService) + _ = sliceInformer.SetTransform(networkpeer.TrimEndpointSlice) + serviceLister := networkpeer.NewInformerLister( + svcInformers.Core().V1().Services().Lister(), + svcInformers.Discovery().V1().EndpointSlices().Lister(), + nodeInformers.Core().V1().Nodes().Lister(), + cfg.NodeName, + ) + // Advance the lister generation on any Service/EndpointSlice/Node + // change, so the reconciler re-projects serviceRef/entity profiles when + // the cluster view moves (endpoint churn, or caches that fill after + // startup). Per-event cost is a single atomic increment; the + // re-projection itself is coalesced onto the reconcile tick, and only + // serviceRef-using profiles are eligible. + bump := toolscache.ResourceEventHandlerFuncs{ + AddFunc: func(interface{}) { serviceLister.Bump() }, + UpdateFunc: func(_, _ interface{}) { serviceLister.Bump() }, + DeleteFunc: func(interface{}) { serviceLister.Bump() }, + } + _, _ = svcInformer.AddEventHandler(bump) + _, _ = sliceInformer.AddEventHandler(bump) + _, _ = nodeInformers.Core().V1().Nodes().Informer().AddEventHandler(bump) + // Start the informers and hand the lister over WITHOUT blocking on + // cache sync: node-agent's core startup (container watcher, profiling) + // must not wait on these, and a bounded wait here previously delayed + // learning enough to trip the tight completion budget of Test_22. The + // caches fill in the background; serviceRef/entity neighbors resolve on + // the next reconcile once populated. nil-until-set is a no-op in + // projection. + svcInformers.Start(ctx.Done()) + nodeInformers.Start(ctx.Done()) + cpc.SetServiceLister(serviceLister) } - _, _ = svcInformers.Core().V1().Services().Informer().AddEventHandler(bump) - _, _ = svcInformers.Discovery().V1().EndpointSlices().Informer().AddEventHandler(bump) - _, _ = nodeInformers.Core().V1().Nodes().Informer().AddEventHandler(bump) - // Start the informers and hand the lister over WITHOUT blocking on cache - // sync: node-agent's core startup (container watcher, profiling) must not - // wait on these, and a bounded wait here previously delayed learning - // enough to trip the tight completion budget of Test_22. The caches fill - // in the background; serviceRef/entity neighbors resolve on the next - // reconcile once populated. nil-until-set is a no-op in projection. - svcInformers.Start(ctx.Done()) - nodeInformers.Start(ctx.Done()) - cpc.SetServiceLister(serviceLister) cpc.Start(ctx) if cpm, ok := containerProfileManager.(*containerprofilemanagerv1.ContainerProfileManager); ok { cpm.SetCompletionNotifier(cpc) diff --git a/pkg/config/config.go b/pkg/config/config.go index cec9f41ab6..e2d6e5d4a9 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -83,6 +83,7 @@ type Config struct { EnableMalwareDetection bool `mapstructure:"malwareDetectionEnabled"` EnableNetworkStreaming bool `mapstructure:"networkStreamingEnabled"` EnableNetworkTracing bool `mapstructure:"networkServiceEnabled"` + EnableNetworkServiceResolution bool `mapstructure:"networkServiceResolutionEnabled"` EnableNodeProfile bool `mapstructure:"nodeProfileServiceEnabled"` EnablePartialProfileGeneration bool `mapstructure:"partialProfileGenerationEnabled"` EnableMetricsExporter bool `mapstructure:"prometheusExporterEnabled"` diff --git a/pkg/networkpeer/expand.go b/pkg/networkpeer/expand.go index 2f76c97897..747c7b2d62 100644 --- a/pkg/networkpeer/expand.go +++ b/pkg/networkpeer/expand.go @@ -11,10 +11,12 @@ import ( // synthesized ipAddresses neighbors (one per source neighbor, carrying the // resolved IPs and the source neighbor's own ports). // -// The synthesized neighbors are ordinary selector-free ipAddresses entries, so -// the existing port-sensitive address matcher handles them with no further +// The synthesized neighbors are ordinary selector-free ipAddresses entries +// (plus, for Service-backed specs, the Service's cluster FQDN as a dnsName so a +// client dialling it by name is allowlisted too), so the existing +// port-sensitive address matcher and DNS matcher handle them with no further // change — a serviceRef/host neighbor becomes exactly the narrow, resolved -// ipAddresses entry it stands for. Neighbors that resolve to nothing (unknown +// entry it stands for. Neighbors that resolve to nothing (unknown // Service, selector matching nothing, unknown entity) contribute nothing — // never a match-all. Callers append the result to the same direction (egress // or ingress) before projecting the profile. @@ -30,13 +32,15 @@ func ExpandServiceNeighbors(neighbors []v1beta1.NetworkNeighbor, l Lister) []v1b continue } ips := ResolveIPs(spec, l) - if len(ips) == 0 { + dnsNames := ResolveDNSNames(spec, l) + if len(ips) == 0 && len(dnsNames) == 0 { continue } out = append(out, v1beta1.NetworkNeighbor{ Identifier: n.Identifier + "-resolved", Type: n.Type, IPAddresses: ips, + DNSNames: dnsNames, Ports: n.Ports, }) } @@ -96,6 +100,12 @@ func hasServiceFields(n *v1beta1.NetworkNeighbor) bool { // if the neighbor declares none of the service/entity selectors (a plain // ipAddresses / dnsNames / podSelector neighbor is left untouched). func specFromNeighbor(n *v1beta1.NetworkNeighbor) (PeerSpec, bool) { + // Cheap-reject a plain ipAddresses/dnsNames neighbor before allocating a + // []PortProto it would only discard (hot on every projection's non-service + // neighbors). + if n.Entity == "" && n.ServiceRefName == "" && n.ServiceSelector == nil { + return PeerSpec{}, false + } spec := PeerSpec{Ports: portsFromNeighbor(n.Ports)} switch { case n.Entity != "": diff --git a/pkg/networkpeer/expand_test.go b/pkg/networkpeer/expand_test.go index dfb397b879..715a76e607 100644 --- a/pkg/networkpeer/expand_test.go +++ b/pkg/networkpeer/expand_test.go @@ -48,6 +48,10 @@ func TestExpandServiceNeighbors_Egress(t *testing.T) { if len(got.Ports) != 1 || got.Ports[0].Port == nil || *got.Ports[0].Port != 9093 { t.Errorf("ports not carried over: %+v", got.Ports) } + // serviceRef implies the Service cluster FQDN as a dnsName (R0005 suppression). + if len(got.DNSNames) != 1 || got.DNSNames[0] != "alertmanager.honey.svc.cluster.local" { + t.Errorf("serviceRef FQDN not emitted: %v", got.DNSNames) + } } // TestExpandServiceNeighbors_HostEntity: fromEntity host resolves to node + @@ -61,6 +65,10 @@ func TestExpandServiceNeighbors_HostEntity(t *testing.T) { if len(out) != 1 { t.Fatalf("expected 1 synthesized host neighbor, got %d", len(out)) } + // host entity is not a Service: no FQDN. + if len(out[0].DNSNames) != 0 { + t.Errorf("host entity must not emit a dnsName: %v", out[0].DNSNames) + } // Feed the synthesized entry through the address matcher the same way the // projection would, to prove end-to-end intent. tuples := Resolve(PeerSpec{Entity: "host", Ports: []PortProto{{Port: 9440, Protocol: "TCP"}}}, l) diff --git a/pkg/networkpeer/lister.go b/pkg/networkpeer/lister.go index eff6d349db..5c88c7d7ba 100644 --- a/pkg/networkpeer/lister.go +++ b/pkg/networkpeer/lister.go @@ -126,6 +126,31 @@ func (l *InformerLister) endpointIPs(namespace, service string) []string { return dedupe(ips) } +// TrimService and TrimEndpointSlice are informer TransformFuncs that drop the +// bulk the resolver never reads — managedFields and annotations (1–4 KiB per +// real object), and for EndpointSlices every per-endpoint field but Addresses — +// before objects enter the cluster-wide cache. Wire via Informer().SetTransform +// so a DaemonSet's per-node Service/EndpointSlice cache stays small. Identity +// and resourceVersion are preserved so listing/indexing is unaffected. +func TrimService(obj interface{}) (interface{}, error) { + if svc, ok := obj.(*corev1.Service); ok { + svc.ManagedFields = nil + svc.Annotations = nil + } + return obj, nil +} + +func TrimEndpointSlice(obj interface{}) (interface{}, error) { + if es, ok := obj.(*discoveryv1.EndpointSlice); ok { + es.ManagedFields = nil + es.Annotations = nil + for i := range es.Endpoints { + es.Endpoints[i] = discoveryv1.Endpoint{Addresses: es.Endpoints[i].Addresses} + } + } + return obj, nil +} + func podCIDRs(n *corev1.Node) []string { if len(n.Spec.PodCIDRs) > 0 { return n.Spec.PodCIDRs diff --git a/pkg/networkpeer/perf_bench_test.go b/pkg/networkpeer/perf_bench_test.go new file mode 100644 index 0000000000..e3343214b8 --- /dev/null +++ b/pkg/networkpeer/perf_bench_test.go @@ -0,0 +1,283 @@ +package networkpeer + +import ( + "fmt" + "runtime" + "testing" + + "github.com/kubescape/storage/pkg/apis/softwarecomposition/v1beta1" + corev1 "k8s.io/api/core/v1" + discoveryv1 "k8s.io/api/discovery/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + corelisters "k8s.io/client-go/listers/core/v1" + discoverylisters "k8s.io/client-go/listers/discovery/v1" + "k8s.io/client-go/tools/cache" +) + +func benchService(i int) *corev1.Service { + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: fmt.Sprintf("ns-%d", i%50), + Name: fmt.Sprintf("svc-%d", i), + Labels: map[string]string{"app": fmt.Sprintf("app-%d", i), "team": fmt.Sprintf("team-%d", i%20)}, + }, + Spec: corev1.ServiceSpec{ + ClusterIP: fmt.Sprintf("10.43.%d.%d", i/256, i%256), + ClusterIPs: []string{fmt.Sprintf("10.43.%d.%d", i/256, i%256)}, + Ports: []corev1.ServicePort{{Name: "http", Port: 8080, Protocol: corev1.ProtocolTCP}}, + }, + } +} + +func benchSlice(svcIdx, sliceIdx, endpoints int) *discoveryv1.EndpointSlice { + eps := make([]discoveryv1.Endpoint, 0, endpoints) + ready := true + for e := 0; e < endpoints; e++ { + eps = append(eps, discoveryv1.Endpoint{ + Addresses: []string{fmt.Sprintf("10.42.%d.%d", (svcIdx*7+e)%256, (sliceIdx*31+e)%256)}, + Conditions: discoveryv1.EndpointConditions{Ready: &ready}, + TargetRef: &corev1.ObjectReference{Kind: "Pod", Namespace: fmt.Sprintf("ns-%d", svcIdx%50), Name: fmt.Sprintf("pod-%d-%d-%d", svcIdx, sliceIdx, e)}, + NodeName: ptrTo(fmt.Sprintf("node-%d", e%10)), + }) + } + return &discoveryv1.EndpointSlice{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: fmt.Sprintf("ns-%d", svcIdx%50), + Name: fmt.Sprintf("svc-%d-%d", svcIdx, sliceIdx), + Labels: map[string]string{discoveryv1.LabelServiceName: fmt.Sprintf("svc-%d", svcIdx)}, + }, + AddressType: discoveryv1.AddressTypeIPv4, + Endpoints: eps, + Ports: []discoveryv1.EndpointPort{{Name: ptrTo("http"), Port: ptrTo(int32(8080)), Protocol: &[]corev1.Protocol{corev1.ProtocolTCP}[0]}}, + } +} + +func ptrTo[T any](v T) *T { return &v } + +// buildBenchLister backs an InformerLister with plain cache indexers (the same +// store type a SharedInformer uses) so benchmarks measure lister/resolution +// cost without fake-clientset watch machinery. +func buildBenchLister(tb testing.TB, nServices, slicesPerSvc, endpointsPerSlice int) *InformerLister { + tb.Helper() + svcIdx := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + sliceIdx := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + nodeIdx := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{}) + for i := 0; i < nServices; i++ { + if err := svcIdx.Add(benchService(i)); err != nil { + tb.Fatal(err) + } + for s := 0; s < slicesPerSvc; s++ { + if err := sliceIdx.Add(benchSlice(i, s, endpointsPerSlice)); err != nil { + tb.Fatal(err) + } + } + } + if err := nodeIdx.Add(&corev1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: "bench-node"}, + Spec: corev1.NodeSpec{PodCIDR: "10.42.0.0/24", PodCIDRs: []string{"10.42.0.0/24"}}, + Status: corev1.NodeStatus{Addresses: []corev1.NodeAddress{{Type: corev1.NodeInternalIP, Address: "192.168.0.191"}}}, + }); err != nil { + tb.Fatal(err) + } + return NewInformerLister( + corelisters.NewServiceLister(svcIdx), + discoverylisters.NewEndpointSliceLister(sliceIdx), + corelisters.NewNodeLister(nodeIdx), + "bench-node", + ) +} + +// benchProfile builds a ContainerProfile with nPlain ordinary ipAddresses +// egress neighbors, nRefs serviceRef neighbors, and some opens/execs bulk so +// DeepCopy cost is realistic. +func benchProfile(nPlain, nRefs, nOpens int) *v1beta1.ContainerProfile { + cp := &v1beta1.ContainerProfile{} + cp.Name = "bench-cp" + port := int32(8080) + for i := 0; i < nPlain; i++ { + cp.Spec.Egress = append(cp.Spec.Egress, v1beta1.NetworkNeighbor{ + Identifier: fmt.Sprintf("plain-%d", i), + Type: "external", + IPAddresses: []string{fmt.Sprintf("52.216.%d.%d", i/256, i%256)}, + Ports: []v1beta1.NetworkPort{{Name: "TCP-8080", Protocol: "TCP", Port: &port}}, + }) + } + for i := 0; i < nRefs; i++ { + cp.Spec.Egress = append(cp.Spec.Egress, v1beta1.NetworkNeighbor{ + Identifier: fmt.Sprintf("ref-%d", i), + Type: "internal", + ServiceRefNamespace: fmt.Sprintf("ns-%d", i%50), + ServiceRefName: fmt.Sprintf("svc-%d", i), + Ports: []v1beta1.NetworkPort{{Name: "TCP-8080", Protocol: "TCP", Port: &port}}, + }) + } + for i := 0; i < nOpens; i++ { + cp.Spec.Opens = append(cp.Spec.Opens, v1beta1.OpenCalls{ + Path: fmt.Sprintf("/usr/lib/x86_64-linux-gnu/lib-%d.so.%d", i, i%9), + Flags: []string{"O_RDONLY", "O_CLOEXEC"}, + }) + cp.Spec.Execs = append(cp.Spec.Execs, v1beta1.ExecCalls{ + Path: fmt.Sprintf("/usr/bin/tool-%d", i), + Args: []string{fmt.Sprintf("--flag-%d", i)}, + }) + } + return cp +} + +func BenchmarkServiceByName_1kSvc_5kSlices(b *testing.B) { + l := buildBenchLister(b, 1000, 5, 10) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, ok := l.ServiceByName("ns-7", "svc-7"); !ok { + b.Fatal("service must resolve") + } + } +} + +func BenchmarkServicesByLabels_1kSvc(b *testing.B) { + l := buildBenchLister(b, 1000, 5, 10) + sel := map[string]string{"app": "app-7"} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if out := l.ServicesByLabels(sel, nil); len(out) != 1 { + b.Fatalf("want 1 service, got %d", len(out)) + } + } +} + +func BenchmarkResolveIPs_ServiceRef(b *testing.B) { + l := buildBenchLister(b, 1000, 5, 10) + spec := PeerSpec{ServiceRef: &ServiceRef{Namespace: "ns-7", Name: "svc-7"}} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if ips := ResolveIPs(spec, l); len(ips) == 0 { + b.Fatal("must resolve") + } + } +} + +func BenchmarkResolveIPs_Entity_Host(b *testing.B) { + l := buildBenchLister(b, 10, 1, 2) + spec := PeerSpec{Entity: "host"} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if ips := ResolveIPs(spec, l); len(ips) == 0 { + b.Fatal("must resolve") + } + } +} + +// BenchmarkWithResolvedServiceNeighbors_NoServiceFields is the 99% case: a +// profile with only plain ipAddresses neighbors. The function documents itself +// as a no-op then — this measures whether the no-op is actually free. +func BenchmarkWithResolvedServiceNeighbors_NoServiceFields(b *testing.B) { + l := buildBenchLister(b, 1000, 5, 10) + for _, n := range []int{100, 1000} { + cp := benchProfile(n, 0, 0) + b.Run(fmt.Sprintf("plainNeighbors=%d", n), func(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + out := WithResolvedServiceNeighbors(cp, l) + if out != cp { + b.Fatal("no-op path must return the same pointer") + } + } + }) + } +} + +// BenchmarkWithResolvedServiceNeighbors_Resolving measures the full expansion: +// resolution + DeepCopy of the whole profile (opens/execs bulk included). +func BenchmarkWithResolvedServiceNeighbors_Resolving(b *testing.B) { + l := buildBenchLister(b, 1000, 5, 10) + for _, tc := range []struct{ plain, refs, opens int }{ + {plain: 20, refs: 1, opens: 200}, + {plain: 20, refs: 8, opens: 200}, + {plain: 20, refs: 8, opens: 2000}, + } { + cp := benchProfile(tc.plain, tc.refs, tc.opens) + b.Run(fmt.Sprintf("plain=%d/refs=%d/opens=%d", tc.plain, tc.refs, tc.opens), func(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + out := WithResolvedServiceNeighbors(cp, l) + if out == cp { + b.Fatal("resolving path must copy") + } + } + }) + } +} + +func BenchmarkHasServiceNeighbors_1kPlain(b *testing.B) { + cp := benchProfile(1000, 0, 0) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if HasServiceNeighbors(cp) { + b.Fatal("plain profile must not report service neighbors") + } + } +} + +// TestInformerCacheMemoryEstimate approximates the heap retained by a +// cluster-wide Service + EndpointSlice informer cache at 1k Services / 5k +// EndpointSlices (10 endpoints each), vs Services alone. Run with -run +// InformerCacheMemoryEstimate -v. +func TestInformerCacheMemoryEstimate(t *testing.T) { + measure := func(build func() []interface{}) uint64 { + runtime.GC() + var before, after runtime.MemStats + runtime.ReadMemStats(&before) + objs := build() + runtime.GC() + runtime.ReadMemStats(&after) + runtime.KeepAlive(objs) + if after.HeapAlloc < before.HeapAlloc { + return 0 + } + return after.HeapAlloc - before.HeapAlloc + } + + svcBytes := measure(func() []interface{} { + idx := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + for i := 0; i < 1000; i++ { + _ = idx.Add(benchService(i)) + } + return []interface{}{idx} + }) + sliceBytes := measure(func() []interface{} { + idx := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + for i := 0; i < 1000; i++ { + for s := 0; s < 5; s++ { + _ = idx.Add(benchSlice(i, s, 10)) + } + } + return []interface{}{idx} + }) + strippedBytes := measure(func() []interface{} { + idx := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + for i := 0; i < 1000; i++ { + for s := 0; s < 5; s++ { + full := benchSlice(i, s, 10) + stripped := &discoveryv1.EndpointSlice{ + ObjectMeta: metav1.ObjectMeta{Namespace: full.Namespace, Name: full.Name, Labels: full.Labels}, + AddressType: full.AddressType, + } + for _, ep := range full.Endpoints { + stripped.Endpoints = append(stripped.Endpoints, discoveryv1.Endpoint{Addresses: ep.Addresses}) + } + _ = idx.Add(stripped) + } + } + return []interface{}{idx} + }) + t.Logf("1000 Services in indexer: ~%d KiB total, ~%d B/object", svcBytes/1024, svcBytes/1000) + t.Logf("5000 EndpointSlices (10 endpoints each) in indexer: ~%d KiB total, ~%d B/object", sliceBytes/1024, sliceBytes/5000) + t.Logf("5000 STRIPPED EndpointSlices (addresses+labels only, SetTransform mitigation): ~%d KiB total, ~%d B/object", strippedBytes/1024, strippedBytes/5000) +} diff --git a/pkg/networkpeer/resolve.go b/pkg/networkpeer/resolve.go index 0467c14b50..de8c1375d7 100644 --- a/pkg/networkpeer/resolve.go +++ b/pkg/networkpeer/resolve.go @@ -24,6 +24,12 @@ import "strings" // represent (kubelet health probes, node-sourced / masqueraded traffic). const EntityHost = "host" +// clusterDNSSuffix is the in-cluster Service DNS zone. A serviceRef / +// serviceSelector implies the cluster FQDN ..svc. so a +// client dialling the Service by DNS is allowlisted without a parallel +// dnsNames entry. Kubernetes' default zone; overridden per-cluster is rare. +const clusterDNSSuffix = "svc.cluster.local" + // PortProto is a single allowed destination port/protocol. Protocol is // upper-case ("TCP"/"UDP"); an empty Protocol matches any protocol. type PortProto struct { @@ -105,33 +111,58 @@ func ResolveIPs(spec PeerSpec, l Lister) []string { } func resolveIPs(spec PeerSpec, l Lister) []string { - switch { - case spec.Entity != "": + if spec.Entity != "" { if strings.EqualFold(spec.Entity, EntityHost) { return dedupe(l.HostIPs()) } return nil + } + var ips []string + for _, svc := range resolveServices(spec, l) { + ips = append(ips, serviceIPs(svc)...) + } + return dedupe(ips) +} + +// resolveServices returns the Services a serviceRef / serviceSelector spec +// matches. An entity spec matches no Service and returns nil; an empty +// serviceSelector fails closed (never every Service). +func resolveServices(spec PeerSpec, l Lister) []*ServiceInfo { + switch { case spec.ServiceRef != nil: svc, ok := l.ServiceByName(spec.ServiceRef.Namespace, spec.ServiceRef.Name) if !ok || svc == nil { return nil } - return serviceIPs(svc) + return []*ServiceInfo{svc} case spec.ServiceSelector != nil: - // An empty selector is NOT a cluster-wide match-all: fail closed. if len(spec.ServiceSelector) == 0 { return nil } - var ips []string - for _, svc := range l.ServicesByLabels(spec.ServiceSelector, spec.NamespaceLabels) { - ips = append(ips, serviceIPs(svc)...) - } - return dedupe(ips) + return l.ServicesByLabels(spec.ServiceSelector, spec.NamespaceLabels) default: return nil } } +// ResolveDNSNames returns the cluster FQDN(s) — ..svc. — +// of the Services a serviceRef / serviceSelector spec matches, so a client +// dialling the Service by DNS is allowlisted alongside its IPs. Entity specs +// and unresolvable selectors yield nothing. +func ResolveDNSNames(spec PeerSpec, l Lister) []string { + if l == nil { + return nil + } + var out []string + for _, svc := range resolveServices(spec, l) { + if svc == nil || svc.Namespace == "" || svc.Name == "" { + continue + } + out = append(out, svc.Name+"."+svc.Namespace+"."+clusterDNSSuffix) + } + return dedupe(out) +} + func serviceIPs(svc *ServiceInfo) []string { out := make([]string, 0, len(svc.ClusterIPs)+len(svc.EndpointIPs)) out = append(out, svc.ClusterIPs...) diff --git a/pkg/networkpeer/resolve_test.go b/pkg/networkpeer/resolve_test.go index f131f774fe..0ddcc0ccbe 100644 --- a/pkg/networkpeer/resolve_test.go +++ b/pkg/networkpeer/resolve_test.go @@ -191,6 +191,30 @@ func TestResolve_NoMatchAll(t *testing.T) { } } +// Test A7 — serviceRef/serviceSelector imply the Service cluster FQDN(s); +// entity and unresolvable specs imply none. +func TestResolveDNSNames(t *testing.T) { + l := realFluxTopology() + one := ResolveDNSNames(PeerSpec{ServiceRef: &ServiceRef{"honey", "alertmanager"}}, l) + if len(one) != 1 || one[0] != "alertmanager.honey.svc.cluster.local" { + t.Errorf("serviceRef FQDN: got %v", one) + } + fan := ResolveDNSNames(PeerSpec{ + ServiceSelector: map[string]string{"app": "guestbook"}, + NamespaceLabels: map[string]string{"kubernetes.io/metadata.name": "gitops-demo"}, + }, l) + want := map[string]bool{"guestbook-ui.gitops-demo.svc.cluster.local": true, "helm-guestbook.gitops-demo.svc.cluster.local": true} + if len(fan) != 2 || !want[fan[0]] || !want[fan[1]] { + t.Errorf("selector FQDN fanout: got %v", fan) + } + if got := ResolveDNSNames(PeerSpec{Entity: EntityHost}, l); got != nil { + t.Errorf("host entity implies no FQDN, got %v", got) + } + if got := ResolveDNSNames(PeerSpec{ServiceRef: &ServiceRef{"honey", "nope"}}, l); got != nil { + t.Errorf("unresolvable serviceRef implies no FQDN, got %v", got) + } +} + // Test A6 — a nil Lister and any-port (no Ports) behave safely. func TestResolve_Edges(t *testing.T) { if got := Resolve(PeerSpec{Entity: EntityHost}, nil); got != nil { diff --git a/tests/chart/templates/node-agent/clusterrole.yaml b/tests/chart/templates/node-agent/clusterrole.yaml index 03d5137555..a9feeed81a 100644 --- a/tests/chart/templates/node-agent/clusterrole.yaml +++ b/tests/chart/templates/node-agent/clusterrole.yaml @@ -11,6 +11,9 @@ rules: - apiGroups: [""] resources: ["events"] verbs: ["list", "watch", "create"] +- apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["get", "watch", "list"] - apiGroups: ["batch"] resources: ["jobs", "cronjobs"] verbs: ["get", "watch", "list"] diff --git a/tests/chart/templates/node-agent/configmap.yaml b/tests/chart/templates/node-agent/configmap.yaml index 523b5bbac6..6108427e90 100644 --- a/tests/chart/templates/node-agent/configmap.yaml +++ b/tests/chart/templates/node-agent/configmap.yaml @@ -14,6 +14,7 @@ data: "prometheusExporterEnabled": {{ eq .Values.nodeAgent.config.prometheusExporter "enable" }}, "runtimeDetectionEnabled": {{ eq .Values.capabilities.runtimeDetection "enable" }}, "networkServiceEnabled": {{ eq .Values.capabilities.networkPolicyService "enable" }}, + "networkServiceResolutionEnabled": true, "malwareDetectionEnabled": {{ eq .Values.capabilities.malwareDetection "enable" }}, "httpDetectionEnabled": {{ eq .Values.capabilities.httpDetection "enable" }}, "initialDelay": "{{ .Values.nodeAgent.config.learningPeriod }}", diff --git a/tests/chart/templates/node-agent/default-rules.yaml b/tests/chart/templates/node-agent/default-rules.yaml index 512b4d9ec8..913fb1f927 100644 --- a/tests/chart/templates/node-agent/default-rules.yaml +++ b/tests/chart/templates/node-agent/default-rules.yaml @@ -313,7 +313,7 @@ spec: uniqueId: "event.dstAddr + '_' + string(event.dstPort) + '_' + event.proto" ruleExpression: - eventType: "network" - expression: "event.pktType == 'OUTGOING' && !net.is_private_ip(event.dstAddr) && !cp.was_address_in_egress(event.containerId, event.dstAddr)" + expression: "event.pktType == 'OUTGOING' && !cp.was_address_in_egress(event.containerId, event.dstAddr)" profileDependency: 0 profileDataRequired: egressAddresses: all diff --git a/tests/component_test.go b/tests/component_test.go index fa37964a44..fbb10b6211 100644 --- a/tests/component_test.go +++ b/tests/component_test.go @@ -3907,91 +3907,165 @@ func Test_49_EphemeralContainerFullTreatment(t *testing.T) { }, 2*time.Minute, 10*time.Second, "id was not in the ephemeral container's learned profile — it must fire R0001 (detected + alerted like any other container)") } -// Test_50_ServiceRefNetworkNeighbor validates the serviceRef selector end to -// end (k8sstormcenter/node-agent#92): a workload whose ContainerProfile -// allowlists egress by Service NAME (default/kubernetes — the apiserver, a -// service every workload legitimately reaches) must NOT fire R0011 for that -// egress, while egress to a real but UNLISTED in-cluster Service (kube-dns) -// MUST still fire R0011. That contrast is the whole point of serviceRef over a -// broad serviceCIDR ipAddresses entry: a narrow, portable allowlist that does -// not blind R0011 to lateral movement. No toy target manifests — the peers are -// the cluster's own infrastructure Services. +// Test_50_ServiceRefNetworkNeighbor validates serviceRef/serviceSelector end to +// end (k8sstormcenter/node-agent#92) against REAL GitOps traffic: a Flux +// source-controller reconciling HelmRepository CRs. Its egress is authored +// purely as Kubernetes-native selectors — serviceRef default/kubernetes for the +// apiserver, serviceRef kube-system/kube-dns for name resolution, and a +// serviceSelector role=helm-repo fanning across the two repo Services it is +// allowed to fetch. Nothing is exec'd and no address is hardcoded; the +// controller's own reconcile loop generates every connection. +// +// The negative is the lateral move a broad serviceCIDR entry would hide: the +// HelmRepository URL is repointed at decoy-repo — a sibling Service on the same +// port, backed by its own pod, carrying none of the selector's labels — and the +// controller itself fetches it. R0011 MUST fire for that and MUST NOT fire for +// the allowlisted Services. func Test_50_ServiceRefNetworkNeighbor(t *testing.T) { start := time.Now() defer tearDownTest(t, start) - getClusterIP := func(t *testing.T, ns, name string) string { + const ( + cpName = "serviceref-flux-cp" + containerName = "manager" + ) + port80, port443, port53 := int32(80), int32(443), int32(53) + + ns := testutils.NewRandomNamespace() + k8sClient := k8sinterface.NewKubernetesApi() + storageClient := spdxv1beta1client.NewForConfigOrDie(k8sClient.K8SConfig) + + // Authored profile: NETWORK ONLY. No syscalls/execs — they are irrelevant to + // a network test and only add false-positive surface. Every peer is named, + // never addressed. + cp := &v1beta1.ContainerProfile{ + ObjectMeta: metav1.ObjectMeta{Name: cpName, Namespace: ns.Name}, + Spec: v1beta1.ContainerProfileSpec{ + LabelSelector: metav1.LabelSelector{MatchLabels: map[string]string{"app": "source-controller"}}, + Egress: []v1beta1.NetworkNeighbor{ + { + Identifier: "apiserver", + Type: v1beta1.CommunicationTypeEgress, + ServiceRefNamespace: "default", + ServiceRefName: "kubernetes", + Ports: []v1beta1.NetworkPort{{Name: "TCP-443", Protocol: v1beta1.ProtocolTCP, Port: &port443}}, + }, + { + Identifier: "cluster-dns", + Type: v1beta1.CommunicationTypeEgress, + ServiceRefNamespace: "kube-system", + ServiceRefName: "kube-dns", + Ports: []v1beta1.NetworkPort{ + {Name: "UDP-53", Protocol: v1beta1.ProtocolUDP, Port: &port53}, + {Name: "TCP-53", Protocol: v1beta1.ProtocolTCP, Port: &port53}, + }, + }, + { + Identifier: "helm-repos", + Type: v1beta1.CommunicationTypeEgress, + ServiceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"role": "helm-repo"}}, + NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/metadata.name": ns.Name}}, + Ports: []v1beta1.NetworkPort{{Name: "TCP-80", Protocol: v1beta1.ProtocolTCP, Port: &port80}}, + }, + }, + }, + } + _, err := storageClient.ContainerProfiles(ns.Name).Create(context.Background(), cp, metav1.CreateOptions{}) + require.NoError(t, err, "create authored ContainerProfile") + require.Eventually(t, func() bool { + _, e := storageClient.ContainerProfiles(ns.Name).Get(context.Background(), cpName, v1.GetOptions{}) + return e == nil + }, 30*time.Second, time.Second, "authored CP must be in storage before pod deploy") + + require.NoError(t, testutils.ApplyMultiDocDir(ns.Name, path.Join(utils.CurrentDir(), "resources/serviceref-suite")), + "apply flux source-controller + helm repo suite") + + waitDeploy := func(name string) { t.Helper() - k := k8sinterface.NewKubernetesApi() - svc, err := k.KubernetesClient.CoreV1().Services(ns).Get(context.TODO(), name, metav1.GetOptions{}) - require.NoError(t, err, "must read %s/%s ClusterIP", ns, name) - require.NotEmpty(t, svc.Spec.ClusterIP) - return svc.Spec.ClusterIP + require.Eventually(t, func() bool { + d, e := k8sClient.KubernetesClient.AppsV1().Deployments(ns.Name).Get(context.TODO(), name, metav1.GetOptions{}) + return e == nil && d.Status.ReadyReplicas > 0 + }, 3*time.Minute, 5*time.Second, "%s must become ready", name) } - countR0011 := func(alerts []testutils.Alert) int { + waitDeploy("helm-repo") + waitDeploy("decoy-repo") + waitDeploy("source-controller") + + countRule := func(ruleID string) int { + alerts, _ := testutils.GetAlerts(ns.Name) n := 0 for _, a := range alerts { - if a.Labels["rule_id"] == "R0011" { + if a.Labels["rule_id"] == ruleID && a.Labels["container_name"] == containerName { n++ } } return n } - waitAlerts := func(t *testing.T, ns string) []testutils.Alert { - t.Helper() - var alerts []testutils.Alert - require.Eventually(t, func() bool { - var err error - alerts, err = testutils.GetAlerts(ns) - return err == nil - }, 60*time.Second, 5*time.Second, "must be able to fetch alerts") - time.Sleep(10 * time.Second) - alerts, _ = testutils.GetAlerts(ns) - return alerts - } - ns := testutils.NewRandomNamespace() - _ = applyUserDefinedContainerProfile(t, ns.Name, "resources/containerprofile-serviceref-network.yaml") + helmRepoGVR := schema.GroupVersionResource{Group: "source.toolkit.fluxcd.io", Version: "v1", Resource: "helmrepositories"} + repoClient := k8sClient.DynamicClient.Resource(helmRepoGVR).Namespace(ns.Name) + newRepo := func(name, svc string) *unstructured.Unstructured { + return &unstructured.Unstructured{Object: map[string]interface{}{ + "apiVersion": "source.toolkit.fluxcd.io/v1", + "kind": "HelmRepository", + "metadata": map[string]interface{}{"name": name, "namespace": ns.Name}, + // Trailing-dot FQDN: absolute, so the resolver does not walk the + // search list and emit extra lookups. + "spec": map[string]interface{}{ + "interval": "30s", + "url": fmt.Sprintf("http://%s.%s.svc.cluster.local./", svc, ns.Name), + }, + }} + } + repoReady := func(name string) bool { + obj, e := repoClient.Get(context.TODO(), name, metav1.GetOptions{}) + if e != nil { + return false + } + conds, _, _ := unstructured.NestedSlice(obj.Object, "status", "conditions") + for _, c := range conds { + m, ok := c.(map[string]interface{}) + if ok && m["type"] == "Ready" && m["status"] == "True" { + return true + } + } + return false + } - wl, err := testutils.NewTestWorkload(ns.Name, - path.Join(utils.CurrentDir(), "resources/serviceref-client-deployment.yaml")) - require.NoError(t, err) - require.NoError(t, wl.WaitForReady(80)) - // Let node-agent load the bound profile AND sync its Service informer - // (serviceRef resolves against live cluster state) before generating traffic. + // Let node-agent bind the profile and fill its Service/EndpointSlice caches + // before any reconcile traffic is judged. time.Sleep(40 * time.Second) - apiserverIP := getClusterIP(t, "default", "kubernetes") - t.Logf("apiserver ClusterIP=%s (serviceRef-allowed); unlisted egress target=1.1.1.1:80", apiserverIP) - - // Phase 1 — egress to the apiserver, allowlisted by serviceRef - // default/kubernetes. The TCP connect is what R0011 evaluates; -k so curl - // attempts it despite the self-signed cert. - t.Run("serviceref_allowed_no_r0011", func(t *testing.T) { - for i := 0; i < 3; i++ { - so, se, e := wl.ExecIntoPod([]string{"curl", "-skm", "5", fmt.Sprintf("https://%s:443/healthz", apiserverIP)}, "curl") - t.Logf("curl apiserver → err=%v out=%q stderr=%q", e, so, se) + // Phase 1 — the controller reconciles both allowlisted repo Services while + // continuously talking to the apiserver and cluster DNS. Every one of those + // peers is named by the profile, so no egress alert may fire. + t.Run("selector_allowed_no_alert", func(t *testing.T) { + for _, r := range []struct{ name, svc string }{{"primary", "helm-primary"}, {"mirror", "helm-mirror"}} { + _, e := repoClient.Create(context.TODO(), newRepo(r.name, r.svc), metav1.CreateOptions{}) + require.NoError(t, e, "create HelmRepository %s", r.name) } - alerts := waitAlerts(t, wl.Namespace) - assert.Equal(t, 0, countR0011(alerts), - "apiserver egress is allowlisted by serviceRef default/kubernetes — R0011 must NOT fire") + for _, n := range []string{"primary", "mirror"} { + require.Eventually(t, func() bool { return repoReady(n) }, 3*time.Minute, 10*time.Second, + "HelmRepository %s must reconcile (real fetch through an allowlisted Service)", n) + } + // Two further reconcile intervals of steady-state traffic. + time.Sleep(90 * time.Second) + assert.Equal(t, 0, countRule("R0011"), + "apiserver/DNS/helm-repo egress is fully named by serviceRef+serviceSelector — R0011 must NOT fire") }) - // Phase 2 — egress NOT covered by the serviceRef must still fire R0011, - // proving serviceRef is a NARROW allowlist (only default/kubernetes), not a - // blanket that suppresses everything. Raw-IP egress to 1.1.1.1:80 is the - // proven R0011 trigger in this suite (mirrors Test_28c) and is the faithful - // analog of the flux RCA, where R0011 fired for the external github egress - // the named-service allowlist did not cover. - t.Run("uncovered_egress_fires_r0011", func(t *testing.T) { - before := countR0011(waitAlerts(t, wl.Namespace)) - for i := 0; i < 3; i++ { - so, se, e := wl.ExecIntoPod([]string{"curl", "-sm", "5", "http://1.1.1.1:80"}, "curl") - t.Logf("curl 1.1.1.1 (uncovered) → err=%v out=%q stderr=%q", e, so, se) - } + // Phase 2 — the GitOps source of truth is tampered with: primary is + // repointed at decoy-repo, a sibling Service on the same port that the + // role=helm-repo selector does not cover. source-controller fetches it on + // its own next reconcile. This is the lateral move a serviceCIDR entry hides. + t.Run("sibling_service_pivot_fires_r0011", func(t *testing.T) { + before := countRule("R0011") + patch := []byte(fmt.Sprintf(`{"spec":{"url":"http://decoy-repo.%s.svc.cluster.local./"}}`, ns.Name)) + _, e := repoClient.Patch(context.TODO(), "primary", types.MergePatchType, patch, metav1.PatchOptions{}) + require.NoError(t, e, "repoint HelmRepository at the decoy Service") require.Eventually(t, func() bool { - return countR0011(waitAlerts(t, wl.Namespace)) > before - }, 3*time.Minute, 15*time.Second, - "egress uncovered by serviceRef MUST fire R0011 — serviceRef is narrow, not a blanket allow") + return countRule("R0011") > before + }, 4*time.Minute, 15*time.Second, + "egress to an unlisted sibling Service MUST fire R0011 — the selector is narrow, not a blanket") }) } diff --git a/tests/resources/containerprofile-serviceref-network.yaml b/tests/resources/containerprofile-serviceref-network.yaml deleted file mode 100644 index 4db29ba9b2..0000000000 --- a/tests/resources/containerprofile-serviceref-network.yaml +++ /dev/null @@ -1,48 +0,0 @@ -apiVersion: spdx.softwarecomposition.kubescape.io/v1beta1 -kind: ContainerProfile -metadata: - name: serviceref-overlay -spec: - execs: - - path: /bin/sleep - - path: /usr/bin/curl - syscalls: - - socket - - connect - - sendto - - recvfrom - - read - - write - - close - - openat - - mmap - - mprotect - - munmap - - fcntl - - ioctl - - poll - - epoll_create1 - - epoll_ctl - - epoll_wait - - bind - - listen - - accept4 - - getsockopt - - setsockopt - - getsockname - - getpid - - fstat - - rt_sigaction - - rt_sigprocmask - - writev - matchLabels: - app: serviceref-client - egress: - - identifier: apiserver-serviceref - type: internal - serviceRefNamespace: default - serviceRefName: kubernetes - ports: - - name: TCP-443 - protocol: TCP - port: 443 diff --git a/tests/resources/serviceref-client-deployment.yaml b/tests/resources/serviceref-client-deployment.yaml deleted file mode 100644 index 7459155909..0000000000 --- a/tests/resources/serviceref-client-deployment.yaml +++ /dev/null @@ -1,21 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - labels: - app: serviceref-client - name: serviceref-client -spec: - replicas: 1 - selector: - matchLabels: - app: serviceref-client - template: - metadata: - labels: - app: serviceref-client - kubescape.io/user-defined-profile: serviceref-overlay - spec: - containers: - - name: curl - image: docker.io/curlimages/curl@sha256:08e466006f0860e54fc299378de998935333e0e130a15f6f98482e9f8dab3058 - command: ["sleep", "infinity"] diff --git a/tests/resources/serviceref-suite/00-flux-source-crds.yaml b/tests/resources/serviceref-suite/00-flux-source-crds.yaml new file mode 100644 index 0000000000..bfc14c9c8d --- /dev/null +++ b/tests/resources/serviceref-suite/00-flux-source-crds.yaml @@ -0,0 +1,132 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: helmrepositories.source.toolkit.fluxcd.io +spec: + group: source.toolkit.fluxcd.io + names: + kind: HelmRepository + listKind: HelmRepositoryList + plural: helmrepositories + singular: helmrepository + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + x-kubernetes-preserve-unknown-fields: true +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: helmcharts.source.toolkit.fluxcd.io +spec: + group: source.toolkit.fluxcd.io + names: + kind: HelmChart + listKind: HelmChartList + plural: helmcharts + singular: helmchart + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + x-kubernetes-preserve-unknown-fields: true +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: gitrepositories.source.toolkit.fluxcd.io +spec: + group: source.toolkit.fluxcd.io + names: + kind: GitRepository + listKind: GitRepositoryList + plural: gitrepositories + singular: gitrepository + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + x-kubernetes-preserve-unknown-fields: true +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: buckets.source.toolkit.fluxcd.io +spec: + group: source.toolkit.fluxcd.io + names: + kind: Bucket + listKind: BucketList + plural: buckets + singular: bucket + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + x-kubernetes-preserve-unknown-fields: true + - name: v1beta2 + served: true + storage: false + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + x-kubernetes-preserve-unknown-fields: true +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: ocirepositories.source.toolkit.fluxcd.io +spec: + group: source.toolkit.fluxcd.io + names: + kind: OCIRepository + listKind: OCIRepositoryList + plural: ocirepositories + singular: ocirepository + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + x-kubernetes-preserve-unknown-fields: true + - name: v1beta2 + served: true + storage: false + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + x-kubernetes-preserve-unknown-fields: true diff --git a/tests/resources/serviceref-suite/10-helm-repo.yaml b/tests/resources/serviceref-suite/10-helm-repo.yaml new file mode 100644 index 0000000000..830be5de98 --- /dev/null +++ b/tests/resources/serviceref-suite/10-helm-repo.yaml @@ -0,0 +1,108 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: helm-repo-index +data: + index.yaml: | + apiVersion: v1 + entries: {} + generated: "2020-01-01T00:00:00Z" +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: helm-repo +spec: + replicas: 1 + selector: + matchLabels: + app: helm-repo + template: + metadata: + labels: + app: helm-repo + spec: + containers: + - name: nginx + image: nginx:1.27-alpine + ports: + - containerPort: 80 + volumeMounts: + - name: index + mountPath: /usr/share/nginx/html + resources: + limits: + cpu: 200m + memory: 128Mi + volumes: + - name: index + configMap: + name: helm-repo-index +--- +apiVersion: v1 +kind: Service +metadata: + name: helm-primary + labels: + role: helm-repo +spec: + selector: + app: helm-repo + ports: + - port: 80 + targetPort: 80 +--- +apiVersion: v1 +kind: Service +metadata: + name: helm-mirror + labels: + role: helm-repo +spec: + selector: + app: helm-repo + ports: + - port: 80 + targetPort: 80 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: decoy-repo +spec: + replicas: 1 + selector: + matchLabels: + app: decoy-repo + template: + metadata: + labels: + app: decoy-repo + spec: + containers: + - name: nginx + image: nginx:1.27-alpine + ports: + - containerPort: 80 + volumeMounts: + - name: index + mountPath: /usr/share/nginx/html + resources: + limits: + cpu: 200m + memory: 128Mi + volumes: + - name: index + configMap: + name: helm-repo-index +--- +apiVersion: v1 +kind: Service +metadata: + name: decoy-repo +spec: + selector: + app: decoy-repo + ports: + - port: 80 + targetPort: 80 diff --git a/tests/resources/serviceref-suite/20-source-controller.yaml b/tests/resources/serviceref-suite/20-source-controller.yaml new file mode 100644 index 0000000000..613fcaae50 --- /dev/null +++ b/tests/resources/serviceref-suite/20-source-controller.yaml @@ -0,0 +1,117 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: source-controller +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: source-controller +rules: + - apiGroups: ["source.toolkit.fluxcd.io"] + resources: ["*"] + verbs: ["*"] + - apiGroups: [""] + resources: ["configmaps", "secrets", "events", "serviceaccounts"] + verbs: ["*"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["*"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: source-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: source-controller +subjects: + - kind: ServiceAccount + name: source-controller +--- +apiVersion: v1 +kind: Service +metadata: + name: source-controller +spec: + selector: + app: source-controller + ports: + - name: http + port: 80 + targetPort: 9090 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: source-controller +spec: + replicas: 1 + selector: + matchLabels: + app: source-controller + strategy: + type: Recreate + template: + metadata: + labels: + app: source-controller + kubescape.io/user-defined-profile: serviceref-flux-cp + spec: + serviceAccountName: source-controller + terminationGracePeriodSeconds: 10 + containers: + - name: manager + image: ghcr.io/fluxcd/source-controller:v1.3.0 + imagePullPolicy: IfNotPresent + args: + - --log-level=info + - --log-encoding=json + - --storage-path=/data + - --storage-adv-addr=source-controller.$(RUNTIME_NAMESPACE).svc.cluster.local. + - --watch-all-namespaces=false + - --enable-leader-election=false + env: + - name: RUNTIME_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: TUF_ROOT + value: /tmp/.sigstore + ports: + - containerPort: 9090 + name: http + - containerPort: 9440 + name: healthz + readinessProbe: + httpGet: + path: / + port: http + resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 50m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + readOnlyRootFilesystem: true + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + volumeMounts: + - mountPath: /data + name: data + - mountPath: /tmp + name: tmp + securityContext: + fsGroup: 1337 + volumes: + - name: data + emptyDir: {} + - name: tmp + emptyDir: {} diff --git a/tests/testutils/k8s.go b/tests/testutils/k8s.go index b2792d04b9..585af679a6 100644 --- a/tests/testutils/k8s.go +++ b/tests/testutils/k8s.go @@ -10,6 +10,7 @@ import ( "math/rand" "os" "path/filepath" + "sort" "strings" "testing" "time" @@ -28,10 +29,15 @@ import ( "github.com/stretchr/testify/assert" v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/discovery" + "k8s.io/client-go/discovery/cached/memory" "k8s.io/client-go/dynamic" + "k8s.io/client-go/restmapper" "k8s.io/client-go/tools/remotecommand" "k8s.io/kubectl/pkg/scheme" ) @@ -84,6 +90,75 @@ func NewTestWorkload(namespace, resourcePath string) (*TestWorkload, error) { }, nil } +// ApplyMultiDocYAML creates every document in a multi-document YAML file, +// mapping each object's apiVersion/kind to its resource via server discovery so +// custom resources work without a compiled-in table. Namespaced objects land in +// namespace; cluster-scoped ones (CRDs) ignore it. Already-existing objects are +// not an error, so a fixture may be applied by more than one test. +func ApplyMultiDocYAML(namespace, resourcePath string) error { + k8sClient := k8sinterface.NewKubernetesApi() + raw, err := os.ReadFile(resourcePath) + if err != nil { + return err + } + dc, err := discovery.NewDiscoveryClientForConfig(k8sClient.K8SConfig) + if err != nil { + return err + } + mapper := restmapper.NewDeferredDiscoveryRESTMapper(memory.NewMemCacheClient(dc)) + for _, doc := range strings.Split(string(raw), "\n---") { + if strings.TrimSpace(doc) == "" { + continue + } + jsonData, err := yaml.YAMLToJSON([]byte(doc)) + if err != nil { + return fmt.Errorf("%s: %w", resourcePath, err) + } + obj := &unstructured.Unstructured{} + if err := obj.UnmarshalJSON(jsonData); err != nil { + return fmt.Errorf("%s: %w", resourcePath, err) + } + if obj.GetKind() == "" { + continue + } + gvk := obj.GroupVersionKind() + m, err := mapper.RESTMapping(gvk.GroupKind(), gvk.Version) + if err != nil { + return fmt.Errorf("%s %s: %w", resourcePath, gvk.String(), err) + } + var ri dynamic.ResourceInterface = k8sClient.DynamicClient.Resource(m.Resource) + if m.Scope.Name() == meta.RESTScopeNameNamespace { + ri = k8sClient.DynamicClient.Resource(m.Resource).Namespace(namespace) + } + if _, err := ri.Create(context.TODO(), obj, metav1.CreateOptions{}); err != nil && !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("%s %s/%s: %w", resourcePath, obj.GetKind(), obj.GetName(), err) + } + } + return nil +} + +// ApplyMultiDocDir applies every YAML file in dir in lexical order (filenames +// are numbered so CRDs land before the resources that use them). +func ApplyMultiDocDir(namespace, dir string) error { + entries, err := os.ReadDir(dir) + if err != nil { + return err + } + var names []string + for _, e := range entries { + if !e.IsDir() { + names = append(names, e.Name()) + } + } + sort.Strings(names) + for _, n := range names { + if err := ApplyMultiDocYAML(namespace, filepath.Join(dir, n)); err != nil { + return err + } + } + return nil +} + func (w *TestWorkload) ExecIntoPod(command []string, container string) (string, string, error) { pods, err := w.GetPods() if err != nil { From 7d294ca3f1d3422eeea281d015b9767ae10b53f1 Mon Sep 17 00:00:00 2001 From: tanzee Date: Sun, 23 Aug 2026 23:31:53 +0200 Subject: [PATCH 07/19] test(component): scope the internal-egress rule to the serviceRef suite Relaxing the shipped R0011 to fire on private destinations made kube-dns egress alert for every workload that does not name it: Test_21 gained a spurious R0011 and Test_28 lost allowed_fusioncore_no_alert and mitm_coredns_poisoning. Restore the stock expression and express the internal-egress predicate as a test-only rule (R9911) bound by podSelector to this suite's pods, so nothing outside it changes. Verified on kind: Test_50 passes both phases against the stock ruleset, and Test_21 + all six Test_28 subtests are green again. Signed-off-by: tanzee --- .../templates/node-agent/default-rules.yaml | 2 +- tests/component_test.go | 25 +++++++++++++---- tests/resources/serviceref-rulebinding.yaml | 11 ++++++++ tests/resources/serviceref-rules.yaml | 27 +++++++++++++++++++ 4 files changed, 59 insertions(+), 6 deletions(-) create mode 100644 tests/resources/serviceref-rulebinding.yaml create mode 100644 tests/resources/serviceref-rules.yaml diff --git a/tests/chart/templates/node-agent/default-rules.yaml b/tests/chart/templates/node-agent/default-rules.yaml index 913fb1f927..512b4d9ec8 100644 --- a/tests/chart/templates/node-agent/default-rules.yaml +++ b/tests/chart/templates/node-agent/default-rules.yaml @@ -313,7 +313,7 @@ spec: uniqueId: "event.dstAddr + '_' + string(event.dstPort) + '_' + event.proto" ruleExpression: - eventType: "network" - expression: "event.pktType == 'OUTGOING' && !cp.was_address_in_egress(event.containerId, event.dstAddr)" + expression: "event.pktType == 'OUTGOING' && !net.is_private_ip(event.dstAddr) && !cp.was_address_in_egress(event.containerId, event.dstAddr)" profileDependency: 0 profileDataRequired: egressAddresses: all diff --git a/tests/component_test.go b/tests/component_test.go index fbb10b6211..a12c1008d0 100644 --- a/tests/component_test.go +++ b/tests/component_test.go @@ -3931,6 +3931,19 @@ func Test_50_ServiceRefNetworkNeighbor(t *testing.T) { ) port80, port443, port53 := int32(80), int32(443), int32(53) + // The shipped R0011 ignores private destinations, so in-cluster lateral + // movement — the very thing a named-Service allowlist narrows down — cannot + // trip it. R9911 is the same predicate restricted to internal addresses, + // applied for this test only and bound to this suite's pods, so no other + // namespace's expectations move. + rulesPath := path.Join(utils.CurrentDir(), "resources/serviceref-rules.yaml") + bindingPath := path.Join(utils.CurrentDir(), "resources/serviceref-rulebinding.yaml") + require.Equal(t, 0, testutils.RunCommand("kubectl", "apply", "--validate=false", "-f", rulesPath), "apply serviceRef test rules") + defer testutils.RunCommand("kubectl", "delete", "--ignore-not-found", "-f", rulesPath) + require.Equal(t, 0, testutils.RunCommand("kubectl", "apply", "--validate=false", "-f", bindingPath), "apply serviceRef test rule binding") + defer testutils.RunCommand("kubectl", "delete", "--ignore-not-found", "-f", bindingPath) + time.Sleep(20 * time.Second) + ns := testutils.NewRandomNamespace() k8sClient := k8sinterface.NewKubernetesApi() storageClient := spdxv1beta1client.NewForConfigOrDie(k8sClient.K8SConfig) @@ -4050,22 +4063,24 @@ func Test_50_ServiceRefNetworkNeighbor(t *testing.T) { } // Two further reconcile intervals of steady-state traffic. time.Sleep(90 * time.Second) + assert.Equal(t, 0, countRule("R9911"), + "apiserver/DNS/helm-repo egress is fully named by serviceRef+serviceSelector — no internal-egress alert may fire") assert.Equal(t, 0, countRule("R0011"), - "apiserver/DNS/helm-repo egress is fully named by serviceRef+serviceSelector — R0011 must NOT fire") + "no external egress is expected from the controller either") }) // Phase 2 — the GitOps source of truth is tampered with: primary is // repointed at decoy-repo, a sibling Service on the same port that the // role=helm-repo selector does not cover. source-controller fetches it on // its own next reconcile. This is the lateral move a serviceCIDR entry hides. - t.Run("sibling_service_pivot_fires_r0011", func(t *testing.T) { - before := countRule("R0011") + t.Run("sibling_service_pivot_fires_alert", func(t *testing.T) { + before := countRule("R9911") patch := []byte(fmt.Sprintf(`{"spec":{"url":"http://decoy-repo.%s.svc.cluster.local./"}}`, ns.Name)) _, e := repoClient.Patch(context.TODO(), "primary", types.MergePatchType, patch, metav1.PatchOptions{}) require.NoError(t, e, "repoint HelmRepository at the decoy Service") require.Eventually(t, func() bool { - return countRule("R0011") > before + return countRule("R9911") > before }, 4*time.Minute, 15*time.Second, - "egress to an unlisted sibling Service MUST fire R0011 — the selector is narrow, not a blanket") + "egress to an unlisted sibling Service MUST alert — the selector is narrow, not a blanket") }) } diff --git a/tests/resources/serviceref-rulebinding.yaml b/tests/resources/serviceref-rulebinding.yaml new file mode 100644 index 0000000000..24955a4841 --- /dev/null +++ b/tests/resources/serviceref-rulebinding.yaml @@ -0,0 +1,11 @@ +apiVersion: kubescape.io/v1 +kind: RuntimeRuleAlertBinding +metadata: + name: serviceref-test-binding +spec: + namespaceSelector: + podSelector: + matchLabels: + app: source-controller + rules: + - ruleName: "TEST internal egress not in profile" diff --git a/tests/resources/serviceref-rules.yaml b/tests/resources/serviceref-rules.yaml new file mode 100644 index 0000000000..b989ae570b --- /dev/null +++ b/tests/resources/serviceref-rules.yaml @@ -0,0 +1,27 @@ +apiVersion: kubescape.io/v1 +kind: Rules +metadata: + name: serviceref-test-rules + namespace: kubescape + labels: + app: kubescape +spec: + rules: + - name: "TEST internal egress not in profile" + enabled: true + id: "R9911" + description: "Test rule: egress to a cluster-internal address the profile does not allow. The shipped R0011 excludes private destinations, so in-cluster lateral movement — exactly what serviceRef/serviceSelector narrow down — is invisible to it. Scoped to the serviceRef suite via its own binding so no other test's namespace is affected." + expressions: + message: "'Unexpected internal egress to: ' + event.dstAddr + ':' + string(event.dstPort) + ' from: ' + event.containerName" + uniqueId: "'R9911_' + event.dstAddr + '_' + string(event.dstPort)" + ruleExpression: + - eventType: "network" + expression: "event.pktType == 'OUTGOING' && net.is_private_ip(event.dstAddr) && !cp.was_address_in_egress(event.containerId, event.dstAddr)" + profileDependency: 0 + profileDataRequired: + egressAddresses: all + severity: 5 + supportPolicy: false + isTriggerAlert: false + mitreTactic: "TA0008" + mitreTechnique: "T1210" From 8d5a019e985e80d78362ed8e23f52b4a59c5e07e Mon Sep 17 00:00:00 2001 From: tanzee Date: Mon, 24 Aug 2026 06:33:26 +0200 Subject: [PATCH 08/19] test(chart): make network service resolution a value Hardcoding the flag in the ConfigMap made it impossible to measure the feature's cost against itself. Expose it as nodeAgent.config.networkServiceResolution (on in the test chart, so Test_50 still exercises it) so an A/B can toggle resolution without rebuilding the image. Signed-off-by: tanzee --- tests/chart/templates/node-agent/configmap.yaml | 2 +- tests/chart/values.yaml | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/chart/templates/node-agent/configmap.yaml b/tests/chart/templates/node-agent/configmap.yaml index 6108427e90..053acf0808 100644 --- a/tests/chart/templates/node-agent/configmap.yaml +++ b/tests/chart/templates/node-agent/configmap.yaml @@ -14,7 +14,7 @@ data: "prometheusExporterEnabled": {{ eq .Values.nodeAgent.config.prometheusExporter "enable" }}, "runtimeDetectionEnabled": {{ eq .Values.capabilities.runtimeDetection "enable" }}, "networkServiceEnabled": {{ eq .Values.capabilities.networkPolicyService "enable" }}, - "networkServiceResolutionEnabled": true, + "networkServiceResolutionEnabled": {{ .Values.nodeAgent.config.networkServiceResolution | default false }}, "malwareDetectionEnabled": {{ eq .Values.capabilities.malwareDetection "enable" }}, "httpDetectionEnabled": {{ eq .Values.capabilities.httpDetection "enable" }}, "initialDelay": "{{ .Values.nodeAgent.config.learningPeriod }}", diff --git a/tests/chart/values.yaml b/tests/chart/values.yaml index 1aea3a150f..e6c87ef73d 100644 --- a/tests/chart/values.yaml +++ b/tests/chart/values.yaml @@ -58,6 +58,7 @@ nodeAgent: maxLearningPeriod: 2m learningPeriod: 1m updatePeriod: 30s + networkServiceResolution: true maxDelaySeconds: 1 prometheusExporter: enable httpExporterConfig: {} From 369fbe6bc0be2e08ebbe94d3347f5c4e7a5b23c9 Mon Sep 17 00:00:00 2001 From: tanzee Date: Mon, 24 Aug 2026 15:04:34 +0200 Subject: [PATCH 09/19] fix(cel): invalidate cached results when a profile is re-resolved MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The CEL result cache keys on SpecHash + SyncChecksum. Re-projecting a serviceRef/serviceSelector/entity profile against a moved cluster view changes neither: SpecHash tracks the rule projection spec, and SyncChecksum comes from a learned CP annotation an authored profile does not carry at all. So a result computed before the Service/EndpointSlice informers filled — 'this ClusterIP is not in egress' — was served from the LRU indefinitely, and the re-projection the lister generation correctly triggered had no observable effect. Egress to an allowlisted Service kept alerting. Carry the resolution generation on the projected profile and include it in the key, so the cache moves whenever the resolved addresses can have moved. Signed-off-by: tanzee --- .../containerprofilecache.go | 1 + .../containerprofilecache/reconciler.go | 1 + .../containerprofilecache/resolvedgen_test.go | 48 +++++++++++++++++++ pkg/objectcache/projection_types.go | 9 +++- .../cel/libraries/cache/function_cache.go | 7 ++- 5 files changed, 63 insertions(+), 3 deletions(-) create mode 100644 pkg/objectcache/containerprofilecache/resolvedgen_test.go diff --git a/pkg/objectcache/containerprofilecache/containerprofilecache.go b/pkg/objectcache/containerprofilecache/containerprofilecache.go index 410b39ec47..0c2b1574a3 100644 --- a/pkg/objectcache/containerprofilecache/containerprofilecache.go +++ b/pkg/objectcache/containerprofilecache/containerprofilecache.go @@ -617,6 +617,7 @@ func (c *ContainerProfileCacheImpl) buildEntry( entry.UsesServiceResolution = networkpeer.HasServiceNeighbors(userMerged) entry.ListerGen = c.listerGen() projected := Apply(spec, networkpeer.WithResolvedServiceNeighbors(userMerged, c.serviceLister), tree) + projected.ResolvedGen = entry.ListerGen entry.Projected = projected entry.SpecHash = projected.SpecHash diff --git a/pkg/objectcache/containerprofilecache/reconciler.go b/pkg/objectcache/containerprofilecache/reconciler.go index 85c4943aaf..b0fe54d8fe 100644 --- a/pkg/objectcache/containerprofilecache/reconciler.go +++ b/pkg/objectcache/containerprofilecache/reconciler.go @@ -497,6 +497,7 @@ func (c *ContainerProfileCacheImpl) rebuildEntryFromSources( spec := c.snapshotSpec() applyStart := time.Now() projectedCP := Apply(spec, networkpeer.WithResolvedServiceNeighbors(projected, c.serviceLister), tree) + projectedCP.ResolvedGen = c.listerGen() if c.cfg.ProfileProjection.DetailedMetricsEnabled { c.metricsManager.ObserveProjectionApplyDuration(time.Since(applyStart)) c.observeMemoryMetrics(projected, projectedCP) diff --git a/pkg/objectcache/containerprofilecache/resolvedgen_test.go b/pkg/objectcache/containerprofilecache/resolvedgen_test.go new file mode 100644 index 0000000000..7abdc844d1 --- /dev/null +++ b/pkg/objectcache/containerprofilecache/resolvedgen_test.go @@ -0,0 +1,48 @@ +package containerprofilecache + +import ( + "strconv" + "testing" + + "github.com/kubescape/node-agent/pkg/networkpeer" + "github.com/kubescape/node-agent/pkg/objectcache" +) + +// genLister is a cluster view whose generation the test drives directly. +type genLister struct{ gen int64 } + +func (g *genLister) ServiceByName(string, string) (*networkpeer.ServiceInfo, bool) { return nil, false } +func (g *genLister) ServicesByLabels(map[string]string, map[string]string) []*networkpeer.ServiceInfo { + return nil +} +func (g *genLister) HostIPs() []string { return nil } +func (g *genLister) Generation() int64 { return g.gen } + +// TestProjectedResolvedGenFeedsCacheKey: the CEL result cache keys on the +// projected profile's SpecHash+SyncChecksum+ResolvedGen. Re-resolving against a +// moved cluster view changes neither of the first two — an authored profile +// carries no SyncChecksum at all — so without ResolvedGen a result computed +// before the informers filled (e.g. "this address is not in egress") would be +// served from cache forever, and the profile's own re-projection would never +// take effect. +func TestProjectedResolvedGenFeedsCacheKey(t *testing.T) { + l := &genLister{gen: 7} + c := &ContainerProfileCacheImpl{} + c.SetServiceLister(l) + + if got := c.listerGen(); got != 7 { + t.Fatalf("listerGen: got %d want 7", got) + } + + key := func(p *objectcache.ProjectedContainerProfile) string { + return p.SpecHash + "|" + p.SyncChecksum + "|" + strconv.FormatInt(p.ResolvedGen, 10) + } + before := &objectcache.ProjectedContainerProfile{SpecHash: "spec", ResolvedGen: c.listerGen()} + + l.gen = 8 + after := &objectcache.ProjectedContainerProfile{SpecHash: "spec", ResolvedGen: c.listerGen()} + + if key(before) == key(after) { + t.Errorf("cache key must change when the profile is re-resolved against a moved cluster view") + } +} diff --git a/pkg/objectcache/projection_types.go b/pkg/objectcache/projection_types.go index 8d452c7a36..d76fe2b325 100644 --- a/pkg/objectcache/projection_types.go +++ b/pkg/objectcache/projection_types.go @@ -67,7 +67,14 @@ type ProjectedContainerProfile struct { // constraint" (back-compat for pre-projection profiles). ExecsByPath map[string][][]string - SpecHash string + SpecHash string + // ResolvedGen is the cluster-view generation the profile's serviceRef/ + // serviceSelector/entity neighbors were resolved against. It participates in + // the CEL result-cache key: re-projecting against a moved cluster view + // changes the projected addresses without touching SpecHash or SyncChecksum, + // so a result cached before the informers filled would otherwise be served + // forever. Zero for profiles that resolve nothing. + ResolvedGen int64 SyncChecksum string PolicyByRuleId map[string]v1beta1.RulePolicy CallStackTree *callstackcache.CallStackSearchTree diff --git a/pkg/rulemanager/cel/libraries/cache/function_cache.go b/pkg/rulemanager/cel/libraries/cache/function_cache.go index ba07eafcd3..22af6beed6 100644 --- a/pkg/rulemanager/cel/libraries/cache/function_cache.go +++ b/pkg/rulemanager/cel/libraries/cache/function_cache.go @@ -2,6 +2,7 @@ package cache import ( "fmt" + "strconv" "strings" "time" @@ -101,8 +102,10 @@ func HashForContainerProfile(oc objectcache.ObjectCache) func([]ref.Val) string } // Include SyncChecksum so the key changes when profile content is updated // under the same projection spec, preventing stale cached results after - // the profile learns new paths/execs/etc. - return pcp.SpecHash + "|" + pcp.SyncChecksum + // the profile learns new paths/execs/etc. ResolvedGen covers the same + // hazard for serviceRef/entity neighbors, whose projected addresses move + // with the cluster view while both other components stay put. + return pcp.SpecHash + "|" + pcp.SyncChecksum + "|" + strconv.FormatInt(pcp.ResolvedGen, 10) } } From e240448bd5d79d5e91abd31656b0912dd548ec65 Mon Sep 17 00:00:00 2001 From: tanzee Date: Mon, 24 Aug 2026 18:58:48 +0200 Subject: [PATCH 10/19] feat(rules): R0012 unexpected internal egress MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit R0011 keeps its external-only scope (!is_private_ip); internal traffic gets its own rule instead of widening R0011 — rewriting R0011's scope broke Test_21/28 (kube-dns FPs) when tried in the fork CT. R0012 alerts on OUTGOING to private addresses (loopback excluded — is_private_ip counts 127.0.0.1/::1 as private) not allowlisted by the profile's egress addresses, which includes serviceRef/serviceSelector-resolved entries. Uses the port-aware matcher; behaves address-only until the port projection lands, then becomes port-sensitive with no rules change. A selector clause (was_selector_in_egress) is added one-line when the peer-selector fields merge. Same defaults as R0011; uniqueId keyed on addr_port_proto; bound in the default binding (new rule names are inert until bound). Signed-off-by: tanzee --- .../node-agent/default-rule-binding.yaml | 1 + .../templates/node-agent/default-rules.yaml | 25 +++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/tests/chart/templates/node-agent/default-rule-binding.yaml b/tests/chart/templates/node-agent/default-rule-binding.yaml index 3d8f7847b4..cd7b178e86 100644 --- a/tests/chart/templates/node-agent/default-rule-binding.yaml +++ b/tests/chart/templates/node-agent/default-rule-binding.yaml @@ -39,5 +39,6 @@ spec: - ruleName: "Exec to pod" - ruleName: "Port forward to pod" - ruleName: "Unexpected Egress Network Traffic" + - ruleName: "Unexpected Internal Egress Network Traffic" - ruleName: "Unexpected Ptrace Syscall Usage" - ruleName: "Unexpected io_uring Operation Detected" diff --git a/tests/chart/templates/node-agent/default-rules.yaml b/tests/chart/templates/node-agent/default-rules.yaml index 512b4d9ec8..1298cc2654 100644 --- a/tests/chart/templates/node-agent/default-rules.yaml +++ b/tests/chart/templates/node-agent/default-rules.yaml @@ -329,6 +329,31 @@ spec: - "network" - "anomaly" - "networkprofile" + - name: "Unexpected Internal Egress Network Traffic" + enabled: true + id: "R0012" + description: "Detecting egress to cluster-internal addresses that is not allowlisted by the application profile. Complements R0011 (external egress): serviceCIDR-wide entries blind detection to lateral movement, so internal peers should be allowlisted narrowly (addresses, or resolved serviceRef/serviceSelector entries) and everything else alerts." + expressions: + message: "'Unexpected internal egress network communication to: ' + event.dstAddr + ':' + string(event.dstPort) + ' using ' + event.proto + ' from: ' + event.containerName" + uniqueId: "event.dstAddr + '_' + string(event.dstPort) + '_' + event.proto" + ruleExpression: + - eventType: "network" + expression: "event.pktType == 'OUTGOING' && net.is_private_ip(event.dstAddr) && !event.dstAddr.startsWith('127.') && event.dstAddr != '::1' && !cp.was_address_port_protocol_in_egress(event.containerId, event.dstAddr, event.dstPort, event.proto)" + profileDependency: 0 + profileDataRequired: + egressAddresses: all + severity: 5 # Medium + supportPolicy: false + isTriggerAlert: false + mitreTactic: "TA0008" + mitreTechnique: "T1210" + tags: + - "context:kubernetes" + - "context:container" + - "whitelisted" + - "network" + - "anomaly" + - "networkprofile" - name: "Unexpected process arguments" enabled: true id: "R0040" From 9537cf0e540170619886000ba1cfecbb073454f7 Mon Sep 17 00:00:00 2001 From: tanzee Date: Mon, 24 Aug 2026 19:01:41 +0200 Subject: [PATCH 11/19] feat(rules): R0011/R0012 symmetric egress/ingress, no IP-class gate MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per design review: R0011 (egress) and R0012 (ingress, new) are symmetric twins. Neither uses is_private_ip — internal and external peers are treated alike, so lateral movement to unlisted internal peers alerts; only loopback is excluded. Allowlisting internal traffic is the profile's job (addresses, resolved serviceRef/serviceSelector entries), not the rule's. Both use the port-aware matcher (address-only until port projection lands). On HOST (incoming) events the gadget's dstAddr/dstPort carry the remote peer and local port. R0011's scope widens to internal egress: component tests whose profiles do not list kube-dns et al. will alert until their profiles do — that pressure is the feature. Signed-off-by: tanzee --- .../templates/node-agent/default-rule-binding.yaml | 2 +- tests/chart/templates/node-agent/default-rules.yaml | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/chart/templates/node-agent/default-rule-binding.yaml b/tests/chart/templates/node-agent/default-rule-binding.yaml index cd7b178e86..755bd39055 100644 --- a/tests/chart/templates/node-agent/default-rule-binding.yaml +++ b/tests/chart/templates/node-agent/default-rule-binding.yaml @@ -39,6 +39,6 @@ spec: - ruleName: "Exec to pod" - ruleName: "Port forward to pod" - ruleName: "Unexpected Egress Network Traffic" - - ruleName: "Unexpected Internal Egress Network Traffic" + - ruleName: "Unexpected Ingress Network Traffic" - ruleName: "Unexpected Ptrace Syscall Usage" - ruleName: "Unexpected io_uring Operation Detected" diff --git a/tests/chart/templates/node-agent/default-rules.yaml b/tests/chart/templates/node-agent/default-rules.yaml index 1298cc2654..98bda9333f 100644 --- a/tests/chart/templates/node-agent/default-rules.yaml +++ b/tests/chart/templates/node-agent/default-rules.yaml @@ -313,7 +313,7 @@ spec: uniqueId: "event.dstAddr + '_' + string(event.dstPort) + '_' + event.proto" ruleExpression: - eventType: "network" - expression: "event.pktType == 'OUTGOING' && !net.is_private_ip(event.dstAddr) && !cp.was_address_in_egress(event.containerId, event.dstAddr)" + expression: "event.pktType == 'OUTGOING' && !event.dstAddr.startsWith('127.') && event.dstAddr != '::1' && !cp.was_address_port_protocol_in_egress(event.containerId, event.dstAddr, event.dstPort, event.proto)" profileDependency: 0 profileDataRequired: egressAddresses: all @@ -329,19 +329,19 @@ spec: - "network" - "anomaly" - "networkprofile" - - name: "Unexpected Internal Egress Network Traffic" + - name: "Unexpected Ingress Network Traffic" enabled: true id: "R0012" - description: "Detecting egress to cluster-internal addresses that is not allowlisted by the application profile. Complements R0011 (external egress): serviceCIDR-wide entries blind detection to lateral movement, so internal peers should be allowlisted narrowly (addresses, or resolved serviceRef/serviceSelector entries) and everything else alerts." + description: "Detecting unexpected ingress network traffic that is not allowlisted by application profile. Symmetric twin of R0011: internal and external peers alike, only loopback excluded; internal peers are allowlisted narrowly via addresses or resolved serviceRef/serviceSelector entries rather than a serviceCIDR that blinds detection to lateral movement." expressions: - message: "'Unexpected internal egress network communication to: ' + event.dstAddr + ':' + string(event.dstPort) + ' using ' + event.proto + ' from: ' + event.containerName" + message: "'Unexpected ingress network communication from: ' + event.dstAddr + ':' + string(event.dstPort) + ' using ' + event.proto + ' to: ' + event.containerName" uniqueId: "event.dstAddr + '_' + string(event.dstPort) + '_' + event.proto" ruleExpression: - eventType: "network" - expression: "event.pktType == 'OUTGOING' && net.is_private_ip(event.dstAddr) && !event.dstAddr.startsWith('127.') && event.dstAddr != '::1' && !cp.was_address_port_protocol_in_egress(event.containerId, event.dstAddr, event.dstPort, event.proto)" + expression: "event.pktType == 'HOST' && !event.dstAddr.startsWith('127.') && event.dstAddr != '::1' && !cp.was_address_port_protocol_in_ingress(event.containerId, event.dstAddr, event.dstPort, event.proto)" profileDependency: 0 profileDataRequired: - egressAddresses: all + ingressAddresses: all severity: 5 # Medium supportPolicy: false isTriggerAlert: false From 9a5dca610c294509326629465820f1296ab260a8 Mon Sep 17 00:00:00 2001 From: tanzee Date: Mon, 24 Aug 2026 19:18:22 +0200 Subject: [PATCH 12/19] dedup selector engine after portalerts merge portalerts carried its own copy of the celnetworkselector peer-selector functions; the merge kept both and the package no longer compiled. One copy remains. Signed-off-by: tanzee --- .../containerprofilenetwork/network.go | 102 ------------------ 1 file changed, 102 deletions(-) diff --git a/pkg/rulemanager/cel/libraries/containerprofilenetwork/network.go b/pkg/rulemanager/cel/libraries/containerprofilenetwork/network.go index 0b906006d2..723827e982 100644 --- a/pkg/rulemanager/cel/libraries/containerprofilenetwork/network.go +++ b/pkg/rulemanager/cel/libraries/containerprofilenetwork/network.go @@ -346,105 +346,3 @@ func refValToStringMap(v ref.Val) map[string]string { m, _ := native.(map[string]string) return m } - -// namespaceSelectorMatches matches a namespaceSelector against the peer's -// namespace via the implicit kubernetes.io/metadata.name label every namespace -// carries (the form these profiles use). A nil selector matches only the -// profiled workload's own namespace: the learned generator omits the selector -// exactly for same-namespace peers, and NetworkPolicyPeer gives an absent -// namespaceSelector the same meaning. Selectors keyed on other namespace -// labels are not resolved here. -func namespaceSelectorMatches(sel *metav1.LabelSelector, ns, profileNs string) bool { - if sel == nil { - return ns == profileNs - } - s, err := metav1.LabelSelectorAsSelector(sel) - if err != nil { - return false - } - return s.Matches(labels.Set{"kubernetes.io/metadata.name": ns}) -} - -// wasSelectorInPeers reports whether the peer identified by (podLabels, ns) -// matches any peer entry's podSelector AND its namespaceSelector. -func wasSelectorInPeers(peers []objectcache.PeerSelector, podLabels labels.Set, ns, profileNs string) bool { - for i := range peers { - peer := &peers[i] - if peer.PodSelector == nil { - continue - } - ps, err := metav1.LabelSelectorAsSelector(peer.PodSelector) - if err != nil { - continue - } - if ps.Matches(podLabels) && namespaceSelectorMatches(peer.NamespaceSelector, ns, profileNs) { - return true - } - } - return false -} - -func (l *containerProfileNetworkLibrary) wasSelectorInIngress(containerID, namespace, podLabels ref.Val) ref.Val { - return l.wasSelectorIn(containerID, namespace, podLabels, true) -} - -func (l *containerProfileNetworkLibrary) wasSelectorInEgress(containerID, namespace, podLabels ref.Val) ref.Val { - return l.wasSelectorIn(containerID, namespace, podLabels, false) -} - -// wasSelectorIn reports whether the runtime peer — identified by the namespace -// and pod labels that Inspektor Gadget's kubeipresolver stamps onto the network -// event — matches any of the profile's ingress-or-egress peer selectors. -// -// Matching on the peer's identity (namespace + labels) rather than its IP is the -// whole point: it is stable across pod IP churn AND works across nodes, because -// kubeipresolver resolves the peer against a cluster-wide pod inventory before -// the event ever reaches CEL. There is deliberately no IP→pod lookup here — that -// would reintroduce a dependency on node-agent's node-local pod cache, which is -// exactly what breaks cross-node peers. -func (l *containerProfileNetworkLibrary) wasSelectorIn(containerID, namespace, podLabels ref.Val, ingress bool) ref.Val { - if l.objectCache == nil { - return types.NewErr("objectCache is nil") - } - containerIDStr, ok := containerID.Value().(string) - if !ok { - return types.MaybeNoSuchOverloadErr(containerID) - } - nsStr, ok := namespace.Value().(string) - if !ok { - return types.MaybeNoSuchOverloadErr(namespace) - } - if nsStr == "" { - // The peer did not resolve to a pod (external IP, or the resolver had no - // inventory entry): it cannot satisfy any selector. A resolved pod with - // zero labels is NOT this case - an empty podSelector may still match it. - return types.Bool(false) - } - peerLabels := refValToStringMap(podLabels) - cp, _, err := profilehelper.GetProjectedContainerProfile(l.objectCache, containerIDStr) - if err != nil { - return cache.NewProfileNotAvailableErr("%v", err) - } - peers := cp.EgressPeers - if ingress { - peers = cp.IngressPeers - } - if len(peers) == 0 { - return types.Bool(false) - } - return types.Bool(wasSelectorInPeers(peers, labels.Set(peerLabels), nsStr, cp.Namespace)) -} - -// refValToStringMap converts a CEL map argument to a Go map[string]string. A nil -// or non-map value yields nil (treated as "peer has no labels"). -func refValToStringMap(v ref.Val) map[string]string { - if v == nil { - return nil - } - native, err := v.ConvertToNative(reflect.TypeOf(map[string]string(nil))) - if err != nil { - return nil - } - m, _ := native.(map[string]string) - return m -} From 60962b8aac593313d4294a2d0e7327d115dd39af Mon Sep 17 00:00:00 2001 From: tanzee Date: Mon, 24 Aug 2026 19:21:04 +0200 Subject: [PATCH 13/19] pin storage to k8sstormcenter/storage@3844202a (dnsNames + deflate fixes) Signed-off-by: tanzee --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 4013bc3b28..0a7d31cceb 100644 --- a/go.mod +++ b/go.mod @@ -32,10 +32,10 @@ require ( github.com/iceber/iouring-go v0.0.0-20230403020409-002cfd2e2a90 github.com/inspektor-gadget/inspektor-gadget v0.45.1-0.20251020222545-c91c23581ebf github.com/joncrlsn/dque v0.0.0-20241024143830-7723fd131a64 - github.com/kubescape/backend v0.0.39 + github.com/kubescape/backend v0.0.31 github.com/kubescape/go-logger v0.0.32 github.com/kubescape/k8s-interface v0.0.214 - github.com/kubescape/storage v0.0.303 + github.com/kubescape/storage v0.0.0-00010101000000-000000000000 github.com/kubescape/workerpool v0.0.0-20250526074519-0e4a4e7f44cf github.com/moby/sys/mountinfo v0.7.2 github.com/oleiade/lane/v2 v2.0.0 @@ -480,4 +480,4 @@ replace github.com/anchore/stereoscope => github.com/anchore/stereoscope v0.1.9 replace github.com/opencontainers/runtime-spec => github.com/opencontainers/runtime-spec v1.2.1 -replace github.com/kubescape/storage => github.com/k8sstormcenter/storage v0.0.240-0.20260823123818-6f3a2ee6385d +replace github.com/kubescape/storage => github.com/k8sstormcenter/storage v0.0.240-0.20260824140105-3844202af06c diff --git a/go.sum b/go.sum index 0c78ed4da6..f33e276f9d 100644 --- a/go.sum +++ b/go.sum @@ -859,8 +859,8 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= -github.com/k8sstormcenter/storage v0.0.240-0.20260823123818-6f3a2ee6385d h1:4d7cpcoXpp8ZJXV2W/ubX5WW78gn9diy9qvLFbROvV0= -github.com/k8sstormcenter/storage v0.0.240-0.20260823123818-6f3a2ee6385d/go.mod h1:d/1hqWPda2clsjx2wmQgysnB5dThIo3rDKP7RWx+v+M= +github.com/k8sstormcenter/storage v0.0.240-0.20260824140105-3844202af06c h1:UWyIu2P3eDT4VUwxDkphPFKYGo2BfR7GkNGw7Nh/LiA= +github.com/k8sstormcenter/storage v0.0.240-0.20260824140105-3844202af06c/go.mod h1:d/1hqWPda2clsjx2wmQgysnB5dThIo3rDKP7RWx+v+M= github.com/kastenhq/goversion v0.0.0-20230811215019-93b2f8823953 h1:WdAeg/imY2JFPc/9CST4bZ80nNJbiBFCAdSZCSgrS5Y= github.com/kastenhq/goversion v0.0.0-20230811215019-93b2f8823953/go.mod h1:6o+UrvuZWc4UTyBhQf0LGjW9Ld7qJxLz/OqvSOWWlEc= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= @@ -889,8 +889,8 @@ github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/kubescape/backend v0.0.39 h1:B1QRfKCSFlzuE+jWOnk/l7EpH71/Q3n14KKq0QSnZwg= -github.com/kubescape/backend v0.0.39/go.mod h1:cMEGP8cXUZgY89YU4GRBGIla9HZW7grZsUtlCwvZgAE= +github.com/kubescape/backend v0.0.31 h1:pLMic67Vuiksdfh1t7ATq9M9wkrjXtvQfPDopzuGWkA= +github.com/kubescape/backend v0.0.31/go.mod h1:FpazfN+c3Ucuvv4jZYCnk99moSBRNMVIxl5aWCZAEBo= github.com/kubescape/go-logger v0.0.32 h1:4mI+XJOV8VFCMewrEE9VIFEIOhzXokYT3nFpNfXf4fM= github.com/kubescape/go-logger v0.0.32/go.mod h1:Alj7JBQ8/WCxbXe8Ura6ZheSRK45E0p21M3xeqedX90= github.com/kubescape/k8s-interface v0.0.214 h1:j7KP0/5VvYOoQdBGV2+gRM3qnR8PWLAGF8RM/k/DmJ0= @@ -2034,8 +2034,8 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= -gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= +gotest.tools/v3 v3.5.0 h1:Ljk6PdHdOhAb5aDMWXjDLMMhph+BpztA4v1QdqEW2eY= +gotest.tools/v3 v3.5.0/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= From 70ebf6d0da38de29a46f08fcea4881ad545801d6 Mon Sep 17 00:00:00 2001 From: tanzee Date: Mon, 24 Aug 2026 19:24:58 +0200 Subject: [PATCH 14/19] test(component): Test_50 asserts shipped R0011; Test_51 ingress R0012 twin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The scoped R9911 rule and its binding are gone — the widened R0011 covers internal egress, so the decoy pivot asserts the shipped rule. Test_51 mirrors it for ingress: nginx serves a serviceRef-listed client (flux source-controller, resolution covers its ClusterIP and pod endpoint IPs) with zero R0012, then an unlisted k6 client joins and R0012 must fire. Both use only real controller/loadgen traffic. Signed-off-by: tanzee --- .github/workflows/component-tests.yaml | 3 +- tests/component_test.go | 118 ++++++++++++++++---- tests/resources/serviceref-k6.yaml | 43 +++++++ tests/resources/serviceref-rulebinding.yaml | 11 -- tests/resources/serviceref-rules.yaml | 27 ----- 5 files changed, 144 insertions(+), 58 deletions(-) create mode 100644 tests/resources/serviceref-k6.yaml delete mode 100644 tests/resources/serviceref-rulebinding.yaml delete mode 100644 tests/resources/serviceref-rules.yaml diff --git a/.github/workflows/component-tests.yaml b/.github/workflows/component-tests.yaml index f89cfb5639..d1c6dfceec 100644 --- a/.github/workflows/component-tests.yaml +++ b/.github/workflows/component-tests.yaml @@ -108,7 +108,8 @@ jobs: Test_43_RelativeOpenPathResolution, Test_48_MultiSubtypeGroupedProfileDocument, Test_49_EphemeralContainerFullTreatment, - Test_50_ServiceRefNetworkNeighbor + Test_50_ServiceRefNetworkNeighbor, + Test_51_ServiceRefIngressR0012 ] steps: - name: Checkout code diff --git a/tests/component_test.go b/tests/component_test.go index 2ee318bc00..35ec4f27bb 100644 --- a/tests/component_test.go +++ b/tests/component_test.go @@ -3992,19 +3992,6 @@ func Test_50_ServiceRefNetworkNeighbor(t *testing.T) { ) port80, port443, port53 := int32(80), int32(443), int32(53) - // The shipped R0011 ignores private destinations, so in-cluster lateral - // movement — the very thing a named-Service allowlist narrows down — cannot - // trip it. R9911 is the same predicate restricted to internal addresses, - // applied for this test only and bound to this suite's pods, so no other - // namespace's expectations move. - rulesPath := path.Join(utils.CurrentDir(), "resources/serviceref-rules.yaml") - bindingPath := path.Join(utils.CurrentDir(), "resources/serviceref-rulebinding.yaml") - require.Equal(t, 0, testutils.RunCommand("kubectl", "apply", "--validate=false", "-f", rulesPath), "apply serviceRef test rules") - defer testutils.RunCommand("kubectl", "delete", "--ignore-not-found", "-f", rulesPath) - require.Equal(t, 0, testutils.RunCommand("kubectl", "apply", "--validate=false", "-f", bindingPath), "apply serviceRef test rule binding") - defer testutils.RunCommand("kubectl", "delete", "--ignore-not-found", "-f", bindingPath) - time.Sleep(20 * time.Second) - ns := testutils.NewRandomNamespace() k8sClient := k8sinterface.NewKubernetesApi() storageClient := spdxv1beta1client.NewForConfigOrDie(k8sClient.K8SConfig) @@ -4124,10 +4111,8 @@ func Test_50_ServiceRefNetworkNeighbor(t *testing.T) { } // Two further reconcile intervals of steady-state traffic. time.Sleep(90 * time.Second) - assert.Equal(t, 0, countRule("R9911"), - "apiserver/DNS/helm-repo egress is fully named by serviceRef+serviceSelector — no internal-egress alert may fire") assert.Equal(t, 0, countRule("R0011"), - "no external egress is expected from the controller either") + "apiserver/DNS/helm-repo egress is fully named by serviceRef+serviceSelector — R0011 may not fire") }) // Phase 2 — the GitOps source of truth is tampered with: primary is @@ -4135,13 +4120,108 @@ func Test_50_ServiceRefNetworkNeighbor(t *testing.T) { // role=helm-repo selector does not cover. source-controller fetches it on // its own next reconcile. This is the lateral move a serviceCIDR entry hides. t.Run("sibling_service_pivot_fires_alert", func(t *testing.T) { - before := countRule("R9911") + before := countRule("R0011") patch := []byte(fmt.Sprintf(`{"spec":{"url":"http://decoy-repo.%s.svc.cluster.local./"}}`, ns.Name)) _, e := repoClient.Patch(context.TODO(), "primary", types.MergePatchType, patch, metav1.PatchOptions{}) require.NoError(t, e, "repoint HelmRepository at the decoy Service") require.Eventually(t, func() bool { - return countRule("R9911") > before + return countRule("R0011") > before + }, 4*time.Minute, 15*time.Second, + "egress to an unlisted sibling Service MUST fire R0011 — the selector is narrow, not a blanket") + }) +} + +// Test_51_ServiceRefIngressR0012 is the ingress twin of Test_50: nginx serves +// two real clients; the profile names one of them (ingress serviceRef, whose +// resolution covers the client Service's ClusterIP and pod endpoint IPs), and +// R0012 must stay silent for it while firing for the unlisted one (k6). +func Test_51_ServiceRefIngressR0012(t *testing.T) { + start := time.Now() + defer tearDownTest(t, start) + + const cpName = "serviceref-ingress-cp" + port80 := int32(80) + + ns := testutils.NewRandomNamespace() + k8sClient := k8sinterface.NewKubernetesApi() + storageClient := spdxv1beta1client.NewForConfigOrDie(k8sClient.K8SConfig) + + cp := &v1beta1.ContainerProfile{ + ObjectMeta: metav1.ObjectMeta{Name: cpName, Namespace: ns.Name}, + Spec: v1beta1.ContainerProfileSpec{ + LabelSelector: metav1.LabelSelector{MatchLabels: map[string]string{"app": "helm-repo"}}, + Ingress: []v1beta1.NetworkNeighbor{ + { + Identifier: "gitops-clients", + Type: v1beta1.CommunicationTypeIngress, + ServiceRefNamespace: ns.Name, + ServiceRefName: "source-controller", + Ports: []v1beta1.NetworkPort{{Name: "TCP-80", Protocol: v1beta1.ProtocolTCP, Port: &port80}}, + }, + }, + }, + } + _, err := storageClient.ContainerProfiles(ns.Name).Create(context.Background(), cp, metav1.CreateOptions{}) + require.NoError(t, err, "create authored ContainerProfile") + + require.NoError(t, testutils.ApplyMultiDocDir(ns.Name, path.Join(utils.CurrentDir(), "resources/serviceref-suite")), + "apply flux source-controller + helm repo suite") + + patch := []byte(fmt.Sprintf(`{"spec":{"template":{"metadata":{"labels":{"kubescape.io/user-defined-profile":%q}}}}}`, cpName)) + _, err = k8sClient.KubernetesClient.AppsV1().Deployments(ns.Name).Patch(context.TODO(), "helm-repo", types.StrategicMergePatchType, patch, metav1.PatchOptions{}) + require.NoError(t, err, "bind profile to helm-repo pods") + + waitDeploy := func(name string) { + t.Helper() + require.Eventually(t, func() bool { + d, e := k8sClient.KubernetesClient.AppsV1().Deployments(ns.Name).Get(context.TODO(), name, metav1.GetOptions{}) + return e == nil && d.Status.ReadyReplicas > 0 && d.Status.UpdatedReplicas == d.Status.ReadyReplicas + }, 3*time.Minute, 5*time.Second, "%s must become ready", name) + } + waitDeploy("helm-repo") + waitDeploy("source-controller") + + countR0012 := func() int { + alerts, _ := testutils.GetAlerts(ns.Name) + n := 0 + for _, a := range alerts { + if a.Labels["rule_id"] == "R0012" && a.Labels["container_name"] == "nginx" { + n++ + } + } + return n + } + + helmRepoGVR := schema.GroupVersionResource{Group: "source.toolkit.fluxcd.io", Version: "v1", Resource: "helmrepositories"} + repoClient := k8sClient.DynamicClient.Resource(helmRepoGVR).Namespace(ns.Name) + repo := &unstructured.Unstructured{Object: map[string]interface{}{ + "apiVersion": "source.toolkit.fluxcd.io/v1", + "kind": "HelmRepository", + "metadata": map[string]interface{}{"name": "primary", "namespace": ns.Name}, + "spec": map[string]interface{}{ + "interval": "30s", + "url": fmt.Sprintf("http://helm-primary.%s.svc.cluster.local./", ns.Name), + }, + }} + _, err = repoClient.Create(context.TODO(), repo, metav1.CreateOptions{}) + require.NoError(t, err, "create HelmRepository") + + time.Sleep(40 * time.Second) + + t.Run("listed_client_no_r0012", func(t *testing.T) { + // Two reconcile intervals of real source-controller fetches into nginx. + time.Sleep(90 * time.Second) + assert.Equal(t, 0, countR0012(), + "ingress from the serviceRef-listed client must not fire R0012") + }) + + t.Run("unlisted_client_fires_r0012", func(t *testing.T) { + before := countR0012() + require.NoError(t, testutils.ApplyMultiDocYAML(ns.Name, path.Join(utils.CurrentDir(), "resources/serviceref-k6.yaml")), + "deploy unlisted k6 client") + require.Eventually(t, func() bool { + return countR0012() > before }, 4*time.Minute, 15*time.Second, - "egress to an unlisted sibling Service MUST alert — the selector is narrow, not a blanket") + "ingress from a client no serviceRef names MUST fire R0012") }) } diff --git a/tests/resources/serviceref-k6.yaml b/tests/resources/serviceref-k6.yaml new file mode 100644 index 0000000000..7d703b33b1 --- /dev/null +++ b/tests/resources/serviceref-k6.yaml @@ -0,0 +1,43 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: k6-script +data: + load.js: | + import http from 'k6/http'; + import { sleep } from 'k6'; + export const options = { vus: 2, duration: '30m' }; + export default function () { + http.get('http://helm-primary/index.yaml', { timeout: '5s' }); + sleep(1); + } +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: k6-load +spec: + replicas: 1 + selector: + matchLabels: + app: k6-load + template: + metadata: + labels: + app: k6-load + spec: + containers: + - name: k6 + image: grafana/k6:0.49.0 + args: ["run", "/scripts/load.js", "--quiet", "--no-usage-report"] + volumeMounts: + - name: scripts + mountPath: /scripts + resources: + limits: + cpu: 200m + memory: 192Mi + volumes: + - name: scripts + configMap: + name: k6-script diff --git a/tests/resources/serviceref-rulebinding.yaml b/tests/resources/serviceref-rulebinding.yaml deleted file mode 100644 index 24955a4841..0000000000 --- a/tests/resources/serviceref-rulebinding.yaml +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: kubescape.io/v1 -kind: RuntimeRuleAlertBinding -metadata: - name: serviceref-test-binding -spec: - namespaceSelector: - podSelector: - matchLabels: - app: source-controller - rules: - - ruleName: "TEST internal egress not in profile" diff --git a/tests/resources/serviceref-rules.yaml b/tests/resources/serviceref-rules.yaml deleted file mode 100644 index b989ae570b..0000000000 --- a/tests/resources/serviceref-rules.yaml +++ /dev/null @@ -1,27 +0,0 @@ -apiVersion: kubescape.io/v1 -kind: Rules -metadata: - name: serviceref-test-rules - namespace: kubescape - labels: - app: kubescape -spec: - rules: - - name: "TEST internal egress not in profile" - enabled: true - id: "R9911" - description: "Test rule: egress to a cluster-internal address the profile does not allow. The shipped R0011 excludes private destinations, so in-cluster lateral movement — exactly what serviceRef/serviceSelector narrow down — is invisible to it. Scoped to the serviceRef suite via its own binding so no other test's namespace is affected." - expressions: - message: "'Unexpected internal egress to: ' + event.dstAddr + ':' + string(event.dstPort) + ' from: ' + event.containerName" - uniqueId: "'R9911_' + event.dstAddr + '_' + string(event.dstPort)" - ruleExpression: - - eventType: "network" - expression: "event.pktType == 'OUTGOING' && net.is_private_ip(event.dstAddr) && !cp.was_address_in_egress(event.containerId, event.dstAddr)" - profileDependency: 0 - profileDataRequired: - egressAddresses: all - severity: 5 - supportPolicy: false - isTriggerAlert: false - mitreTactic: "TA0008" - mitreTechnique: "T1210" From 20791f14dd6b2deeae8c56d540bc25eccae98588 Mon Sep 17 00:00:00 2001 From: tanzee Date: Mon, 24 Aug 2026 19:27:56 +0200 Subject: [PATCH 15/19] chart(kubescape-rules): standalone chart shipping the R0011/R0012 ruleset Deployable over any kubescape install to replace the stock rules; namespace templated. A drift test pins the chart copy to the CI-validated test-chart copy so the shipped semantics are always the tested ones. Signed-off-by: tanzee --- charts/kubescape-rules/Chart.yaml | 6 + charts/kubescape-rules/templates/binding.yaml | 44 + charts/kubescape-rules/templates/rules.yaml | 795 ++++++++++++++++++ charts/kubescape-rules/values.yaml | 1 + tests/resources/rules_chart_drift_test.go | 30 + 5 files changed, 876 insertions(+) create mode 100644 charts/kubescape-rules/Chart.yaml create mode 100644 charts/kubescape-rules/templates/binding.yaml create mode 100644 charts/kubescape-rules/templates/rules.yaml create mode 100644 charts/kubescape-rules/values.yaml create mode 100644 tests/resources/rules_chart_drift_test.go diff --git a/charts/kubescape-rules/Chart.yaml b/charts/kubescape-rules/Chart.yaml new file mode 100644 index 0000000000..4d977a85e1 --- /dev/null +++ b/charts/kubescape-rules/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: kubescape-rules +description: Kubescape runtime detection rules — symmetric R0011 egress / R0012 ingress, port-aware, selector- and serviceRef-allowlisted internal traffic +type: application +version: 0.1.0 +appVersion: "network-v2" diff --git a/charts/kubescape-rules/templates/binding.yaml b/charts/kubescape-rules/templates/binding.yaml new file mode 100644 index 0000000000..755bd39055 --- /dev/null +++ b/charts/kubescape-rules/templates/binding.yaml @@ -0,0 +1,44 @@ +apiVersion: kubescape.io/v1 +kind: RuntimeRuleAlertBinding +metadata: + name: all-rules-all-pods +spec: + namespaceSelector: + # exclude K8s system namespaces + matchExpressions: + - key: "kubernetes.io/metadata.name" + operator: "NotIn" + values: + - "kube-system" + - "kube-public" + - "kube-node-lease" + - "kubeconfig" + rules: + - ruleName: "Unexpected process launched" + - ruleName: "Unexpected process arguments" + - ruleName: "Files Access Anomalies in container" + - ruleName: "Syscalls Anomalies in container" + - ruleName: "Linux Capabilities Anomalies in container" + - ruleName: "DNS Anomalies in container" + - ruleName: "Unexpected service account token access" + - ruleName: "Workload uses Kubernetes API unexpectedly" + - ruleName: "Process Executed from /dev/shm" + - ruleName: "Process tries to load a kernel module" + - ruleName: "Drifted process executed" + - ruleName: "SSH Connection to Unexpected Destination on Non-Standard Port" + - ruleName: "Fileless execution detected" + - ruleName: "Crypto miner launched" + - ruleName: "Process executed from mount" + - ruleName: "Crypto Mining Related Port Communication" + - ruleName: "Crypto Mining Domain Communication" + - ruleName: "Read Environment Variables from procfs" + - ruleName: "eBPF Program Load" + - ruleName: "Soft link created over sensitive file" + - ruleName: "Unexpected Sensitive File Access" + - ruleName: "Hard link created over sensitive file" + - ruleName: "Exec to pod" + - ruleName: "Port forward to pod" + - ruleName: "Unexpected Egress Network Traffic" + - ruleName: "Unexpected Ingress Network Traffic" + - ruleName: "Unexpected Ptrace Syscall Usage" + - ruleName: "Unexpected io_uring Operation Detected" diff --git a/charts/kubescape-rules/templates/rules.yaml b/charts/kubescape-rules/templates/rules.yaml new file mode 100644 index 0000000000..814319879a --- /dev/null +++ b/charts/kubescape-rules/templates/rules.yaml @@ -0,0 +1,795 @@ +apiVersion: kubescape.io/v1 +kind: Rules +metadata: + name: kubescape-rules + namespace: {{ .Values.ksNamespace }} + annotations: + kubescape.io/namespace: {{ .Values.ksNamespace }} + labels: + app: kubescape +spec: + rules: + - name: "Unexpected process launched" + enabled: true + id: "R0001" + description: "Detects unexpected process launches that are not in the baseline" + expressions: + message: "'Unexpected process launched: ' + event.comm + ' with PID ' + string(event.pid)" + uniqueId: "event.comm + '_' + event.exepath" + ruleExpression: + - eventType: "exec" + expression: "!cp.was_executed(event.containerId, (event.exepath != \"\" ? event.exepath : parse.get_exec_path(event.args, event.comm)))" + profileDependency: 0 + profileDataRequired: + execs: all + severity: 1 + supportPolicy: false + isTriggerAlert: true + mitreTactic: "TA0002" + mitreTechnique: "T1059" + tags: + - "context:kubernetes" + - "context:container" + - "anomaly" + - "process" + - "exec" + - "applicationprofile" + - name: "Files Access Anomalies in container" + enabled: false + id: "R0002" + description: "Detects unexpected file access that is not in the baseline" + expressions: + message: "'Unexpected file access detected: ' + event.comm + ' with PID ' + string(event.pid) + ' to ' + event.path" + uniqueId: "event.comm + '_' + event.path" + ruleExpression: + - eventType: "open" + expression: > + (event.path.startsWith('/etc/') || + event.path.startsWith('/var/log/') || + event.path.startsWith('/var/run/') || + event.path.startsWith('/run/') || + event.path.startsWith('/var/spool/cron/') || + event.path.startsWith('/var/www/') || + event.path.startsWith('/var/lib/') || + event.path.startsWith('/opt/') || + event.path.startsWith('/usr/local/') || + event.path.startsWith('/app/') || + event.path == '/.dockerenv' || + event.path == '/proc/self/environ') + && + !(event.path.startsWith('/run/secrets/kubernetes.io/serviceaccount') || + event.path.startsWith('/var/run/secrets/kubernetes.io/serviceaccount') || + event.path.startsWith('/tmp')) + && + !cp.was_path_opened(event.containerId, event.path) + profileDependency: 0 + profileDataRequired: + opens: + - prefix: "/etc/" + - prefix: "/var/log/" + - prefix: "/var/run/" + - prefix: "/run/" + - prefix: "/var/spool/cron/" + - prefix: "/var/www/" + - prefix: "/var/lib/" + - prefix: "/opt/" + - prefix: "/usr/local/" + - prefix: "/app/" + - exact: "/.dockerenv" + - exact: "/proc/self/environ" + severity: 1 + supportPolicy: false + isTriggerAlert: false + mitreTactic: "TA0009" + mitreTechnique: "T1005" + tags: + - "context:kubernetes" + - "context:container" + - "anomaly" + - "file" + - "open" + - "applicationprofile" + - name: "Syscalls Anomalies in container" + enabled: true + id: "R0003" + description: "Detects unexpected system calls that are not allowlisted by application profile" + expressions: + message: "'Unexpected system call detected: ' + event.syscallName + ' with PID ' + string(event.pid)" + uniqueId: "event.syscallName" + ruleExpression: + - eventType: "syscall" + expression: "!cp.was_syscall_used(event.containerId, event.syscallName)" + profileDependency: 0 + profileDataRequired: + syscalls: all + severity: 1 + supportPolicy: false + isTriggerAlert: false + mitreTactic: "TA0002" + mitreTechnique: "T1059" + tags: + - "context:kubernetes" + - "context:container" + - "anomaly" + - "syscall" + - "applicationprofile" + - name: "Linux Capabilities Anomalies in container" + enabled: true + id: "R0004" + description: "Detects unexpected capabilities that are not allowlisted by application profile" + expressions: + message: "'Unexpected capability used: ' + event.capName + ' in syscall ' + event.syscallName + ' with PID ' + string(event.pid)" + uniqueId: "event.comm + '_' + event.capName" + ruleExpression: + - eventType: "capabilities" + expression: "!cp.was_capability_used(event.containerId, event.capName)" + profileDependency: 0 + profileDataRequired: + capabilities: all + severity: 1 + supportPolicy: false + isTriggerAlert: false + mitreTactic: "TA0002" + mitreTechnique: "T1059" + tags: + - "context:kubernetes" + - "context:container" + - "anomaly" + - "capabilities" + - "applicationprofile" + - name: "DNS Anomalies in container" + enabled: true + id: "R0005" + description: "Detecting unexpected domain requests that are not allowlisted by application profile." + expressions: + message: "'Unexpected domain communication: ' + event.name + ' from: ' + event.containerName" + uniqueId: "event.comm + '_' + event.name" + ruleExpression: + - eventType: "dns" + expression: "!event.name.endsWith('.svc.cluster.local.') && !cp.is_domain_in_egress(event.containerId, event.name)" + profileDependency: 0 + profileDataRequired: + egressDomains: all + severity: 1 + supportPolicy: false + isTriggerAlert: false + mitreTactic: "TA0011" + mitreTechnique: "T1071.004" + tags: + - "context:kubernetes" + - "context:container" + - "dns" + - "anomaly" + - "networkprofile" + - name: "Unexpected service account token access" + enabled: true + id: "R0006" + description: "Detecting unexpected access to service account token." + expressions: + message: "'Unexpected access to service account token: ' + event.path + ' with flags: ' + event.flags.join(',')" + uniqueId: "event.comm" + ruleExpression: + - eventType: "open" + expression: > + ((event.path.startsWith('/run/secrets/kubernetes.io/serviceaccount') && event.path.endsWith('/token')) || + (event.path.startsWith('/var/run/secrets/kubernetes.io/serviceaccount') && event.path.endsWith('/token')) || + (event.path.startsWith('/run/secrets/eks.amazonaws.com/serviceaccount') && event.path.endsWith('/token')) || + (event.path.startsWith('/var/run/secrets/eks.amazonaws.com/serviceaccount') && event.path.endsWith('/token'))) && + !cp.was_path_opened_with_suffix(event.containerId, '/token') + state: + includePrefixes: + - /run/secrets + - /var/run/secrets + profileDependency: 0 + profileDataRequired: + opens: + - suffix: "/token" + severity: 5 + supportPolicy: false + isTriggerAlert: true + mitreTactic: "TA0006" + mitreTechnique: "T1528" + tags: + - "context:kubernetes" + - "anomaly" + - "serviceaccount" + - "applicationprofile" + - name: "Workload uses Kubernetes API unexpectedly" + enabled: true + id: "R0007" + description: "Detecting execution of kubernetes client" + expressions: + message: "eventType == 'exec' ? 'Kubernetes client (' + event.comm + ') was executed with PID ' + string(event.pid) : 'Network connection to Kubernetes API server from container ' + event.containerName" + uniqueId: "eventType == 'exec' ? 'exec_' + event.comm : 'network_' + event.dstAddr" + ruleExpression: + - eventType: "exec" + expression: "(event.comm == 'kubectl' || event.exepath.endsWith('/kubectl')) && !cp.was_executed(event.containerId, (event.exepath != \"\" ? event.exepath : parse.get_exec_path(event.args, event.comm)))" + - eventType: "network" + expression: "event.pktType == 'OUTGOING' && k8s.is_api_server_address(event.dstAddr) && !cp.was_address_in_egress(event.containerId, event.dstAddr)" + profileDependency: 0 + profileDataRequired: + execs: all + egressAddresses: all + severity: 5 # Medium + supportPolicy: false + isTriggerAlert: false + mitreTactic: "TA0008" + mitreTechnique: "T1210" + tags: + - "context:kubernetes" + - "exec" + - "network" + - "anomaly" + - "applicationprofile" + - name: "Read Environment Variables from procfs" + enabled: true + id: "R0008" + description: "Detecting reading environment variables from procfs." + expressions: + message: "'Reading environment variables from procfs: ' + event.path + ' by process ' + event.comm" + uniqueId: "event.comm" + ruleExpression: + - eventType: "open" + expression: > + event.path.startsWith('/proc/') && + event.path.endsWith('/environ') && + !cp.was_path_opened_with_suffix(event.containerId, '/environ') + state: + includePrefixes: + - /proc + profileDependency: 0 # Required + profileDataRequired: + opens: + - suffix: "/environ" + severity: 5 # Medium + supportPolicy: false + isTriggerAlert: true + mitreTactic: "TA0006" + mitreTechnique: "T1552.001" + tags: + - "context:kubernetes" + - "context:container" + - "anomaly" + - "procfs" + - "environment" + - "applicationprofile" + - name: "eBPF Program Load" + enabled: true + id: "R0009" + description: "Detecting eBPF program load." + expressions: + message: "'bpf program load system call (bpf) was called by process (' + event.comm + ') with command (BPF_PROG_LOAD)'" + uniqueId: "event.comm + '_' + 'bpf' + '_' + string(event.cmd)" + ruleExpression: + - eventType: "bpf" + expression: "event.cmd == uint(5) && !cp.was_syscall_used(event.containerId, 'bpf')" + profileDependency: 1 + profileDataRequired: + syscalls: + - exact: "bpf" + severity: 5 + supportPolicy: false + isTriggerAlert: true + mitreTactic: "TA0005" + mitreTechnique: "T1218" + tags: + - "context:kubernetes" + - "context:host" + - "bpf" + - "ebpf" + - "applicationprofile" + - name: "Unexpected Sensitive File Access" + enabled: true + id: "R0010" + description: "Detecting access to sensitive files." + expressions: + message: "'Unexpected sensitive file access: ' + event.path + ' by process ' + event.comm" + uniqueId: "event.comm + '_' + event.path" + ruleExpression: + - eventType: "open" + expression: "event.path.startsWith('/etc/shadow') && !cp.was_path_opened(event.containerId, event.path)" + profileDependency: 1 + profileDataRequired: + opens: + - prefix: "/etc/shadow" + severity: 5 + supportPolicy: false + isTriggerAlert: true + mitreTactic: "TA0006" + mitreTechnique: "T1005" + tags: + - "context:kubernetes" + - "context:container" + - "context:host" + - "files" + - "anomaly" + - "applicationprofile" + - name: "Unexpected Egress Network Traffic" + enabled: true + id: "R0011" + description: "Detecting unexpected egress network traffic that is not allowlisted by application profile." + expressions: + message: "'Unexpected egress network communication to: ' + event.dstAddr + ':' + string(event.dstPort) + ' using ' + event.proto + ' from: ' + event.containerName" + uniqueId: "event.dstAddr + '_' + string(event.dstPort) + '_' + event.proto" + ruleExpression: + - eventType: "network" + expression: "event.pktType == 'OUTGOING' && !event.dstAddr.startsWith('127.') && event.dstAddr != '::1' && !cp.was_address_port_protocol_in_egress(event.containerId, event.dstAddr, event.dstPort, event.proto)" + profileDependency: 0 + profileDataRequired: + egressAddresses: all + severity: 5 # Medium + supportPolicy: false + isTriggerAlert: false + mitreTactic: "TA0010" + mitreTechnique: "T1041" + tags: + - "context:kubernetes" + - "context:container" + - "whitelisted" + - "network" + - "anomaly" + - "networkprofile" + - name: "Unexpected Ingress Network Traffic" + enabled: true + id: "R0012" + description: "Detecting unexpected ingress network traffic that is not allowlisted by application profile. Symmetric twin of R0011: internal and external peers alike, only loopback excluded; internal peers are allowlisted narrowly via addresses or resolved serviceRef/serviceSelector entries rather than a serviceCIDR that blinds detection to lateral movement." + expressions: + message: "'Unexpected ingress network communication from: ' + event.dstAddr + ':' + string(event.dstPort) + ' using ' + event.proto + ' to: ' + event.containerName" + uniqueId: "event.dstAddr + '_' + string(event.dstPort) + '_' + event.proto" + ruleExpression: + - eventType: "network" + expression: "event.pktType == 'HOST' && !event.dstAddr.startsWith('127.') && event.dstAddr != '::1' && !cp.was_address_port_protocol_in_ingress(event.containerId, event.dstAddr, event.dstPort, event.proto)" + profileDependency: 0 + profileDataRequired: + ingressAddresses: all + severity: 5 # Medium + supportPolicy: false + isTriggerAlert: false + mitreTactic: "TA0008" + mitreTechnique: "T1210" + tags: + - "context:kubernetes" + - "context:container" + - "whitelisted" + - "network" + - "anomaly" + - "networkprofile" + - name: "Unexpected process arguments" + enabled: true + id: "R0040" + description: "Detects an exec event whose path IS in the application profile but whose argv vector does not match any recorded argv pattern for that path. Consumes cp.was_executed_with_args, which walks the ExecsByPath projection surface and delegates argv comparison to dynamicpathdetector.MatchExecArgs (storage). Stays silent when the path is unknown (R0001 covers that case) and when the argv vector matches any recorded pattern (including the trailing zero-or-more form and the single-arg form); a '*' in a recorded arg is a literal character, not a wildcard." + expressions: + message: "'Unexpected process arguments: ' + event.comm + ' with PID ' + string(event.pid) + ' argv=' + event.args.map(a, string(a)).join(' ')" + uniqueId: "event.comm + '_' + event.exepath + '_' + event.args.map(a, string(a)).join(' ')" + ruleExpression: + - eventType: "exec" + expression: "cp.was_executed(event.containerId, parse.get_exec_path(event.args, event.comm, event.exepath)) && !cp.was_executed_with_args(event.containerId, parse.get_exec_path(event.args, event.comm, event.exepath), event.args)" + profileDependency: 0 + profileDataRequired: + execs: all + severity: 3 + supportPolicy: false + isTriggerAlert: true + mitreTactic: "TA0002" + mitreTechnique: "T1059" + tags: + - "context:kubernetes" + - "context:container" + - "anomaly" + - "process" + - "exec" + - "applicationprofile" + - name: "Process Executed from /dev/shm" + enabled: true + id: "R1000" + description: "Detecting exec calls whose executable path or working directory is under /dev/shm, a world-writable memory-backed (tmpfs) directory." + expressions: + message: "'Process executed from /dev/shm: ' + event.exepath + ' in directory ' + event.cwd" + uniqueId: "event.comm + '_' + event.exepath + '_' + event.pcomm" + ruleExpression: + - eventType: "exec" + expression: > + (event.exepath == '/dev/shm' || event.exepath.startsWith('/dev/shm/')) || + (event.cwd == '/dev/shm' || event.cwd.startsWith('/dev/shm/')) || + (event.args.size() > 0 && (event.args[0] == '/dev/shm' || event.args[0].startsWith('/dev/shm/'))) + profileDependency: 2 + severity: 8 + supportPolicy: false + isTriggerAlert: true + mitreTactic: "TA0002" + mitreTechnique: "T1059" + tags: + - "context:kubernetes" + - "context:host" + - "exec" + - "signature" + - "malicious" + - name: "Drifted process executed" + enabled: true + id: "R1001" + description: "Detecting exec calls of binaries that are not included in the base image" + expressions: + message: "'Process (' + event.comm + ') was executed and is not part of the image'" + uniqueId: "event.comm + '_' + event.exepath + '_' + event.pcomm" + ruleExpression: + - eventType: "exec" + expression: > + (event.upperlayer == true || + event.pupperlayer == true) && + !cp.was_executed(event.containerId, (event.exepath != "" ? event.exepath : parse.get_exec_path(event.args, event.comm))) + profileDependency: 1 + profileDataRequired: + execs: all + severity: 8 + supportPolicy: false + isTriggerAlert: true + mitreTactic: "TA0005" + mitreTechnique: "T1036" + tags: + - "context:kubernetes" + - "context:container" + - "exec" + - "malicious" + - "binary" + - "base image" + - "applicationprofile" + - name: "Process tries to load a kernel module" + enabled: true + id: "R1002" + description: "Detecting Kernel Module Load." + expressions: + message: "'Kernel module (' + event.module + ') loading attempt with syscall (' + event.syscallName + ') was called by process (' + event.comm + ')'" + uniqueId: "event.comm + '_' + event.syscallName + '_' + event.module" + ruleExpression: + - eventType: "kmod" + expression: "event.syscallName == 'init_module' || event.syscallName == 'finit_module'" + profileDependency: 2 + severity: 10 + supportPolicy: false + isTriggerAlert: true + mitreTactic: "TA0005" + mitreTechnique: "T1547.006" + tags: + - "context:kubernetes" + - "context:host" + - "kmod" + - "kernel" + - "module" + - "load" + - name: "SSH Connection to Unexpected Destination on Non-Standard Port" + enabled: false + id: "R1003" + description: "Detecting an SSH connection to a non-standard port where the destination address is not in the container's learned egress baseline." + expressions: + message: "'SSH connection to unexpected destination on non-standard port: ' + event.dstIp + ':' + string(dyn(event.dstPort))" + uniqueId: "event.comm + '_' + event.dstIp + '_' + string(dyn(event.dstPort))" + ruleExpression: + - eventType: "ssh" + expression: "dyn(event.srcPort) >= 32768 && dyn(event.srcPort) <= 60999 && !(dyn(event.dstPort) in [22, 2022]) && !cp.was_address_in_egress(event.containerId, event.dstIp)" + profileDependency: 1 + profileDataRequired: + egressAddresses: all + severity: 5 + supportPolicy: false + isTriggerAlert: true + mitreTactic: "TA0008" + mitreTechnique: "T1021.001" + tags: + - "context:kubernetes" + - "context:container" + - "ssh" + - "connection" + - "port" + - "malicious" + - "networkprofile" + - name: "Process executed from mount" + enabled: true + id: "R1004" + description: "Detecting exec calls from mounted paths." + expressions: + message: "'Process (' + event.comm + ') was executed from a mounted path'" + uniqueId: "event.comm" + ruleExpression: + - eventType: "exec" + expression: "!cp.was_executed(event.containerId, (event.exepath != \"\" ? event.exepath : parse.get_exec_path(event.args, event.comm))) && k8s.get_container_mount_paths(event.namespace, event.podName, event.containerName).exists(mount, event.exepath.startsWith(mount) || (event.exepath != \"\" ? event.exepath : parse.get_exec_path(event.args, event.comm)).startsWith(mount))" + profileDependency: 1 + profileDataRequired: + execs: all + severity: 5 + supportPolicy: false + isTriggerAlert: true + mitreTactic: "TA0002" + mitreTechnique: "T1059" + tags: + - "context:kubernetes" + - "context:container" + - "exec" + - "mount" + - "applicationprofile" + - name: "Fileless execution detected" + enabled: true + id: "R1005" + description: "Detecting Fileless Execution" + expressions: + message: '''Fileless execution detected: exec call "'' + event.comm + ''" runs from a memory-backed source (memfd / /proc/self/fd)''' + uniqueId: "event.comm + '_' + event.exepath + '_' + event.pcomm" + ruleExpression: + - eventType: "exec" + expression: "event.exepath.contains('memfd') || event.exepath.startsWith('/proc/self/fd') || event.exepath.matches('/proc/[0-9]+/fd/[0-9]+')" + profileDependency: 2 + severity: 8 + supportPolicy: false + isTriggerAlert: true + mitreTactic: "TA0005" + mitreTechnique: "T1055" + tags: + - "context:kubernetes" + - "context:host" + - "fileless" + - "execution" + - "malicious" + - name: "Unexpected unshare Syscall in Container" + enabled: true + id: "R1006" + description: "Detecting use of the unshare system call (a namespace-manipulation capability that can be used to escape a container) by a non-runc process, where it was not seen in the container's application-profile baseline." + expressions: + message: "'Unshare system call (unshare) was called by process (' + event.comm + ')'" + uniqueId: "event.comm + '_' + 'unshare'" + ruleExpression: + - eventType: "unshare" + expression: "event.pcomm != 'runc' && !cp.was_syscall_used(event.containerId, 'unshare')" + profileDependency: 1 + profileDataRequired: + syscalls: + - exact: "unshare" + severity: 5 + supportPolicy: false + isTriggerAlert: true + mitreTactic: "TA0004" + mitreTechnique: "T1611" + tags: + - "context:kubernetes" + - "context:container" + - "unshare" + - "escape" + - "unshare" + - "anomaly" + - "applicationprofile" + - name: "Crypto miner launched" + enabled: true + id: "R1007" + description: "Detecting XMR Crypto Miners by randomx algorithm usage." + expressions: + message: "'XMR Crypto Miner process: (' + event.exepath + ') executed'" + uniqueId: "event.exepath + '_' + event.comm" + ruleExpression: + - eventType: "randomx" + expression: "true" + profileDependency: 2 + severity: 10 + supportPolicy: false + isTriggerAlert: true + mitreTactic: "TA0040" + mitreTechnique: "T1496" + tags: + - "context:kubernetes" + - "context:container" + - "crypto" + - "miners" + - "malicious" + - name: "Crypto Mining Domain Communication" + enabled: true + id: "R1008" + description: "Detecting Crypto miners communication by domain" + expressions: + message: "'Communication with a known crypto mining domain: ' + event.name" + uniqueId: "event.name + '_' + event.comm" + ruleExpression: + - eventType: "dns" + expression: "event.name in ['2cryptocalc.com.', '2miners.com.', 'antpool.com.', 'asia1.ethpool.org.', 'bohemianpool.com.', 'botbox.dev.', 'btm.antpool.com.', 'c3pool.com.', 'c4pool.org.', 'ca.minexmr.com.', 'cn.stratum.slushpool.com.', 'dash.antpool.com.', 'data.miningpoolstats.stream.', 'de.minexmr.com.', 'eth-ar.dwarfpool.com.', 'eth-asia.dwarfpool.com.', 'eth-asia1.nanopool.org.', 'eth-au.dwarfpool.com.', 'eth-au1.nanopool.org.', 'eth-br.dwarfpool.com.', 'eth-cn.dwarfpool.com.', 'eth-cn2.dwarfpool.com.', 'eth-eu.dwarfpool.com.', 'eth-eu1.nanopool.org.', 'eth-eu2.nanopool.org.', 'eth-hk.dwarfpool.com.', 'eth-jp1.nanopool.org.', 'eth-ru.dwarfpool.com.', 'eth-ru2.dwarfpool.com.', 'eth-sg.dwarfpool.com.', 'eth-us-east1.nanopool.org.', 'eth-us-west1.nanopool.org.', 'eth-us.dwarfpool.com.', 'eth-us2.dwarfpool.com.', 'eth.antpool.com.', 'eu.stratum.slushpool.com.', 'eu1.ethermine.org.', 'eu1.ethpool.org.', 'fastpool.xyz.', 'fr.minexmr.com.', 'kriptokyng.com.', 'mine.moneropool.com.', 'mine.xmrpool.net.', 'miningmadness.com.', 'monero.cedric-crispin.com.', 'monero.crypto-pool.fr.', 'monero.fairhash.org.', 'monero.hashvault.pro.', 'monero.herominers.com.', 'monerod.org.', 'monerohash.com.', 'moneroocean.stream.', 'monerop.com.', 'multi-pools.com.', 'p2pool.io.', 'pool.kryptex.com.', 'pool.minexmr.com.', 'pool.monero.hashvault.pro.', 'pool.rplant.xyz.', 'pool.supportxmr.com.', 'pool.xmr.pt.', 'prohashing.com.', 'rx.unmineable.com.', 'sg.minexmr.com.', 'sg.stratum.slushpool.com.', 'skypool.org.', 'solo-xmr.2miners.com.', 'ss.antpool.com.', 'stratum-btm.antpool.com.', 'stratum-dash.antpool.com.', 'stratum-eth.antpool.com.', 'stratum-ltc.antpool.com.', 'stratum-xmc.antpool.com.', 'stratum-zec.antpool.com.', 'stratum.antpool.com.', 'supportxmr.com.', 'trustpool.cc.', 'us-east.stratum.slushpool.com.', 'us1.ethermine.org.', 'us1.ethpool.org.', 'us2.ethermine.org.', 'us2.ethpool.org.', 'web.xmrpool.eu.', 'www.domajorpool.com.', 'www.dxpool.com.', 'www.mining-dutch.nl.', 'xmc.antpool.com.', 'xmr-asia1.nanopool.org.', 'xmr-au1.nanopool.org.', 'xmr-eu1.nanopool.org.', 'xmr-eu2.nanopool.org.', 'xmr-jp1.nanopool.org.', 'xmr-us-east1.nanopool.org.', 'xmr-us-west1.nanopool.org.', 'xmr.2miners.com.', 'xmr.crypto-pool.fr.', 'xmr.gntl.uk.', 'xmr.nanopool.org.', 'xmr.pool-pay.com.', 'xmr.pool.minergate.com.', 'xmr.solopool.org.', 'xmr.volt-mine.com.', 'xmr.zeropool.io.', 'zec.antpool.com.', 'zergpool.com.', 'auto.c3pool.org.', 'us.monero.herominers.com.', 'xmr.kryptex.network.']" + profileDependency: 2 + severity: 10 + supportPolicy: false + isTriggerAlert: true + mitreTactic: "TA0011" + mitreTechnique: "T1071.004" + tags: + - "context:kubernetes" + - "context:host" + - "network" + - "crypto" + - "miners" + - "malicious" + - "dns" + - name: "Crypto Mining Related Port Communication" + enabled: true + id: "R1009" + description: "Detecting Crypto Miners by suspicious port usage." + expressions: + message: "'Detected crypto mining related port communication on port ' + string(event.dstPort) + ' to ' + event.dstAddr + ' with protocol ' + event.proto" + uniqueId: "event.comm + '_' + string(event.dstPort)" + ruleExpression: + - eventType: "network" + expression: "event.proto == 'TCP' && event.pktType == 'OUTGOING' && event.dstPort in [3333, 45700] && !cp.was_address_in_egress(event.containerId, event.dstAddr)" + state: + ports: + - 3333 + - 45700 + profileDependency: 1 + profileDataRequired: + egressAddresses: all + severity: 3 + supportPolicy: false + isTriggerAlert: false + mitreTactic: "TA0011" + mitreTechnique: "T1071" + tags: + - "context:kubernetes" + - "context:host" + - "network" + - "crypto" + - "miners" + - "malicious" + - "networkprofile" + - name: "Soft link created over sensitive file" + enabled: true + id: "R1010" + description: "Detects symlink creation over sensitive files" + expressions: + message: "'Symlink created over sensitive file: ' + event.oldPath + ' -> ' + event.newPath" + uniqueId: "event.comm + '_' + event.oldPath" + ruleExpression: + - eventType: "symlink" + expression: "(event.oldPath.startsWith('/etc/shadow') || event.oldPath.startsWith('/etc/sudoers')) && !cp.was_path_opened(event.containerId, event.oldPath)" + profileDependency: 1 + profileDataRequired: + opens: + - prefix: "/etc/shadow" + - prefix: "/etc/sudoers" + severity: 5 + supportPolicy: true + isTriggerAlert: true + mitreTactic: "TA0006" + mitreTechnique: "T1005" + tags: + - "context:kubernetes" + - "context:host" + - "anomaly" + - "symlink" + - "applicationprofile" + - name: "ld_preload Mechanism Use or ld.so.preload Modification" + enabled: false + id: "R1011" + description: "Detecting use of the LD_PRELOAD/LD_LIBRARY_PATH dynamic-linker hook mechanism, or an unexpected write to /etc/ld.so.preload relative to the container's application-profile baseline." + expressions: + message: "eventType == 'exec' ? 'Process (' + event.comm + ') is using a dynamic linker hook: ' + process.get_ld_hook_var(event.pid) : 'The dynamic linker configuration file (' + event.path + ') was modified by process (' + event.comm + ')'" + uniqueId: "eventType == 'exec' ? 'exec_' + event.comm : 'open_' + event.path" + ruleExpression: + - eventType: "exec" + expression: "event.comm != 'java' && event.containerName != 'matlab' && process.get_ld_hook_var(event.pid) != ''" + - eventType: "open" + expression: "event.path == '/etc/ld.so.preload' && has(event.flagsRaw) && event.flagsRaw != 0" + profileDependency: 1 + profileDataRequired: + opens: + - exact: "/etc/ld.so.preload" + severity: 5 + supportPolicy: true + isTriggerAlert: true + mitreTactic: "TA0005" + mitreTechnique: "T1574.006" + tags: + - "context:kubernetes" + - "exec" + - "malicious" + - "applicationprofile" + - name: "Hard link created over sensitive file" + enabled: true + id: "R1012" + description: "Detecting hardlink creation over sensitive files." + expressions: + message: "'Hardlink created over sensitive file: ' + event.oldPath + ' - ' + event.newPath" + uniqueId: "event.comm + '_' + event.oldPath" + ruleExpression: + - eventType: "hardlink" + expression: "(event.oldPath.startsWith('/etc/shadow') || event.oldPath.startsWith('/etc/sudoers')) && !cp.was_path_opened(event.containerId, event.oldPath)" + profileDependency: 1 + profileDataRequired: + opens: + - prefix: "/etc/shadow" + - prefix: "/etc/sudoers" + severity: 5 + supportPolicy: true + isTriggerAlert: true + mitreTactic: "TA0006" + mitreTechnique: "T1005" + tags: + - "context:kubernetes" + - "files" + - "malicious" + - "applicationprofile" + - name: "Unexpected Ptrace Syscall Usage" + enabled: true + id: "R1015" + description: "Detecting use of the ptrace syscall that was not seen in the container's application-profile baseline." + expressions: + message: "'Unexpected ptrace syscall usage from: ' + event.comm" + uniqueId: "event.exepath + '_' + event.comm" + ruleExpression: + - eventType: "ptrace" + expression: "true" + profileDependency: 2 + severity: 5 + supportPolicy: false + isTriggerAlert: true + mitreTactic: "TA0005" + mitreTechnique: "T1622" + tags: + - "context:kubernetes" + - "context:host" + - "process" + - "malicious" + - name: "Unexpected io_uring Operation Detected" + enabled: true + id: "R1030" + description: "Detects io_uring operations that were not recorded during the initial observation period, indicating potential unauthorized activity." + expressions: + message: "'Unexpected io_uring operation detected: (opcode=' + string(event.opcode) + ') flags=0x' + (has(event.flagsRaw) ? string(event.flagsRaw) : '0') + ' in ' + event.comm + '.'" + uniqueId: "string(event.opcode) + '_' + event.comm" + ruleExpression: + - eventType: "iouring" + expression: "true" + profileDependency: 0 + profileDataRequired: + syscalls: all + severity: 5 + supportPolicy: true + isTriggerAlert: true + mitreTactic: "TA0002" + mitreTechnique: "T1218" + tags: + - "context:kubernetes" + - "context:container" + - "syscalls" + - "io_uring" + - "applicationprofile" + - name: "Exec to pod" + enabled: true + id: "R2000" + description: "Detects exec operations on pods via the Kubernetes admission webhook (PodExecOptions CONNECT)" + expressions: + message: "'Exec to pod: ' + event.Name + ' in namespace ' + event.Namespace + ' by ' + event.UserInfo.Username" + uniqueId: "event.Namespace + '/' + event.Name" + ruleExpression: + - eventType: "k8s-admission" + expression: 'event.Kind == "PodExecOptions"' + profileDependency: 2 + severity: 8 + supportPolicy: false + isTriggerAlert: true + mitreTactic: "TA0002" + mitreTechnique: "T1609" + tags: + - "context:kubernetes" + - "admission" + - "exec" + - name: "Port forward to pod" + enabled: true + id: "R2001" + description: "Detects port-forward operations on pods via the Kubernetes admission webhook (PodPortForwardOptions CONNECT)" + expressions: + message: "'Port forward to pod: ' + event.Name + ' in namespace ' + event.Namespace + ' by ' + event.UserInfo.Username" + uniqueId: "event.Namespace + '/' + event.Name" + ruleExpression: + - eventType: "k8s-admission" + expression: 'event.Kind == "PodPortForwardOptions"' + profileDependency: 2 + severity: 5 + supportPolicy: false + isTriggerAlert: true + mitreTactic: "TA0011" + mitreTechnique: "T1090" + tags: + - "context:kubernetes" + - "admission" + - "network" diff --git a/charts/kubescape-rules/values.yaml b/charts/kubescape-rules/values.yaml new file mode 100644 index 0000000000..53e2957cc4 --- /dev/null +++ b/charts/kubescape-rules/values.yaml @@ -0,0 +1 @@ +ksNamespace: kubescape diff --git a/tests/resources/rules_chart_drift_test.go b/tests/resources/rules_chart_drift_test.go new file mode 100644 index 0000000000..6629dad0b3 --- /dev/null +++ b/tests/resources/rules_chart_drift_test.go @@ -0,0 +1,30 @@ +package resources + +import ( + "os" + "strings" + "testing" +) + +// The standalone rules chart ships a copy of the test chart's rules; a drift +// between the two would deploy different detection semantics than CI validates. +func TestRulesChartMatchesTestChart(t *testing.T) { + pairs := [][2]string{ + {"../chart/templates/node-agent/default-rules.yaml", "../../charts/kubescape-rules/templates/rules.yaml"}, + {"../chart/templates/node-agent/default-rule-binding.yaml", "../../charts/kubescape-rules/templates/binding.yaml"}, + } + for _, p := range pairs { + a, err := os.ReadFile(p[0]) + if err != nil { + t.Fatal(err) + } + b, err := os.ReadFile(p[1]) + if err != nil { + t.Fatal(err) + } + got := strings.ReplaceAll(string(b), "{{ .Values.ksNamespace }}", "kubescape") + if got != string(a) { + t.Errorf("%s drifted from %s — regenerate the chart copy", p[1], p[0]) + } + } +} From 5a17ba0349249733fd992a1217b6e2809f6d9dff Mon Sep 17 00:00:00 2001 From: tanzee Date: Mon, 24 Aug 2026 20:46:35 +0200 Subject: [PATCH 16/19] rules: consume the peer-selector engine in R0011/R0012 The selector clause was deferred while the engine lived on a separate branch, then forgotten when that branch merged: selectors resolved and matched but no rule consulted them. Both rules now also allowlist via was_selector_in_egress/ingress, matching the form already deployed downstream. Signed-off-by: tanzee --- charts/kubescape-rules/templates/rules.yaml | 4 ++-- tests/chart/templates/node-agent/default-rules.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/charts/kubescape-rules/templates/rules.yaml b/charts/kubescape-rules/templates/rules.yaml index 814319879a..dbcc2c81be 100644 --- a/charts/kubescape-rules/templates/rules.yaml +++ b/charts/kubescape-rules/templates/rules.yaml @@ -313,7 +313,7 @@ spec: uniqueId: "event.dstAddr + '_' + string(event.dstPort) + '_' + event.proto" ruleExpression: - eventType: "network" - expression: "event.pktType == 'OUTGOING' && !event.dstAddr.startsWith('127.') && event.dstAddr != '::1' && !cp.was_address_port_protocol_in_egress(event.containerId, event.dstAddr, event.dstPort, event.proto)" + expression: "event.pktType == 'OUTGOING' && !event.dstAddr.startsWith('127.') && event.dstAddr != '::1' && !cp.was_address_port_protocol_in_egress(event.containerId, event.dstAddr, event.dstPort, event.proto) && !cp.was_selector_in_egress(event.containerId, event.dstNamespace, event.dstPodLabels)" profileDependency: 0 profileDataRequired: egressAddresses: all @@ -338,7 +338,7 @@ spec: uniqueId: "event.dstAddr + '_' + string(event.dstPort) + '_' + event.proto" ruleExpression: - eventType: "network" - expression: "event.pktType == 'HOST' && !event.dstAddr.startsWith('127.') && event.dstAddr != '::1' && !cp.was_address_port_protocol_in_ingress(event.containerId, event.dstAddr, event.dstPort, event.proto)" + expression: "event.pktType == 'HOST' && !event.dstAddr.startsWith('127.') && event.dstAddr != '::1' && !cp.was_address_port_protocol_in_ingress(event.containerId, event.dstAddr, event.dstPort, event.proto) && !cp.was_selector_in_ingress(event.containerId, event.dstNamespace, event.dstPodLabels)" profileDependency: 0 profileDataRequired: ingressAddresses: all diff --git a/tests/chart/templates/node-agent/default-rules.yaml b/tests/chart/templates/node-agent/default-rules.yaml index 98bda9333f..909e9f3f3f 100644 --- a/tests/chart/templates/node-agent/default-rules.yaml +++ b/tests/chart/templates/node-agent/default-rules.yaml @@ -313,7 +313,7 @@ spec: uniqueId: "event.dstAddr + '_' + string(event.dstPort) + '_' + event.proto" ruleExpression: - eventType: "network" - expression: "event.pktType == 'OUTGOING' && !event.dstAddr.startsWith('127.') && event.dstAddr != '::1' && !cp.was_address_port_protocol_in_egress(event.containerId, event.dstAddr, event.dstPort, event.proto)" + expression: "event.pktType == 'OUTGOING' && !event.dstAddr.startsWith('127.') && event.dstAddr != '::1' && !cp.was_address_port_protocol_in_egress(event.containerId, event.dstAddr, event.dstPort, event.proto) && !cp.was_selector_in_egress(event.containerId, event.dstNamespace, event.dstPodLabels)" profileDependency: 0 profileDataRequired: egressAddresses: all @@ -338,7 +338,7 @@ spec: uniqueId: "event.dstAddr + '_' + string(event.dstPort) + '_' + event.proto" ruleExpression: - eventType: "network" - expression: "event.pktType == 'HOST' && !event.dstAddr.startsWith('127.') && event.dstAddr != '::1' && !cp.was_address_port_protocol_in_ingress(event.containerId, event.dstAddr, event.dstPort, event.proto)" + expression: "event.pktType == 'HOST' && !event.dstAddr.startsWith('127.') && event.dstAddr != '::1' && !cp.was_address_port_protocol_in_ingress(event.containerId, event.dstAddr, event.dstPort, event.proto) && !cp.was_selector_in_ingress(event.containerId, event.dstNamespace, event.dstPodLabels)" profileDependency: 0 profileDataRequired: ingressAddresses: all From 4280eae0c7761ce2245faf901ea584a1a7f09437 Mon Sep 17 00:00:00 2001 From: tanzee Date: Tue, 25 Aug 2026 09:18:13 +0200 Subject: [PATCH 17/19] review: address maintainer blockers on network-v2 - reconciler: read lister generation once BEFORE service resolution and stamp that same gen as ResolvedGen/ListerGen, matching addContainer's ordering; a Bump() during resolution now invalidates the projection instead of masking stale IPs behind the fast-skip. - config: networkServiceResolutionEnabled now defaults true (rules ship enabled, resolution must match); test chart configmap falls back to true via hasKey so an explicit false still renders false. - deps: restore accidental downgrades kubescape/backend v0.0.31->v0.0.39 and gotest.tools/v3 v3.5.0->v3.5.2; k8sstormcenter/storage replace pin unchanged (tidy normalized the require placeholder to v0.0.258, the replace still governs). - networkpeer: hasServiceFields no longer counts a ServiceRefNamespace-only neighbor that specFromNeighbor rejects, ending permanent re-projection churn on such profiles; test pins the agreement. - rbac: drop unnecessary get verb on endpointslices (cache-backed informer needs only list+watch). - ct: storage-tag.sh emits the fork storage image tag (net-v2-rc1) while go.mod replaces storage with k8sstormcenter/storage, and the test chart pulls ghcr.io/k8sstormcenter/storage, so CTs run a server that has the serviceRef/dnsNames schema. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01A8UV2B7b6dpDJQci31aC6c Signed-off-by: tanzee --- go.mod | 4 ++-- go.sum | 8 ++++---- pkg/config/config.go | 1 + pkg/config/config_test.go | 1 + pkg/networkpeer/expand.go | 3 ++- pkg/networkpeer/expand_test.go | 5 +++++ pkg/objectcache/containerprofilecache/reconciler.go | 6 ++++-- tests/chart/templates/node-agent/clusterrole.yaml | 2 +- tests/chart/templates/node-agent/configmap.yaml | 2 +- tests/chart/values.yaml | 4 ++-- tests/scripts/storage-tag.sh | 6 ++++++ 11 files changed, 29 insertions(+), 13 deletions(-) diff --git a/go.mod b/go.mod index 0a7d31cceb..13e350ea74 100644 --- a/go.mod +++ b/go.mod @@ -32,10 +32,10 @@ require ( github.com/iceber/iouring-go v0.0.0-20230403020409-002cfd2e2a90 github.com/inspektor-gadget/inspektor-gadget v0.45.1-0.20251020222545-c91c23581ebf github.com/joncrlsn/dque v0.0.0-20241024143830-7723fd131a64 - github.com/kubescape/backend v0.0.31 + github.com/kubescape/backend v0.0.39 github.com/kubescape/go-logger v0.0.32 github.com/kubescape/k8s-interface v0.0.214 - github.com/kubescape/storage v0.0.0-00010101000000-000000000000 + github.com/kubescape/storage v0.0.258 github.com/kubescape/workerpool v0.0.0-20250526074519-0e4a4e7f44cf github.com/moby/sys/mountinfo v0.7.2 github.com/oleiade/lane/v2 v2.0.0 diff --git a/go.sum b/go.sum index f33e276f9d..5c627bb87e 100644 --- a/go.sum +++ b/go.sum @@ -889,8 +889,8 @@ github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/kubescape/backend v0.0.31 h1:pLMic67Vuiksdfh1t7ATq9M9wkrjXtvQfPDopzuGWkA= -github.com/kubescape/backend v0.0.31/go.mod h1:FpazfN+c3Ucuvv4jZYCnk99moSBRNMVIxl5aWCZAEBo= +github.com/kubescape/backend v0.0.39 h1:B1QRfKCSFlzuE+jWOnk/l7EpH71/Q3n14KKq0QSnZwg= +github.com/kubescape/backend v0.0.39/go.mod h1:cMEGP8cXUZgY89YU4GRBGIla9HZW7grZsUtlCwvZgAE= github.com/kubescape/go-logger v0.0.32 h1:4mI+XJOV8VFCMewrEE9VIFEIOhzXokYT3nFpNfXf4fM= github.com/kubescape/go-logger v0.0.32/go.mod h1:Alj7JBQ8/WCxbXe8Ura6ZheSRK45E0p21M3xeqedX90= github.com/kubescape/k8s-interface v0.0.214 h1:j7KP0/5VvYOoQdBGV2+gRM3qnR8PWLAGF8RM/k/DmJ0= @@ -2034,8 +2034,8 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gotest.tools/v3 v3.5.0 h1:Ljk6PdHdOhAb5aDMWXjDLMMhph+BpztA4v1QdqEW2eY= -gotest.tools/v3 v3.5.0/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= +gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= +gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= grpc.go4.org v0.0.0-20170609214715-11d0a25b4919/go.mod h1:77eQGdRu53HpSqPFJFmuJdjuHRquDANNeA4x7B8WQ9o= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/pkg/config/config.go b/pkg/config/config.go index e2d6e5d4a9..61094f8318 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -186,6 +186,7 @@ func LoadConfigOptional(path string, errNotFound bool) (Config, error) { viper.SetDefault("podName", os.Getenv(PodNameEnvVar)) viper.SetDefault("fimEnabled", false) viper.SetDefault("networkStreamingEnabled", false) + viper.SetDefault("networkServiceResolutionEnabled", true) viper.SetDefault("kubernetesMode", true) viper.SetDefault("networkStreamingInterval", 2*time.Minute) viper.SetDefault("workerPoolSize", 3000) diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 754b342279..7624e5e2ce 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -36,6 +36,7 @@ func TestLoadConfig(t *testing.T) { EnableHttpDetection: false, EnableFIM: true, EnableNetworkStreaming: false, + EnableNetworkServiceResolution: true, EnableEmbeddedSboms: false, EnableHostSensor: true, HostSensorInterval: 1 * time.Minute, diff --git a/pkg/networkpeer/expand.go b/pkg/networkpeer/expand.go index 747c7b2d62..fb4afea23d 100644 --- a/pkg/networkpeer/expand.go +++ b/pkg/networkpeer/expand.go @@ -92,8 +92,9 @@ func HasServiceNeighbors(cp *v1beta1.ContainerProfile) bool { return false } +// Must mirror specFromNeighbor's gate: ServiceRefNamespace alone is not a serviceRef. func hasServiceFields(n *v1beta1.NetworkNeighbor) bool { - return n.ServiceRefNamespace != "" || n.ServiceRefName != "" || n.ServiceSelector != nil || n.Entity != "" + return n.ServiceRefName != "" || n.ServiceSelector != nil || n.Entity != "" } // specFromNeighbor extracts a PeerSpec from a NetworkNeighbor, reporting false diff --git a/pkg/networkpeer/expand_test.go b/pkg/networkpeer/expand_test.go index 715a76e607..be7680c59d 100644 --- a/pkg/networkpeer/expand_test.go +++ b/pkg/networkpeer/expand_test.go @@ -214,4 +214,9 @@ func TestHasServiceNeighbors(t *testing.T) { t.Errorf("neighbor %+v should be flagged", n) } } + nsOnly := &v1beta1.ContainerProfile{} + nsOnly.Spec.Ingress = []v1beta1.NetworkNeighbor{{ServiceRefNamespace: "honey"}} + if HasServiceNeighbors(nsOnly) { + t.Error("ServiceRefNamespace without ServiceRefName must not be flagged (specFromNeighbor rejects it)") + } } diff --git a/pkg/objectcache/containerprofilecache/reconciler.go b/pkg/objectcache/containerprofilecache/reconciler.go index b0fe54d8fe..d394b0c61c 100644 --- a/pkg/objectcache/containerprofilecache/reconciler.go +++ b/pkg/objectcache/containerprofilecache/reconciler.go @@ -495,9 +495,11 @@ func (c *ContainerProfileCacheImpl) rebuildEntryFromSources( // Project under the current spec. spec := c.snapshotSpec() + // Read gen BEFORE resolving so a concurrent Bump() invalidates this projection. + gen := c.listerGen() applyStart := time.Now() projectedCP := Apply(spec, networkpeer.WithResolvedServiceNeighbors(projected, c.serviceLister), tree) - projectedCP.ResolvedGen = c.listerGen() + projectedCP.ResolvedGen = gen if c.cfg.ProfileProjection.DetailedMetricsEnabled { c.metricsManager.ObserveProjectionApplyDuration(time.Since(applyStart)) c.observeMemoryMetrics(projected, projectedCP) @@ -507,7 +509,7 @@ func (c *ContainerProfileCacheImpl) rebuildEntryFromSources( Projected: projectedCP, SpecHash: projectedCP.SpecHash, UsesServiceResolution: networkpeer.HasServiceNeighbors(projected), - ListerGen: c.listerGen(), + ListerGen: gen, State: &objectcache.ProfileState{Completion: effectiveCP.Annotations[helpersv1.CompletionMetadataKey], Status: effectiveCP.Annotations[helpersv1.StatusMetadataKey], Name: effectiveCP.Name}, CallStackTree: tree, ContainerName: prev.ContainerName, diff --git a/tests/chart/templates/node-agent/clusterrole.yaml b/tests/chart/templates/node-agent/clusterrole.yaml index a9feeed81a..7d4096a3fe 100644 --- a/tests/chart/templates/node-agent/clusterrole.yaml +++ b/tests/chart/templates/node-agent/clusterrole.yaml @@ -13,7 +13,7 @@ rules: verbs: ["list", "watch", "create"] - apiGroups: ["discovery.k8s.io"] resources: ["endpointslices"] - verbs: ["get", "watch", "list"] + verbs: ["watch", "list"] - apiGroups: ["batch"] resources: ["jobs", "cronjobs"] verbs: ["get", "watch", "list"] diff --git a/tests/chart/templates/node-agent/configmap.yaml b/tests/chart/templates/node-agent/configmap.yaml index 053acf0808..b64a63024b 100644 --- a/tests/chart/templates/node-agent/configmap.yaml +++ b/tests/chart/templates/node-agent/configmap.yaml @@ -14,7 +14,7 @@ data: "prometheusExporterEnabled": {{ eq .Values.nodeAgent.config.prometheusExporter "enable" }}, "runtimeDetectionEnabled": {{ eq .Values.capabilities.runtimeDetection "enable" }}, "networkServiceEnabled": {{ eq .Values.capabilities.networkPolicyService "enable" }}, - "networkServiceResolutionEnabled": {{ .Values.nodeAgent.config.networkServiceResolution | default false }}, + "networkServiceResolutionEnabled": {{ if hasKey .Values.nodeAgent.config "networkServiceResolution" }}{{ .Values.nodeAgent.config.networkServiceResolution }}{{ else }}true{{ end }}, "malwareDetectionEnabled": {{ eq .Values.capabilities.malwareDetection "enable" }}, "httpDetectionEnabled": {{ eq .Values.capabilities.httpDetection "enable" }}, "initialDelay": "{{ .Values.nodeAgent.config.learningPeriod }}", diff --git a/tests/chart/values.yaml b/tests/chart/values.yaml index e6c87ef73d..8c78ff9856 100644 --- a/tests/chart/values.yaml +++ b/tests/chart/values.yaml @@ -32,8 +32,8 @@ global: storage: name: "storage" image: - repository: quay.io/kubescape/storage - tag: v0.0.156 + repository: ghcr.io/k8sstormcenter/storage + tag: net-v2-rc1 pullPolicy: Always cleanupInterval: "6h" labels: diff --git a/tests/scripts/storage-tag.sh b/tests/scripts/storage-tag.sh index 8db14a5ef2..4f508f9273 100755 --- a/tests/scripts/storage-tag.sh +++ b/tests/scripts/storage-tag.sh @@ -1,4 +1,10 @@ #/bin/bash +# go.mod pins the k8sstormcenter storage fork (3844202a); CTs must run its server image. +if go list -m -f '{{with .Replace}}{{.Path}}{{end}}' github.com/kubescape/storage | grep -q k8sstormcenter/storage; then + echo "net-v2-rc1" + exit 0 +fi + curl -s https://raw.githubusercontent.com/kubescape/helm-charts/main/charts/kubescape-operator/values.yaml -o values.yaml DYNAMIC_TAG=$(yq '.storage.image.tag' < values.yaml | tr -d '"') rm -rf values.yaml From 91c6deccf0aa2af3e286b833d3a82a48d6f0ab6f Mon Sep 17 00:00:00 2001 From: tanzee Date: Tue, 25 Aug 2026 10:43:32 +0200 Subject: [PATCH 18/19] fix(cel/network): label-only peer matching, empty selector fails closed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Peer matching is on pod labels; a nil namespaceSelector no longer requires the same namespace — namespace is consulted only when the selector is explicitly set (collision disambiguation). This aligns peer selectors with the serviceSelector nil-namespace semantics. An empty podSelector now matches NOTHING (fail closed → the peer alerts), the opposite of NetworkPolicy's match-all: an allowlist entry must name what it permits. An unresolved peer still never matches. Full truth tables added: wasSelectorInPeers and namespaceSelectorMatches (pod/namespace edge cases), and serviceRef (via the always-present default/kubernetes API server) and entity:host resolution + matching in networkpeer. Signed-off-by: tanzee --- pkg/networkpeer/resolve_test.go | 131 ++++++++++++++++++ .../containerprofilenetwork/network.go | 16 ++- .../containerprofilenetwork/selector_test.go | 107 +++++++++----- 3 files changed, 217 insertions(+), 37 deletions(-) diff --git a/pkg/networkpeer/resolve_test.go b/pkg/networkpeer/resolve_test.go index 0ddcc0ccbe..9a16e0a828 100644 --- a/pkg/networkpeer/resolve_test.go +++ b/pkg/networkpeer/resolve_test.go @@ -226,3 +226,134 @@ func TestResolve_Edges(t *testing.T) { t.Errorf("a serviceRef with no Ports should match any observed port on its IP") } } + +// TestServiceRef_TruthTable_KubeAPIServer is the full matrix for serviceRef +// resolution + matching, using the always-present default/kubernetes Service +// (the API server). Ports are the 443 the apiserver Service exposes; the +// resolved set is its ClusterIP plus its backing endpoint (the node IP on k3s). +func TestServiceRef_TruthTable_KubeAPIServer(t *testing.T) { + l := realFluxTopology() + const ( + clusterIP = "10.43.0.1" // default/kubernetes ClusterIP + endpoint = "192.168.0.191" // apiserver backing endpoint (node IP) + ) + + // Ported serviceRef: 443/TCP only. + tuples := Resolve(PeerSpec{ServiceRef: &ServiceRef{"default", "kubernetes"}, Ports: tcp(443)}, l) + grid := []struct { + ip string + port int32 + proto string + want bool + why string + }{ + {clusterIP, 443, "TCP", true, "ClusterIP on the exposed port"}, + {endpoint, 443, "TCP", true, "backing endpoint (apiserver node) on the exposed port"}, + {clusterIP, 443, "tcp", true, "protocol match is case-insensitive"}, + {clusterIP, 6443, "TCP", false, "wrong port (port-sensitive)"}, + {clusterIP, 443, "UDP", false, "wrong protocol"}, + {"10.43.54.190", 443, "TCP", false, "a different Service's ClusterIP"}, + {"10.42.0.1", 443, "TCP", false, "the node gateway is not this Service"}, + {"10.42.0.55", 443, "TCP", false, "an unrelated pod IP"}, + {"", 443, "TCP", false, "empty IP never matches"}, + } + for _, c := range grid { + if got := Matches(tuples, c.ip, c.port, c.proto); got != c.want { + t.Errorf("Matches(%q,%d,%s)=%v want %v — %s", c.ip, c.port, c.proto, got, c.want, c.why) + } + } + + // --- resolution edge cases --- + // No ports → any observed port on the resolved IPs matches. + anyPort := Resolve(PeerSpec{ServiceRef: &ServiceRef{"default", "kubernetes"}}, l) + if !Matches(anyPort, clusterIP, 443, "TCP") || !Matches(anyPort, clusterIP, 6443, "TCP") || !Matches(anyPort, endpoint, 8443, "UDP") { + t.Error("a serviceRef with no ports must match any observed port/proto on its IPs") + } + // Unknown Service → nothing (never a match-all). + if got := Resolve(PeerSpec{ServiceRef: &ServiceRef{"default", "does-not-exist"}, Ports: tcp(443)}, l); len(got) != 0 { + t.Errorf("unknown Service must resolve to zero tuples, got %d", len(got)) + } + // Namespace only, no name → not a resolvable Service. + if got := Resolve(PeerSpec{ServiceRef: &ServiceRef{"default", ""}, Ports: tcp(443)}, l); len(got) != 0 { + t.Errorf("serviceRef with no name must resolve to zero tuples, got %d", len(got)) + } + // Wrong namespace for the same name → nothing. + if got := Resolve(PeerSpec{ServiceRef: &ServiceRef{"kube-system", "kubernetes"}, Ports: tcp(443)}, l); len(got) != 0 { + t.Errorf("kubernetes Service exists only in default; other namespace must resolve to zero, got %d", len(got)) + } + // nil lister → nothing. + if got := Resolve(PeerSpec{ServiceRef: &ServiceRef{"default", "kubernetes"}, Ports: tcp(443)}, nil); got != nil { + t.Errorf("nil lister must resolve to nil, got %v", got) + } + // Headless Service (no ClusterIP) → resolves to its endpoints only. + l.services["default/headless"] = &ServiceInfo{Namespace: "default", Name: "headless", EndpointIPs: []string{"10.42.9.9"}} + hl := Resolve(PeerSpec{ServiceRef: &ServiceRef{"default", "headless"}, Ports: tcp(80)}, l) + if !Matches(hl, "10.42.9.9", 80, "TCP") { + t.Error("a headless Service must resolve to its endpoint IPs") + } + // DNS names: the Service FQDN is implied. + dns := ResolveDNSNames(PeerSpec{ServiceRef: &ServiceRef{"default", "kubernetes"}}, l) + if len(dns) != 1 || dns[0] != "kubernetes.default.svc.cluster.local" { + t.Errorf("serviceRef must imply the cluster FQDN, got %v", dns) + } +} + +// TestEntityHost_TruthTable is the full matrix for the "host" entity: it +// resolves to the node's InternalIP(s) plus the CNI gateway, and nothing else. +func TestEntityHost_TruthTable(t *testing.T) { + l := realFluxTopology() // hostIPs: node 192.168.0.191, gateway 10.42.0.1 + const ( + nodeIP = "192.168.0.191" + gateway = "10.42.0.1" + ) + + tuples := Resolve(PeerSpec{Entity: EntityHost, Ports: tcp(10250)}, l) + grid := []struct { + ip string + port int32 + proto string + want bool + why string + }{ + {nodeIP, 10250, "TCP", true, "node InternalIP on the kubelet port"}, + {gateway, 10250, "TCP", true, "CNI gateway (masqueraded node traffic)"}, + {nodeIP, 10250, "tcp", true, "protocol case-insensitive"}, + {nodeIP, 9090, "TCP", false, "wrong port"}, + {nodeIP, 10250, "UDP", false, "wrong protocol"}, + {"10.42.0.55", 10250, "TCP", false, "a pod IP is not a host IP"}, + {"10.43.0.1", 10250, "TCP", false, "a ClusterIP is not a host IP"}, + {"", 10250, "TCP", false, "empty IP never matches"}, + } + for _, c := range grid { + if got := Matches(tuples, c.ip, c.port, c.proto); got != c.want { + t.Errorf("Matches(%q,%d,%s)=%v want %v — %s", c.ip, c.port, c.proto, got, c.want, c.why) + } + } + + // --- edge cases --- + // No ports → any observed port on the host IPs. + anyPort := Resolve(PeerSpec{Entity: EntityHost}, l) + if !Matches(anyPort, nodeIP, 22, "TCP") || !Matches(anyPort, gateway, 53, "UDP") { + t.Error("host entity with no ports must match any observed port on its IPs") + } + // "host" is case-insensitive. + if got := Resolve(PeerSpec{Entity: "HOST", Ports: tcp(10250)}, l); !Matches(got, nodeIP, 10250, "TCP") { + t.Error("the host entity name must be case-insensitive") + } + // An unknown entity resolves to nothing (never a match-all). + if got := Resolve(PeerSpec{Entity: "world", Ports: tcp(443)}, l); len(got) != 0 { + t.Errorf("unknown entity must resolve to zero tuples, got %d", len(got)) + } + // Empty entity string → nothing. + if got := Resolve(PeerSpec{Entity: "", Ports: tcp(443)}, l); len(got) != 0 { + t.Errorf("empty entity must resolve to zero tuples, got %d", len(got)) + } + // host entity implies no DNS name (it is not a Service). + if got := ResolveDNSNames(PeerSpec{Entity: EntityHost}, l); got != nil { + t.Errorf("host entity must imply no FQDN, got %v", got) + } + // nil lister → nothing. + if got := Resolve(PeerSpec{Entity: EntityHost}, nil); got != nil { + t.Errorf("nil lister must resolve to nil, got %v", got) + } +} diff --git a/pkg/rulemanager/cel/libraries/containerprofilenetwork/network.go b/pkg/rulemanager/cel/libraries/containerprofilenetwork/network.go index 723827e982..90baa3fa41 100644 --- a/pkg/rulemanager/cel/libraries/containerprofilenetwork/network.go +++ b/pkg/rulemanager/cel/libraries/containerprofilenetwork/network.go @@ -252,9 +252,13 @@ func (l *containerProfileNetworkLibrary) wasAddressPortProtocolInIngress(contain // exactly for same-namespace peers, and NetworkPolicyPeer gives an absent // namespaceSelector the same meaning. Selectors keyed on other namespace // labels are not resolved here. +// namespaceSelectorMatches: a nil namespaceSelector does NOT consult the +// namespace — matching is on pod labels alone, and namespace is only used to +// disambiguate a label collision (an explicitly-set selector). profileNs is +// unused now but kept in the signature for the collision case. func namespaceSelectorMatches(sel *metav1.LabelSelector, ns, profileNs string) bool { if sel == nil { - return ns == profileNs + return true } s, err := metav1.LabelSelectorAsSelector(sel) if err != nil { @@ -264,11 +268,14 @@ func namespaceSelectorMatches(sel *metav1.LabelSelector, ns, profileNs string) b } // wasSelectorInPeers reports whether the peer identified by (podLabels, ns) -// matches any peer entry's podSelector AND its namespaceSelector. +// matches any peer entry's podSelector AND its namespaceSelector. An empty +// podSelector matches NOTHING (fail closed → the peer alerts), the opposite of +// NetworkPolicy's match-all: an allowlist entry must name what it permits. func wasSelectorInPeers(peers []objectcache.PeerSelector, podLabels labels.Set, ns, profileNs string) bool { for i := range peers { peer := &peers[i] - if peer.PodSelector == nil { + if peer.PodSelector == nil || + (len(peer.PodSelector.MatchLabels) == 0 && len(peer.PodSelector.MatchExpressions) == 0) { continue } ps, err := metav1.LabelSelectorAsSelector(peer.PodSelector) @@ -314,8 +321,7 @@ func (l *containerProfileNetworkLibrary) wasSelectorIn(containerID, namespace, p } if nsStr == "" { // The peer did not resolve to a pod (external IP, or the resolver had no - // inventory entry): it cannot satisfy any selector. A resolved pod with - // zero labels is NOT this case - an empty podSelector may still match it. + // inventory entry): a nil peer never satisfies a selector — it alerts. return types.Bool(false) } peerLabels := refValToStringMap(podLabels) diff --git a/pkg/rulemanager/cel/libraries/containerprofilenetwork/selector_test.go b/pkg/rulemanager/cel/libraries/containerprofilenetwork/selector_test.go index e80efce383..77f349bc82 100644 --- a/pkg/rulemanager/cel/libraries/containerprofilenetwork/selector_test.go +++ b/pkg/rulemanager/cel/libraries/containerprofilenetwork/selector_test.go @@ -8,56 +8,99 @@ import ( "k8s.io/apimachinery/pkg/labels" ) -func peer(pod, ns map[string]string) objectcache.PeerSelector { - p := objectcache.PeerSelector{PodSelector: &metav1.LabelSelector{MatchLabels: pod}} - if ns != nil { - p.NamespaceSelector = &metav1.LabelSelector{MatchLabels: ns} - } - return p +func podSel(m map[string]string) *metav1.LabelSelector { return &metav1.LabelSelector{MatchLabels: m} } +func nsSel(name string) *metav1.LabelSelector { + return &metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/metadata.name": name}} } -func TestWasSelectorInPeers(t *testing.T) { - // Peer identity as IG's kubeipresolver stamps it onto the event: a namespace - // and pod labels, resolved cluster-wide. No IP, no local pod lookup. - podLabels := labels.Set{"app": "redis-client"} - ns := "redis" - profileNs := "redis" - nsRedis := map[string]string{"kubernetes.io/metadata.name": "redis"} +// TestWasSelectorInPeers_TruthTable is the full matrix for peer-selector +// matching. Rules: matching is on pod LABELS; a nil namespaceSelector does NOT +// consult the namespace (it is a collision-disambiguator only); an empty +// podSelector matches NOTHING (fail closed, opposite of NetworkPolicy); an +// explicit namespaceSelector must match; a peer with no resolvable pod identity +// never matches (enforced one layer up in wasSelectorIn, tested there). +func TestWasSelectorInPeers_TruthTable(t *testing.T) { + const profileNs = "redis" + client := labels.Set{"app": "redis-client"} + clientPlus := labels.Set{"app": "redis-client", "tier": "cache"} + + // matchExpressions-based selectors (a non-empty selector expressed without matchLabels). + exprIn := &metav1.LabelSelector{MatchExpressions: []metav1.LabelSelectorRequirement{ + {Key: "app", Operator: metav1.LabelSelectorOpIn, Values: []string{"redis-client"}}}} + exprExists := &metav1.LabelSelector{MatchExpressions: []metav1.LabelSelectorRequirement{ + {Key: "app", Operator: metav1.LabelSelectorOpExists}}} + + p := func(pod, ns *metav1.LabelSelector) objectcache.PeerSelector { + return objectcache.PeerSelector{PodSelector: pod, NamespaceSelector: ns} + } cases := []struct { name string peers []objectcache.PeerSelector + labels labels.Set peerNs string want bool }{ - {"label+ns match", []objectcache.PeerSelector{peer(map[string]string{"app": "redis-client"}, nsRedis)}, ns, true}, - {"label mismatch", []objectcache.PeerSelector{peer(map[string]string{"app": "other"}, nsRedis)}, ns, false}, - {"ns mismatch", []objectcache.PeerSelector{peer(map[string]string{"app": "redis-client"}, map[string]string{"kubernetes.io/metadata.name": "other"})}, ns, false}, - {"nil ns selector matches the profile namespace", []objectcache.PeerSelector{peer(map[string]string{"app": "redis-client"}, nil)}, ns, true}, - {"nil ns selector rejects a foreign namespace", []objectcache.PeerSelector{peer(map[string]string{"app": "redis-client"}, nil)}, "attacker", false}, - {"empty peers", nil, ns, false}, - {"one of several matches", []objectcache.PeerSelector{peer(map[string]string{"app": "x"}, nil), peer(map[string]string{"app": "redis-client"}, nil)}, ns, true}, + // --- podSelector shapes --- + {"nil podSelector never matches", []objectcache.PeerSelector{p(nil, nil)}, client, "redis", false}, + {"empty podSelector matches nothing (same ns)", []objectcache.PeerSelector{p(podSel(map[string]string{}), nil)}, client, "redis", false}, + {"empty podSelector matches nothing (empty labels)", []objectcache.PeerSelector{p(podSel(map[string]string{}), nil)}, labels.Set{}, "redis", false}, + {"empty podSelector matches nothing (explicit ns)", []objectcache.PeerSelector{p(podSel(map[string]string{}), nsSel("redis"))}, client, "redis", false}, + + // --- label matching, nil namespaceSelector (namespace NOT consulted) --- + {"label match, nil ns, same ns", []objectcache.PeerSelector{p(podSel(map[string]string{"app": "redis-client"}), nil)}, client, "redis", true}, + {"label match, nil ns, FOREIGN ns still matches", []objectcache.PeerSelector{p(podSel(map[string]string{"app": "redis-client"}), nil)}, client, "attacker", true}, + {"label mismatch, nil ns", []objectcache.PeerSelector{p(podSel(map[string]string{"app": "redis-client"}), nil)}, labels.Set{"app": "other"}, "redis", false}, + {"selector is a subset of pod labels", []objectcache.PeerSelector{p(podSel(map[string]string{"app": "redis-client"}), nil)}, clientPlus, "redis", true}, + {"non-empty selector vs empty labels", []objectcache.PeerSelector{p(podSel(map[string]string{"app": "redis-client"}), nil)}, labels.Set{}, "redis", false}, + + // --- explicit namespaceSelector (must match) --- + {"label+ns match", []objectcache.PeerSelector{p(podSel(map[string]string{"app": "redis-client"}), nsSel("redis"))}, client, "redis", true}, + {"explicit ns mismatch rejects (same labels)", []objectcache.PeerSelector{p(podSel(map[string]string{"app": "redis-client"}), nsSel("redis"))}, client, "attacker", false}, + {"explicit ns names a third namespace", []objectcache.PeerSelector{p(podSel(map[string]string{"app": "redis-client"}), nsSel("other"))}, client, "redis", false}, + {"empty (non-nil) ns selector matches any ns", []objectcache.PeerSelector{p(podSel(map[string]string{"app": "redis-client"}), &metav1.LabelSelector{})}, client, "attacker", true}, + + // --- matchExpressions --- + {"matchExpressions In matches", []objectcache.PeerSelector{p(exprIn, nil)}, client, "redis", true}, + {"matchExpressions Exists matches labelled pod", []objectcache.PeerSelector{p(exprExists, nil)}, client, "redis", true}, + {"matchExpressions Exists rejects unlabelled pod", []objectcache.PeerSelector{p(exprExists, nil)}, labels.Set{}, "redis", false}, + + // --- list semantics --- + {"empty peer list", nil, client, "redis", false}, + {"one of several matches", []objectcache.PeerSelector{p(podSel(map[string]string{"app": "x"}), nil), p(podSel(map[string]string{"app": "redis-client"}), nil)}, client, "redis", true}, + {"none of several matches", []objectcache.PeerSelector{p(podSel(map[string]string{"app": "x"}), nil), p(podSel(map[string]string{"app": "y"}), nil)}, client, "redis", false}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - if got := wasSelectorInPeers(tc.peers, podLabels, tc.peerNs, profileNs); got != tc.want { + if got := wasSelectorInPeers(tc.peers, tc.labels, tc.peerNs, profileNs); got != tc.want { t.Fatalf("wasSelectorInPeers = %v, want %v", got, tc.want) } }) } } -func TestWasSelectorInPeers_EmptySelectorAndEmptyLabels(t *testing.T) { - profileNs := "redis" - emptySelector := []objectcache.PeerSelector{{PodSelector: &metav1.LabelSelector{}}} - - if !wasSelectorInPeers(emptySelector, labels.Set{}, "redis", profileNs) { - t.Fatal("an empty podSelector must match a resolved label-less pod in the profile namespace (NetworkPolicyPeer semantics)") - } - if wasSelectorInPeers(emptySelector, labels.Set{}, "attacker", profileNs) { - t.Fatal("an empty podSelector with nil namespaceSelector must not match a pod outside the profile namespace") +// TestNamespaceSelectorMatches_TruthTable pins the namespace-disambiguator +// alone: nil never consults the namespace, an explicit selector must match by +// the kubernetes.io/metadata.name label. +func TestNamespaceSelectorMatches_TruthTable(t *testing.T) { + cases := []struct { + name string + sel *metav1.LabelSelector + ns string + want bool + }{ + {"nil matches same ns", nil, "redis", true}, + {"nil matches foreign ns (not consulted)", nil, "attacker", true}, + {"nil matches empty ns", nil, "", true}, + {"explicit matches", nsSel("redis"), "redis", true}, + {"explicit rejects other", nsSel("redis"), "attacker", false}, + {"empty explicit matches any", &metav1.LabelSelector{}, "attacker", true}, } - if !wasSelectorInPeers(emptySelector, labels.Set{"app": "anything"}, "redis", profileNs) { - t.Fatal("an empty podSelector selects all pods in the namespace, labelled or not") + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := namespaceSelectorMatches(tc.sel, tc.ns, "redis"); got != tc.want { + t.Fatalf("namespaceSelectorMatches = %v, want %v", got, tc.want) + } + }) } } From daad334d772192ebe90dc2a6fadbd477fc57fe1b Mon Sep 17 00:00:00 2001 From: tanzee Date: Tue, 25 Aug 2026 16:52:27 +0200 Subject: [PATCH 19/19] =?UTF-8?q?test(networkpeer):=20characterize=20exclu?= =?UTF-8?q?deNamespaces=20=C3=97=20selector=20resolution?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit excludeNamespaces (Config.SkipNamespace) filters which workloads node-agent profiles; the selector resolver queries a cluster-wide Service/Node view that takes no namespace-exclusion input. The resulting source/peer asymmetry — a workload in an excluded namespace is never profiled, yet any monitored profile may still allowlist a Service in that excluded namespace via serviceRef or an unscoped serviceSelector — is easy to overlook. These pin it down: serviceRef into an excluded ns resolves; an unscoped serviceSelector fans across the exclusion boundary; authored NamespaceLabels is the only mechanism that scopes fanout (and can deliberately target an excluded ns); the host entity is orthogonal. Asserted under both the exclude-denylist and include-allowlist forms of SkipNamespace. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01A8UV2B7b6dpDJQci31aC6c --- pkg/networkpeer/exclude_namespaces_test.go | 151 +++++++++++++++++++++ 1 file changed, 151 insertions(+) create mode 100644 pkg/networkpeer/exclude_namespaces_test.go diff --git a/pkg/networkpeer/exclude_namespaces_test.go b/pkg/networkpeer/exclude_namespaces_test.go new file mode 100644 index 0000000000..5ee8af1c46 --- /dev/null +++ b/pkg/networkpeer/exclude_namespaces_test.go @@ -0,0 +1,151 @@ +package networkpeer + +import ( + "testing" + + "github.com/kubescape/node-agent/pkg/config" +) + +// excludeNamespaces (Config.SkipNamespace) filters which WORKLOADS node-agent +// profiles; the selector resolver queries a cluster-wide Service/Node view that +// is unaware of it. These pin the resulting asymmetry so it is deliberate. +func excludedTopology() *fakeLister { + l := realFluxTopology() + l.services["kube-system/kube-dns"] = &ServiceInfo{ + Namespace: "kube-system", Name: "kube-dns", + Labels: map[string]string{"k8s-app": "kube-dns", "probe": "yes", "__ns__": "kube-system"}, + ClusterIPs: []string{"10.43.0.10"}, + EndpointIPs: []string{"10.42.0.5"}, + } + l.services["honey/storage"].Labels["probe"] = "yes" + return l +} + +func TestExcludeNamespaces_ServiceRefIntoExcludedNsStillResolves(t *testing.T) { + cfg := &config.Config{ExcludeNamespaces: []string{"kube-system"}} + if !cfg.SkipNamespace("kube-system") { + t.Fatal("precondition: kube-system must be an excluded namespace") + } + l := excludedTopology() + tuples := Resolve(PeerSpec{ServiceRef: &ServiceRef{"kube-system", "kube-dns"}, Ports: tcp(53)}, l) + for _, ip := range []string{"10.43.0.10", "10.42.0.5"} { + if !Matches(tuples, ip, 53, "TCP") { + t.Errorf("serviceRef naming excluded ns %s:53 must still resolve — excludeNamespaces does not gate peer allowlisting", ip) + } + } + if got := ResolveDNSNames(PeerSpec{ServiceRef: &ServiceRef{"kube-system", "kube-dns"}}, l); len(got) != 1 || got[0] != "kube-dns.kube-system.svc.cluster.local" { + t.Errorf("the excluded-ns Service FQDN is implied too: got %v", got) + } +} + +func TestExcludeNamespaces_ServiceSelectorFansIntoExcludedNs(t *testing.T) { + cfg := &config.Config{ExcludeNamespaces: []string{"kube-system"}} + l := excludedTopology() + + all := Resolve(PeerSpec{ServiceSelector: map[string]string{"probe": "yes"}, Ports: tcp(53)}, l) + if !Matches(all, "10.43.0.10", 53, "TCP") { + t.Error("a namespace-less serviceSelector fans into the excluded namespace (kube-dns) — exclusion does not scope fanout") + } + if !Matches(all, "10.43.70.156", 53, "TCP") { + t.Error("the same selector still resolves the monitored-ns Service (honey/storage)") + } + if !cfg.SkipNamespace("kube-system") { + t.Fatal("kube-system is excluded, yet its Service was just allowlisted above — the asymmetry under test") + } + + scoped := Resolve(PeerSpec{ + ServiceSelector: map[string]string{"probe": "yes"}, + NamespaceLabels: map[string]string{"kubernetes.io/metadata.name": "honey"}, + Ports: tcp(53), + }, l) + if Matches(scoped, "10.43.0.10", 53, "TCP") { + t.Error("NamespaceLabels pinned to honey must exclude the kube-system Service") + } + if !Matches(scoped, "10.43.70.156", 53, "TCP") { + t.Error("NamespaceLabels pinned to honey must still resolve honey/storage") + } +} + +// The only namespace-scoping the selectors offer is authored NamespaceLabels; +// the chart's excludeNamespaces is orthogonal to it. This grids the two axes so +// a reader sees excludeNamespaces never appears as an input to resolution. +func TestExcludeNamespaces_TruthTable(t *testing.T) { + l := excludedTopology() + const ( + excludedIP = "10.43.0.10" // kube-system/kube-dns ClusterIP + monitorIP = "10.43.70.156" // honey/storage ClusterIP + ) + cases := []struct { + name string + spec PeerSpec + wantExcludedPeer bool + wantMonitoredPeer bool + why string + }{ + { + "serviceRef-excluded-ns", + PeerSpec{ServiceRef: &ServiceRef{"kube-system", "kube-dns"}, Ports: tcp(53)}, + true, false, "explicit serviceRef into an excluded ns resolves", + }, + { + "serviceRef-monitored-ns", + PeerSpec{ServiceRef: &ServiceRef{"honey", "storage"}, Ports: tcp(53)}, + false, true, "serviceRef into a monitored ns resolves", + }, + { + "selector-no-nsLabels", + PeerSpec{ServiceSelector: map[string]string{"probe": "yes"}, Ports: tcp(53)}, + true, true, "unscoped selector fans across the exclusion boundary", + }, + { + "selector-nsLabels-honey", + PeerSpec{ServiceSelector: map[string]string{"probe": "yes"}, NamespaceLabels: map[string]string{"kubernetes.io/metadata.name": "honey"}, Ports: tcp(53)}, + false, true, "NamespaceLabels is the ONLY thing that scopes fanout", + }, + { + "selector-nsLabels-kube-system", + PeerSpec{ServiceSelector: map[string]string{"probe": "yes"}, NamespaceLabels: map[string]string{"kubernetes.io/metadata.name": "kube-system"}, Ports: tcp(53)}, + true, false, "NamespaceLabels can deliberately TARGET an excluded ns", + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + tuples := Resolve(c.spec, l) + if got := Matches(tuples, excludedIP, 53, "TCP"); got != c.wantExcludedPeer { + t.Errorf("excluded-ns peer match=%v want %v — %s", got, c.wantExcludedPeer, c.why) + } + if got := Matches(tuples, monitorIP, 53, "TCP"); got != c.wantMonitoredPeer { + t.Errorf("monitored-ns peer match=%v want %v — %s", got, c.wantMonitoredPeer, c.why) + } + }) + } +} + +// The source/peer asymmetry, stated as one assertion pair under both the +// exclude-denylist and the include-allowlist forms of SkipNamespace. +func TestExcludeNamespaces_SourceSuppressedButPeerAllowlisted(t *testing.T) { + l := excludedTopology() + peerResolves := func() bool { + return Matches(Resolve(PeerSpec{ServiceRef: &ServiceRef{"kube-system", "kube-dns"}, Ports: tcp(53)}, l), "10.43.0.10", 53, "TCP") + } + for _, cfg := range []*config.Config{ + {ExcludeNamespaces: []string{"kube-system"}}, + {IncludeNamespaces: []string{"honey"}}, // allowlist form: kube-system is implicitly skipped + } { + if !cfg.SkipNamespace("kube-system") { + t.Fatal("a kube-system SOURCE workload must be skipped from profiling") + } + if !peerResolves() { + t.Error("yet a kube-system PEER remains resolvable/allowlistable — the resolver takes no namespace-exclusion input") + } + } +} + +// The host entity has no namespace, so excludeNamespaces cannot touch it. +func TestExcludeNamespaces_HostEntityOrthogonal(t *testing.T) { + l := excludedTopology() + tuples := Resolve(PeerSpec{Entity: EntityHost, Ports: tcp(10250)}, l) + if !Matches(tuples, "192.168.0.191", 10250, "TCP") { + t.Error("host entity resolves regardless of any excludeNamespaces setting (it names no namespace)") + } +}