diff --git a/.github/workflows/component-tests.yaml b/.github/workflows/component-tests.yaml index f3325394eb..d1c6dfceec 100644 --- a/.github/workflows/component-tests.yaml +++ b/.github/workflows/component-tests.yaml @@ -107,7 +107,9 @@ jobs: Test_36_MultiContainerPerContainerBinding, Test_43_RelativeOpenPathResolution, Test_48_MultiSubtypeGroupedProfileDocument, - Test_49_EphemeralContainerFullTreatment + Test_49_EphemeralContainerFullTreatment, + Test_50_ServiceRefNetworkNeighbor, + Test_51_ServiceRefIngressR0012 ] steps: - name: Checkout code diff --git a/charts/kubescape-rules/Chart.yaml b/charts/kubescape-rules/Chart.yaml new file mode 100644 index 0000000000..4d977a85e1 --- /dev/null +++ b/charts/kubescape-rules/Chart.yaml @@ -0,0 +1,6 @@ +apiVersion: v2 +name: kubescape-rules +description: Kubescape runtime detection rules — symmetric R0011 egress / R0012 ingress, port-aware, selector- and serviceRef-allowlisted internal traffic +type: application +version: 0.1.0 +appVersion: "network-v2" diff --git a/charts/kubescape-rules/templates/binding.yaml b/charts/kubescape-rules/templates/binding.yaml new file mode 100644 index 0000000000..755bd39055 --- /dev/null +++ b/charts/kubescape-rules/templates/binding.yaml @@ -0,0 +1,44 @@ +apiVersion: kubescape.io/v1 +kind: RuntimeRuleAlertBinding +metadata: + name: all-rules-all-pods +spec: + namespaceSelector: + # exclude K8s system namespaces + matchExpressions: + - key: "kubernetes.io/metadata.name" + operator: "NotIn" + values: + - "kube-system" + - "kube-public" + - "kube-node-lease" + - "kubeconfig" + rules: + - ruleName: "Unexpected process launched" + - ruleName: "Unexpected process arguments" + - ruleName: "Files Access Anomalies in container" + - ruleName: "Syscalls Anomalies in container" + - ruleName: "Linux Capabilities Anomalies in container" + - ruleName: "DNS Anomalies in container" + - ruleName: "Unexpected service account token access" + - ruleName: "Workload uses Kubernetes API unexpectedly" + - ruleName: "Process Executed from /dev/shm" + - ruleName: "Process tries to load a kernel module" + - ruleName: "Drifted process executed" + - ruleName: "SSH Connection to Unexpected Destination on Non-Standard Port" + - ruleName: "Fileless execution detected" + - ruleName: "Crypto miner launched" + - ruleName: "Process executed from mount" + - ruleName: "Crypto Mining Related Port Communication" + - ruleName: "Crypto Mining Domain Communication" + - ruleName: "Read Environment Variables from procfs" + - ruleName: "eBPF Program Load" + - ruleName: "Soft link created over sensitive file" + - ruleName: "Unexpected Sensitive File Access" + - ruleName: "Hard link created over sensitive file" + - ruleName: "Exec to pod" + - ruleName: "Port forward to pod" + - ruleName: "Unexpected Egress Network Traffic" + - ruleName: "Unexpected Ingress Network Traffic" + - ruleName: "Unexpected Ptrace Syscall Usage" + - ruleName: "Unexpected io_uring Operation Detected" diff --git a/charts/kubescape-rules/templates/rules.yaml b/charts/kubescape-rules/templates/rules.yaml new file mode 100644 index 0000000000..dbcc2c81be --- /dev/null +++ b/charts/kubescape-rules/templates/rules.yaml @@ -0,0 +1,795 @@ +apiVersion: kubescape.io/v1 +kind: Rules +metadata: + name: kubescape-rules + namespace: {{ .Values.ksNamespace }} + annotations: + kubescape.io/namespace: {{ .Values.ksNamespace }} + labels: + app: kubescape +spec: + rules: + - name: "Unexpected process launched" + enabled: true + id: "R0001" + description: "Detects unexpected process launches that are not in the baseline" + expressions: + message: "'Unexpected process launched: ' + event.comm + ' with PID ' + string(event.pid)" + uniqueId: "event.comm + '_' + event.exepath" + ruleExpression: + - eventType: "exec" + expression: "!cp.was_executed(event.containerId, (event.exepath != \"\" ? event.exepath : parse.get_exec_path(event.args, event.comm)))" + profileDependency: 0 + profileDataRequired: + execs: all + severity: 1 + supportPolicy: false + isTriggerAlert: true + mitreTactic: "TA0002" + mitreTechnique: "T1059" + tags: + - "context:kubernetes" + - "context:container" + - "anomaly" + - "process" + - "exec" + - "applicationprofile" + - name: "Files Access Anomalies in container" + enabled: false + id: "R0002" + description: "Detects unexpected file access that is not in the baseline" + expressions: + message: "'Unexpected file access detected: ' + event.comm + ' with PID ' + string(event.pid) + ' to ' + event.path" + uniqueId: "event.comm + '_' + event.path" + ruleExpression: + - eventType: "open" + expression: > + (event.path.startsWith('/etc/') || + event.path.startsWith('/var/log/') || + event.path.startsWith('/var/run/') || + event.path.startsWith('/run/') || + event.path.startsWith('/var/spool/cron/') || + event.path.startsWith('/var/www/') || + event.path.startsWith('/var/lib/') || + event.path.startsWith('/opt/') || + event.path.startsWith('/usr/local/') || + event.path.startsWith('/app/') || + event.path == '/.dockerenv' || + event.path == '/proc/self/environ') + && + !(event.path.startsWith('/run/secrets/kubernetes.io/serviceaccount') || + event.path.startsWith('/var/run/secrets/kubernetes.io/serviceaccount') || + event.path.startsWith('/tmp')) + && + !cp.was_path_opened(event.containerId, event.path) + profileDependency: 0 + profileDataRequired: + opens: + - prefix: "/etc/" + - prefix: "/var/log/" + - prefix: "/var/run/" + - prefix: "/run/" + - prefix: "/var/spool/cron/" + - prefix: "/var/www/" + - prefix: "/var/lib/" + - prefix: "/opt/" + - prefix: "/usr/local/" + - prefix: "/app/" + - exact: "/.dockerenv" + - exact: "/proc/self/environ" + severity: 1 + supportPolicy: false + isTriggerAlert: false + mitreTactic: "TA0009" + mitreTechnique: "T1005" + tags: + - "context:kubernetes" + - "context:container" + - "anomaly" + - "file" + - "open" + - "applicationprofile" + - name: "Syscalls Anomalies in container" + enabled: true + id: "R0003" + description: "Detects unexpected system calls that are not allowlisted by application profile" + expressions: + message: "'Unexpected system call detected: ' + event.syscallName + ' with PID ' + string(event.pid)" + uniqueId: "event.syscallName" + ruleExpression: + - eventType: "syscall" + expression: "!cp.was_syscall_used(event.containerId, event.syscallName)" + profileDependency: 0 + profileDataRequired: + syscalls: all + severity: 1 + supportPolicy: false + isTriggerAlert: false + mitreTactic: "TA0002" + mitreTechnique: "T1059" + tags: + - "context:kubernetes" + - "context:container" + - "anomaly" + - "syscall" + - "applicationprofile" + - name: "Linux Capabilities Anomalies in container" + enabled: true + id: "R0004" + description: "Detects unexpected capabilities that are not allowlisted by application profile" + expressions: + message: "'Unexpected capability used: ' + event.capName + ' in syscall ' + event.syscallName + ' with PID ' + string(event.pid)" + uniqueId: "event.comm + '_' + event.capName" + ruleExpression: + - eventType: "capabilities" + expression: "!cp.was_capability_used(event.containerId, event.capName)" + profileDependency: 0 + profileDataRequired: + capabilities: all + severity: 1 + supportPolicy: false + isTriggerAlert: false + mitreTactic: "TA0002" + mitreTechnique: "T1059" + tags: + - "context:kubernetes" + - "context:container" + - "anomaly" + - "capabilities" + - "applicationprofile" + - name: "DNS Anomalies in container" + enabled: true + id: "R0005" + description: "Detecting unexpected domain requests that are not allowlisted by application profile." + expressions: + message: "'Unexpected domain communication: ' + event.name + ' from: ' + event.containerName" + uniqueId: "event.comm + '_' + event.name" + ruleExpression: + - eventType: "dns" + expression: "!event.name.endsWith('.svc.cluster.local.') && !cp.is_domain_in_egress(event.containerId, event.name)" + profileDependency: 0 + profileDataRequired: + egressDomains: all + severity: 1 + supportPolicy: false + isTriggerAlert: false + mitreTactic: "TA0011" + mitreTechnique: "T1071.004" + tags: + - "context:kubernetes" + - "context:container" + - "dns" + - "anomaly" + - "networkprofile" + - name: "Unexpected service account token access" + enabled: true + id: "R0006" + description: "Detecting unexpected access to service account token." + expressions: + message: "'Unexpected access to service account token: ' + event.path + ' with flags: ' + event.flags.join(',')" + uniqueId: "event.comm" + ruleExpression: + - eventType: "open" + expression: > + ((event.path.startsWith('/run/secrets/kubernetes.io/serviceaccount') && event.path.endsWith('/token')) || + (event.path.startsWith('/var/run/secrets/kubernetes.io/serviceaccount') && event.path.endsWith('/token')) || + (event.path.startsWith('/run/secrets/eks.amazonaws.com/serviceaccount') && event.path.endsWith('/token')) || + (event.path.startsWith('/var/run/secrets/eks.amazonaws.com/serviceaccount') && event.path.endsWith('/token'))) && + !cp.was_path_opened_with_suffix(event.containerId, '/token') + state: + includePrefixes: + - /run/secrets + - /var/run/secrets + profileDependency: 0 + profileDataRequired: + opens: + - suffix: "/token" + severity: 5 + supportPolicy: false + isTriggerAlert: true + mitreTactic: "TA0006" + mitreTechnique: "T1528" + tags: + - "context:kubernetes" + - "anomaly" + - "serviceaccount" + - "applicationprofile" + - name: "Workload uses Kubernetes API unexpectedly" + enabled: true + id: "R0007" + description: "Detecting execution of kubernetes client" + expressions: + message: "eventType == 'exec' ? 'Kubernetes client (' + event.comm + ') was executed with PID ' + string(event.pid) : 'Network connection to Kubernetes API server from container ' + event.containerName" + uniqueId: "eventType == 'exec' ? 'exec_' + event.comm : 'network_' + event.dstAddr" + ruleExpression: + - eventType: "exec" + expression: "(event.comm == 'kubectl' || event.exepath.endsWith('/kubectl')) && !cp.was_executed(event.containerId, (event.exepath != \"\" ? event.exepath : parse.get_exec_path(event.args, event.comm)))" + - eventType: "network" + expression: "event.pktType == 'OUTGOING' && k8s.is_api_server_address(event.dstAddr) && !cp.was_address_in_egress(event.containerId, event.dstAddr)" + profileDependency: 0 + profileDataRequired: + execs: all + egressAddresses: all + severity: 5 # Medium + supportPolicy: false + isTriggerAlert: false + mitreTactic: "TA0008" + mitreTechnique: "T1210" + tags: + - "context:kubernetes" + - "exec" + - "network" + - "anomaly" + - "applicationprofile" + - name: "Read Environment Variables from procfs" + enabled: true + id: "R0008" + description: "Detecting reading environment variables from procfs." + expressions: + message: "'Reading environment variables from procfs: ' + event.path + ' by process ' + event.comm" + uniqueId: "event.comm" + ruleExpression: + - eventType: "open" + expression: > + event.path.startsWith('/proc/') && + event.path.endsWith('/environ') && + !cp.was_path_opened_with_suffix(event.containerId, '/environ') + state: + includePrefixes: + - /proc + profileDependency: 0 # Required + profileDataRequired: + opens: + - suffix: "/environ" + severity: 5 # Medium + supportPolicy: false + isTriggerAlert: true + mitreTactic: "TA0006" + mitreTechnique: "T1552.001" + tags: + - "context:kubernetes" + - "context:container" + - "anomaly" + - "procfs" + - "environment" + - "applicationprofile" + - name: "eBPF Program Load" + enabled: true + id: "R0009" + description: "Detecting eBPF program load." + expressions: + message: "'bpf program load system call (bpf) was called by process (' + event.comm + ') with command (BPF_PROG_LOAD)'" + uniqueId: "event.comm + '_' + 'bpf' + '_' + string(event.cmd)" + ruleExpression: + - eventType: "bpf" + expression: "event.cmd == uint(5) && !cp.was_syscall_used(event.containerId, 'bpf')" + profileDependency: 1 + profileDataRequired: + syscalls: + - exact: "bpf" + severity: 5 + supportPolicy: false + isTriggerAlert: true + mitreTactic: "TA0005" + mitreTechnique: "T1218" + tags: + - "context:kubernetes" + - "context:host" + - "bpf" + - "ebpf" + - "applicationprofile" + - name: "Unexpected Sensitive File Access" + enabled: true + id: "R0010" + description: "Detecting access to sensitive files." + expressions: + message: "'Unexpected sensitive file access: ' + event.path + ' by process ' + event.comm" + uniqueId: "event.comm + '_' + event.path" + ruleExpression: + - eventType: "open" + expression: "event.path.startsWith('/etc/shadow') && !cp.was_path_opened(event.containerId, event.path)" + profileDependency: 1 + profileDataRequired: + opens: + - prefix: "/etc/shadow" + severity: 5 + supportPolicy: false + isTriggerAlert: true + mitreTactic: "TA0006" + mitreTechnique: "T1005" + tags: + - "context:kubernetes" + - "context:container" + - "context:host" + - "files" + - "anomaly" + - "applicationprofile" + - name: "Unexpected Egress Network Traffic" + enabled: true + id: "R0011" + description: "Detecting unexpected egress network traffic that is not allowlisted by application profile." + expressions: + message: "'Unexpected egress network communication to: ' + event.dstAddr + ':' + string(event.dstPort) + ' using ' + event.proto + ' from: ' + event.containerName" + uniqueId: "event.dstAddr + '_' + string(event.dstPort) + '_' + event.proto" + ruleExpression: + - eventType: "network" + expression: "event.pktType == 'OUTGOING' && !event.dstAddr.startsWith('127.') && event.dstAddr != '::1' && !cp.was_address_port_protocol_in_egress(event.containerId, event.dstAddr, event.dstPort, event.proto) && !cp.was_selector_in_egress(event.containerId, event.dstNamespace, event.dstPodLabels)" + profileDependency: 0 + profileDataRequired: + egressAddresses: all + severity: 5 # Medium + supportPolicy: false + isTriggerAlert: false + mitreTactic: "TA0010" + mitreTechnique: "T1041" + tags: + - "context:kubernetes" + - "context:container" + - "whitelisted" + - "network" + - "anomaly" + - "networkprofile" + - name: "Unexpected Ingress Network Traffic" + enabled: true + id: "R0012" + description: "Detecting unexpected ingress network traffic that is not allowlisted by application profile. Symmetric twin of R0011: internal and external peers alike, only loopback excluded; internal peers are allowlisted narrowly via addresses or resolved serviceRef/serviceSelector entries rather than a serviceCIDR that blinds detection to lateral movement." + expressions: + message: "'Unexpected ingress network communication from: ' + event.dstAddr + ':' + string(event.dstPort) + ' using ' + event.proto + ' to: ' + event.containerName" + uniqueId: "event.dstAddr + '_' + string(event.dstPort) + '_' + event.proto" + ruleExpression: + - eventType: "network" + expression: "event.pktType == 'HOST' && !event.dstAddr.startsWith('127.') && event.dstAddr != '::1' && !cp.was_address_port_protocol_in_ingress(event.containerId, event.dstAddr, event.dstPort, event.proto) && !cp.was_selector_in_ingress(event.containerId, event.dstNamespace, event.dstPodLabels)" + profileDependency: 0 + profileDataRequired: + ingressAddresses: all + severity: 5 # Medium + supportPolicy: false + isTriggerAlert: false + mitreTactic: "TA0008" + mitreTechnique: "T1210" + tags: + - "context:kubernetes" + - "context:container" + - "whitelisted" + - "network" + - "anomaly" + - "networkprofile" + - name: "Unexpected process arguments" + enabled: true + id: "R0040" + description: "Detects an exec event whose path IS in the application profile but whose argv vector does not match any recorded argv pattern for that path. Consumes cp.was_executed_with_args, which walks the ExecsByPath projection surface and delegates argv comparison to dynamicpathdetector.MatchExecArgs (storage). Stays silent when the path is unknown (R0001 covers that case) and when the argv vector matches any recorded pattern (including the trailing zero-or-more form and the single-arg form); a '*' in a recorded arg is a literal character, not a wildcard." + expressions: + message: "'Unexpected process arguments: ' + event.comm + ' with PID ' + string(event.pid) + ' argv=' + event.args.map(a, string(a)).join(' ')" + uniqueId: "event.comm + '_' + event.exepath + '_' + event.args.map(a, string(a)).join(' ')" + ruleExpression: + - eventType: "exec" + expression: "cp.was_executed(event.containerId, parse.get_exec_path(event.args, event.comm, event.exepath)) && !cp.was_executed_with_args(event.containerId, parse.get_exec_path(event.args, event.comm, event.exepath), event.args)" + profileDependency: 0 + profileDataRequired: + execs: all + severity: 3 + supportPolicy: false + isTriggerAlert: true + mitreTactic: "TA0002" + mitreTechnique: "T1059" + tags: + - "context:kubernetes" + - "context:container" + - "anomaly" + - "process" + - "exec" + - "applicationprofile" + - name: "Process Executed from /dev/shm" + enabled: true + id: "R1000" + description: "Detecting exec calls whose executable path or working directory is under /dev/shm, a world-writable memory-backed (tmpfs) directory." + expressions: + message: "'Process executed from /dev/shm: ' + event.exepath + ' in directory ' + event.cwd" + uniqueId: "event.comm + '_' + event.exepath + '_' + event.pcomm" + ruleExpression: + - eventType: "exec" + expression: > + (event.exepath == '/dev/shm' || event.exepath.startsWith('/dev/shm/')) || + (event.cwd == '/dev/shm' || event.cwd.startsWith('/dev/shm/')) || + (event.args.size() > 0 && (event.args[0] == '/dev/shm' || event.args[0].startsWith('/dev/shm/'))) + profileDependency: 2 + severity: 8 + supportPolicy: false + isTriggerAlert: true + mitreTactic: "TA0002" + mitreTechnique: "T1059" + tags: + - "context:kubernetes" + - "context:host" + - "exec" + - "signature" + - "malicious" + - name: "Drifted process executed" + enabled: true + id: "R1001" + description: "Detecting exec calls of binaries that are not included in the base image" + expressions: + message: "'Process (' + event.comm + ') was executed and is not part of the image'" + uniqueId: "event.comm + '_' + event.exepath + '_' + event.pcomm" + ruleExpression: + - eventType: "exec" + expression: > + (event.upperlayer == true || + event.pupperlayer == true) && + !cp.was_executed(event.containerId, (event.exepath != "" ? event.exepath : parse.get_exec_path(event.args, event.comm))) + profileDependency: 1 + profileDataRequired: + execs: all + severity: 8 + supportPolicy: false + isTriggerAlert: true + mitreTactic: "TA0005" + mitreTechnique: "T1036" + tags: + - "context:kubernetes" + - "context:container" + - "exec" + - "malicious" + - "binary" + - "base image" + - "applicationprofile" + - name: "Process tries to load a kernel module" + enabled: true + id: "R1002" + description: "Detecting Kernel Module Load." + expressions: + message: "'Kernel module (' + event.module + ') loading attempt with syscall (' + event.syscallName + ') was called by process (' + event.comm + ')'" + uniqueId: "event.comm + '_' + event.syscallName + '_' + event.module" + ruleExpression: + - eventType: "kmod" + expression: "event.syscallName == 'init_module' || event.syscallName == 'finit_module'" + profileDependency: 2 + severity: 10 + supportPolicy: false + isTriggerAlert: true + mitreTactic: "TA0005" + mitreTechnique: "T1547.006" + tags: + - "context:kubernetes" + - "context:host" + - "kmod" + - "kernel" + - "module" + - "load" + - name: "SSH Connection to Unexpected Destination on Non-Standard Port" + enabled: false + id: "R1003" + description: "Detecting an SSH connection to a non-standard port where the destination address is not in the container's learned egress baseline." + expressions: + message: "'SSH connection to unexpected destination on non-standard port: ' + event.dstIp + ':' + string(dyn(event.dstPort))" + uniqueId: "event.comm + '_' + event.dstIp + '_' + string(dyn(event.dstPort))" + ruleExpression: + - eventType: "ssh" + expression: "dyn(event.srcPort) >= 32768 && dyn(event.srcPort) <= 60999 && !(dyn(event.dstPort) in [22, 2022]) && !cp.was_address_in_egress(event.containerId, event.dstIp)" + profileDependency: 1 + profileDataRequired: + egressAddresses: all + severity: 5 + supportPolicy: false + isTriggerAlert: true + mitreTactic: "TA0008" + mitreTechnique: "T1021.001" + tags: + - "context:kubernetes" + - "context:container" + - "ssh" + - "connection" + - "port" + - "malicious" + - "networkprofile" + - name: "Process executed from mount" + enabled: true + id: "R1004" + description: "Detecting exec calls from mounted paths." + expressions: + message: "'Process (' + event.comm + ') was executed from a mounted path'" + uniqueId: "event.comm" + ruleExpression: + - eventType: "exec" + expression: "!cp.was_executed(event.containerId, (event.exepath != \"\" ? event.exepath : parse.get_exec_path(event.args, event.comm))) && k8s.get_container_mount_paths(event.namespace, event.podName, event.containerName).exists(mount, event.exepath.startsWith(mount) || (event.exepath != \"\" ? event.exepath : parse.get_exec_path(event.args, event.comm)).startsWith(mount))" + profileDependency: 1 + profileDataRequired: + execs: all + severity: 5 + supportPolicy: false + isTriggerAlert: true + mitreTactic: "TA0002" + mitreTechnique: "T1059" + tags: + - "context:kubernetes" + - "context:container" + - "exec" + - "mount" + - "applicationprofile" + - name: "Fileless execution detected" + enabled: true + id: "R1005" + description: "Detecting Fileless Execution" + expressions: + message: '''Fileless execution detected: exec call "'' + event.comm + ''" runs from a memory-backed source (memfd / /proc/self/fd)''' + uniqueId: "event.comm + '_' + event.exepath + '_' + event.pcomm" + ruleExpression: + - eventType: "exec" + expression: "event.exepath.contains('memfd') || event.exepath.startsWith('/proc/self/fd') || event.exepath.matches('/proc/[0-9]+/fd/[0-9]+')" + profileDependency: 2 + severity: 8 + supportPolicy: false + isTriggerAlert: true + mitreTactic: "TA0005" + mitreTechnique: "T1055" + tags: + - "context:kubernetes" + - "context:host" + - "fileless" + - "execution" + - "malicious" + - name: "Unexpected unshare Syscall in Container" + enabled: true + id: "R1006" + description: "Detecting use of the unshare system call (a namespace-manipulation capability that can be used to escape a container) by a non-runc process, where it was not seen in the container's application-profile baseline." + expressions: + message: "'Unshare system call (unshare) was called by process (' + event.comm + ')'" + uniqueId: "event.comm + '_' + 'unshare'" + ruleExpression: + - eventType: "unshare" + expression: "event.pcomm != 'runc' && !cp.was_syscall_used(event.containerId, 'unshare')" + profileDependency: 1 + profileDataRequired: + syscalls: + - exact: "unshare" + severity: 5 + supportPolicy: false + isTriggerAlert: true + mitreTactic: "TA0004" + mitreTechnique: "T1611" + tags: + - "context:kubernetes" + - "context:container" + - "unshare" + - "escape" + - "unshare" + - "anomaly" + - "applicationprofile" + - name: "Crypto miner launched" + enabled: true + id: "R1007" + description: "Detecting XMR Crypto Miners by randomx algorithm usage." + expressions: + message: "'XMR Crypto Miner process: (' + event.exepath + ') executed'" + uniqueId: "event.exepath + '_' + event.comm" + ruleExpression: + - eventType: "randomx" + expression: "true" + profileDependency: 2 + severity: 10 + supportPolicy: false + isTriggerAlert: true + mitreTactic: "TA0040" + mitreTechnique: "T1496" + tags: + - "context:kubernetes" + - "context:container" + - "crypto" + - "miners" + - "malicious" + - name: "Crypto Mining Domain Communication" + enabled: true + id: "R1008" + description: "Detecting Crypto miners communication by domain" + expressions: + message: "'Communication with a known crypto mining domain: ' + event.name" + uniqueId: "event.name + '_' + event.comm" + ruleExpression: + - eventType: "dns" + expression: "event.name in ['2cryptocalc.com.', '2miners.com.', 'antpool.com.', 'asia1.ethpool.org.', 'bohemianpool.com.', 'botbox.dev.', 'btm.antpool.com.', 'c3pool.com.', 'c4pool.org.', 'ca.minexmr.com.', 'cn.stratum.slushpool.com.', 'dash.antpool.com.', 'data.miningpoolstats.stream.', 'de.minexmr.com.', 'eth-ar.dwarfpool.com.', 'eth-asia.dwarfpool.com.', 'eth-asia1.nanopool.org.', 'eth-au.dwarfpool.com.', 'eth-au1.nanopool.org.', 'eth-br.dwarfpool.com.', 'eth-cn.dwarfpool.com.', 'eth-cn2.dwarfpool.com.', 'eth-eu.dwarfpool.com.', 'eth-eu1.nanopool.org.', 'eth-eu2.nanopool.org.', 'eth-hk.dwarfpool.com.', 'eth-jp1.nanopool.org.', 'eth-ru.dwarfpool.com.', 'eth-ru2.dwarfpool.com.', 'eth-sg.dwarfpool.com.', 'eth-us-east1.nanopool.org.', 'eth-us-west1.nanopool.org.', 'eth-us.dwarfpool.com.', 'eth-us2.dwarfpool.com.', 'eth.antpool.com.', 'eu.stratum.slushpool.com.', 'eu1.ethermine.org.', 'eu1.ethpool.org.', 'fastpool.xyz.', 'fr.minexmr.com.', 'kriptokyng.com.', 'mine.moneropool.com.', 'mine.xmrpool.net.', 'miningmadness.com.', 'monero.cedric-crispin.com.', 'monero.crypto-pool.fr.', 'monero.fairhash.org.', 'monero.hashvault.pro.', 'monero.herominers.com.', 'monerod.org.', 'monerohash.com.', 'moneroocean.stream.', 'monerop.com.', 'multi-pools.com.', 'p2pool.io.', 'pool.kryptex.com.', 'pool.minexmr.com.', 'pool.monero.hashvault.pro.', 'pool.rplant.xyz.', 'pool.supportxmr.com.', 'pool.xmr.pt.', 'prohashing.com.', 'rx.unmineable.com.', 'sg.minexmr.com.', 'sg.stratum.slushpool.com.', 'skypool.org.', 'solo-xmr.2miners.com.', 'ss.antpool.com.', 'stratum-btm.antpool.com.', 'stratum-dash.antpool.com.', 'stratum-eth.antpool.com.', 'stratum-ltc.antpool.com.', 'stratum-xmc.antpool.com.', 'stratum-zec.antpool.com.', 'stratum.antpool.com.', 'supportxmr.com.', 'trustpool.cc.', 'us-east.stratum.slushpool.com.', 'us1.ethermine.org.', 'us1.ethpool.org.', 'us2.ethermine.org.', 'us2.ethpool.org.', 'web.xmrpool.eu.', 'www.domajorpool.com.', 'www.dxpool.com.', 'www.mining-dutch.nl.', 'xmc.antpool.com.', 'xmr-asia1.nanopool.org.', 'xmr-au1.nanopool.org.', 'xmr-eu1.nanopool.org.', 'xmr-eu2.nanopool.org.', 'xmr-jp1.nanopool.org.', 'xmr-us-east1.nanopool.org.', 'xmr-us-west1.nanopool.org.', 'xmr.2miners.com.', 'xmr.crypto-pool.fr.', 'xmr.gntl.uk.', 'xmr.nanopool.org.', 'xmr.pool-pay.com.', 'xmr.pool.minergate.com.', 'xmr.solopool.org.', 'xmr.volt-mine.com.', 'xmr.zeropool.io.', 'zec.antpool.com.', 'zergpool.com.', 'auto.c3pool.org.', 'us.monero.herominers.com.', 'xmr.kryptex.network.']" + profileDependency: 2 + severity: 10 + supportPolicy: false + isTriggerAlert: true + mitreTactic: "TA0011" + mitreTechnique: "T1071.004" + tags: + - "context:kubernetes" + - "context:host" + - "network" + - "crypto" + - "miners" + - "malicious" + - "dns" + - name: "Crypto Mining Related Port Communication" + enabled: true + id: "R1009" + description: "Detecting Crypto Miners by suspicious port usage." + expressions: + message: "'Detected crypto mining related port communication on port ' + string(event.dstPort) + ' to ' + event.dstAddr + ' with protocol ' + event.proto" + uniqueId: "event.comm + '_' + string(event.dstPort)" + ruleExpression: + - eventType: "network" + expression: "event.proto == 'TCP' && event.pktType == 'OUTGOING' && event.dstPort in [3333, 45700] && !cp.was_address_in_egress(event.containerId, event.dstAddr)" + state: + ports: + - 3333 + - 45700 + profileDependency: 1 + profileDataRequired: + egressAddresses: all + severity: 3 + supportPolicy: false + isTriggerAlert: false + mitreTactic: "TA0011" + mitreTechnique: "T1071" + tags: + - "context:kubernetes" + - "context:host" + - "network" + - "crypto" + - "miners" + - "malicious" + - "networkprofile" + - name: "Soft link created over sensitive file" + enabled: true + id: "R1010" + description: "Detects symlink creation over sensitive files" + expressions: + message: "'Symlink created over sensitive file: ' + event.oldPath + ' -> ' + event.newPath" + uniqueId: "event.comm + '_' + event.oldPath" + ruleExpression: + - eventType: "symlink" + expression: "(event.oldPath.startsWith('/etc/shadow') || event.oldPath.startsWith('/etc/sudoers')) && !cp.was_path_opened(event.containerId, event.oldPath)" + profileDependency: 1 + profileDataRequired: + opens: + - prefix: "/etc/shadow" + - prefix: "/etc/sudoers" + severity: 5 + supportPolicy: true + isTriggerAlert: true + mitreTactic: "TA0006" + mitreTechnique: "T1005" + tags: + - "context:kubernetes" + - "context:host" + - "anomaly" + - "symlink" + - "applicationprofile" + - name: "ld_preload Mechanism Use or ld.so.preload Modification" + enabled: false + id: "R1011" + description: "Detecting use of the LD_PRELOAD/LD_LIBRARY_PATH dynamic-linker hook mechanism, or an unexpected write to /etc/ld.so.preload relative to the container's application-profile baseline." + expressions: + message: "eventType == 'exec' ? 'Process (' + event.comm + ') is using a dynamic linker hook: ' + process.get_ld_hook_var(event.pid) : 'The dynamic linker configuration file (' + event.path + ') was modified by process (' + event.comm + ')'" + uniqueId: "eventType == 'exec' ? 'exec_' + event.comm : 'open_' + event.path" + ruleExpression: + - eventType: "exec" + expression: "event.comm != 'java' && event.containerName != 'matlab' && process.get_ld_hook_var(event.pid) != ''" + - eventType: "open" + expression: "event.path == '/etc/ld.so.preload' && has(event.flagsRaw) && event.flagsRaw != 0" + profileDependency: 1 + profileDataRequired: + opens: + - exact: "/etc/ld.so.preload" + severity: 5 + supportPolicy: true + isTriggerAlert: true + mitreTactic: "TA0005" + mitreTechnique: "T1574.006" + tags: + - "context:kubernetes" + - "exec" + - "malicious" + - "applicationprofile" + - name: "Hard link created over sensitive file" + enabled: true + id: "R1012" + description: "Detecting hardlink creation over sensitive files." + expressions: + message: "'Hardlink created over sensitive file: ' + event.oldPath + ' - ' + event.newPath" + uniqueId: "event.comm + '_' + event.oldPath" + ruleExpression: + - eventType: "hardlink" + expression: "(event.oldPath.startsWith('/etc/shadow') || event.oldPath.startsWith('/etc/sudoers')) && !cp.was_path_opened(event.containerId, event.oldPath)" + profileDependency: 1 + profileDataRequired: + opens: + - prefix: "/etc/shadow" + - prefix: "/etc/sudoers" + severity: 5 + supportPolicy: true + isTriggerAlert: true + mitreTactic: "TA0006" + mitreTechnique: "T1005" + tags: + - "context:kubernetes" + - "files" + - "malicious" + - "applicationprofile" + - name: "Unexpected Ptrace Syscall Usage" + enabled: true + id: "R1015" + description: "Detecting use of the ptrace syscall that was not seen in the container's application-profile baseline." + expressions: + message: "'Unexpected ptrace syscall usage from: ' + event.comm" + uniqueId: "event.exepath + '_' + event.comm" + ruleExpression: + - eventType: "ptrace" + expression: "true" + profileDependency: 2 + severity: 5 + supportPolicy: false + isTriggerAlert: true + mitreTactic: "TA0005" + mitreTechnique: "T1622" + tags: + - "context:kubernetes" + - "context:host" + - "process" + - "malicious" + - name: "Unexpected io_uring Operation Detected" + enabled: true + id: "R1030" + description: "Detects io_uring operations that were not recorded during the initial observation period, indicating potential unauthorized activity." + expressions: + message: "'Unexpected io_uring operation detected: (opcode=' + string(event.opcode) + ') flags=0x' + (has(event.flagsRaw) ? string(event.flagsRaw) : '0') + ' in ' + event.comm + '.'" + uniqueId: "string(event.opcode) + '_' + event.comm" + ruleExpression: + - eventType: "iouring" + expression: "true" + profileDependency: 0 + profileDataRequired: + syscalls: all + severity: 5 + supportPolicy: true + isTriggerAlert: true + mitreTactic: "TA0002" + mitreTechnique: "T1218" + tags: + - "context:kubernetes" + - "context:container" + - "syscalls" + - "io_uring" + - "applicationprofile" + - name: "Exec to pod" + enabled: true + id: "R2000" + description: "Detects exec operations on pods via the Kubernetes admission webhook (PodExecOptions CONNECT)" + expressions: + message: "'Exec to pod: ' + event.Name + ' in namespace ' + event.Namespace + ' by ' + event.UserInfo.Username" + uniqueId: "event.Namespace + '/' + event.Name" + ruleExpression: + - eventType: "k8s-admission" + expression: 'event.Kind == "PodExecOptions"' + profileDependency: 2 + severity: 8 + supportPolicy: false + isTriggerAlert: true + mitreTactic: "TA0002" + mitreTechnique: "T1609" + tags: + - "context:kubernetes" + - "admission" + - "exec" + - name: "Port forward to pod" + enabled: true + id: "R2001" + description: "Detects port-forward operations on pods via the Kubernetes admission webhook (PodPortForwardOptions CONNECT)" + expressions: + message: "'Port forward to pod: ' + event.Name + ' in namespace ' + event.Namespace + ' by ' + event.UserInfo.Username" + uniqueId: "event.Namespace + '/' + event.Name" + ruleExpression: + - eventType: "k8s-admission" + expression: 'event.Kind == "PodPortForwardOptions"' + profileDependency: 2 + severity: 5 + supportPolicy: false + isTriggerAlert: true + mitreTactic: "TA0011" + mitreTechnique: "T1090" + tags: + - "context:kubernetes" + - "admission" + - "network" diff --git a/charts/kubescape-rules/values.yaml b/charts/kubescape-rules/values.yaml new file mode 100644 index 0000000000..53e2957cc4 --- /dev/null +++ b/charts/kubescape-rules/values.yaml @@ -0,0 +1 @@ +ksNamespace: kubescape diff --git a/cmd/main.go b/cmd/main.go index e41b256727..bfec59e977 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -39,11 +39,13 @@ import ( "github.com/kubescape/node-agent/pkg/malwaremanager" malwaremanagerv1 "github.com/kubescape/node-agent/pkg/malwaremanager/v1" otelmetrics "github.com/kubescape/node-agent/pkg/metricsmanager/otel" + "github.com/kubescape/node-agent/pkg/networkpeer" "github.com/kubescape/node-agent/pkg/networkstream" networkstreamv1 "github.com/kubescape/node-agent/pkg/networkstream/v1" "github.com/kubescape/node-agent/pkg/nodeprofilemanager" nodeprofilemanagerv1 "github.com/kubescape/node-agent/pkg/nodeprofilemanager/v1" "github.com/kubescape/node-agent/pkg/objectcache" + "github.com/kubescape/node-agent/pkg/objectcache/containerprofilecache" "github.com/kubescape/node-agent/pkg/objectcache/dnscache" "github.com/kubescape/node-agent/pkg/objectcache/k8scache" @@ -72,6 +74,8 @@ import ( "github.com/kubescape/node-agent/pkg/watcher/seccompprofilewatcher" goruntime "go.opentelemetry.io/contrib/instrumentation/runtime" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/informers" + toolscache "k8s.io/client-go/tools/cache" ) func main() { @@ -321,6 +325,60 @@ func main() { ruleBindingCache.AddNotifier(&ruleBindingNotify) cpc := containerprofilecache.NewContainerProfileCache(cfg, storageClient, k8sObjectCache, metricsProvider) + // Resolve serviceRef/serviceSelector/entity network neighbors against live + // cluster state (Service ClusterIPs + endpoints, Node IPs + CNI gateway) at + // projection time. Gated behind networkServiceResolutionEnabled: the + // cluster-wide Service+EndpointSlice list+watch (one per DaemonSet node) is + // only paid where profiles actually use the feature. Services and + // EndpointSlices are watched cluster-wide (a profile may reference any + // namespace's Service); the Node informer is field-selected to this agent's + // own node — the "host" entity is local, and a cluster-wide Node watch on + // every DaemonSet pod is O(nodes^2) traffic for no benefit. A TransformFunc + // strips managedFields/annotations (and per-endpoint fields beyond + // Addresses) before objects enter the cache to keep its footprint small. + if cfg.EnableNetworkServiceResolution { + svcInformers := informers.NewSharedInformerFactory(k8sClient.GetKubernetesClient(), 0) + nodeInformers := informers.NewSharedInformerFactoryWithOptions( + k8sClient.GetKubernetesClient(), 0, + informers.WithTweakListOptions(func(o *metav1.ListOptions) { + o.FieldSelector = "metadata.name=" + cfg.NodeName + }), + ) + svcInformer := svcInformers.Core().V1().Services().Informer() + sliceInformer := svcInformers.Discovery().V1().EndpointSlices().Informer() + _ = svcInformer.SetTransform(networkpeer.TrimService) + _ = sliceInformer.SetTransform(networkpeer.TrimEndpointSlice) + serviceLister := networkpeer.NewInformerLister( + svcInformers.Core().V1().Services().Lister(), + svcInformers.Discovery().V1().EndpointSlices().Lister(), + nodeInformers.Core().V1().Nodes().Lister(), + cfg.NodeName, + ) + // Advance the lister generation on any Service/EndpointSlice/Node + // change, so the reconciler re-projects serviceRef/entity profiles when + // the cluster view moves (endpoint churn, or caches that fill after + // startup). Per-event cost is a single atomic increment; the + // re-projection itself is coalesced onto the reconcile tick, and only + // serviceRef-using profiles are eligible. + bump := toolscache.ResourceEventHandlerFuncs{ + AddFunc: func(interface{}) { serviceLister.Bump() }, + UpdateFunc: func(_, _ interface{}) { serviceLister.Bump() }, + DeleteFunc: func(interface{}) { serviceLister.Bump() }, + } + _, _ = svcInformer.AddEventHandler(bump) + _, _ = sliceInformer.AddEventHandler(bump) + _, _ = nodeInformers.Core().V1().Nodes().Informer().AddEventHandler(bump) + // Start the informers and hand the lister over WITHOUT blocking on + // cache sync: node-agent's core startup (container watcher, profiling) + // must not wait on these, and a bounded wait here previously delayed + // learning enough to trip the tight completion budget of Test_22. The + // caches fill in the background; serviceRef/entity neighbors resolve on + // the next reconcile once populated. nil-until-set is a no-op in + // projection. + svcInformers.Start(ctx.Done()) + nodeInformers.Start(ctx.Done()) + cpc.SetServiceLister(serviceLister) + } cpc.Start(ctx) if cpm, ok := containerProfileManager.(*containerprofilemanagerv1.ContainerProfileManager); ok { cpm.SetCompletionNotifier(cpc) diff --git a/go.mod b/go.mod index 808f11127a..13e350ea74 100644 --- a/go.mod +++ b/go.mod @@ -35,7 +35,7 @@ require ( github.com/kubescape/backend v0.0.39 github.com/kubescape/go-logger v0.0.32 github.com/kubescape/k8s-interface v0.0.214 - github.com/kubescape/storage v0.0.303 + github.com/kubescape/storage v0.0.258 github.com/kubescape/workerpool v0.0.0-20250526074519-0e4a4e7f44cf github.com/moby/sys/mountinfo v0.7.2 github.com/oleiade/lane/v2 v2.0.0 @@ -479,3 +479,5 @@ replace github.com/anchore/syft => github.com/kubescape/syft v1.32.0-ks.2 replace github.com/anchore/stereoscope => github.com/anchore/stereoscope v0.1.9 replace github.com/opencontainers/runtime-spec => github.com/opencontainers/runtime-spec v1.2.1 + +replace github.com/kubescape/storage => github.com/k8sstormcenter/storage v0.0.240-0.20260824140105-3844202af06c diff --git a/go.sum b/go.sum index b239690414..5c627bb87e 100644 --- a/go.sum +++ b/go.sum @@ -859,6 +859,8 @@ github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHm github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/k8sstormcenter/storage v0.0.240-0.20260824140105-3844202af06c h1:UWyIu2P3eDT4VUwxDkphPFKYGo2BfR7GkNGw7Nh/LiA= +github.com/k8sstormcenter/storage v0.0.240-0.20260824140105-3844202af06c/go.mod h1:d/1hqWPda2clsjx2wmQgysnB5dThIo3rDKP7RWx+v+M= github.com/kastenhq/goversion v0.0.0-20230811215019-93b2f8823953 h1:WdAeg/imY2JFPc/9CST4bZ80nNJbiBFCAdSZCSgrS5Y= github.com/kastenhq/goversion v0.0.0-20230811215019-93b2f8823953/go.mod h1:6o+UrvuZWc4UTyBhQf0LGjW9Ld7qJxLz/OqvSOWWlEc= github.com/kevinburke/ssh_config v1.2.0 h1:x584FjTGwHzMwvHx18PXxbBVzfnxogHaAReU4gf13a4= @@ -893,8 +895,6 @@ github.com/kubescape/go-logger v0.0.32 h1:4mI+XJOV8VFCMewrEE9VIFEIOhzXokYT3nFpNf github.com/kubescape/go-logger v0.0.32/go.mod h1:Alj7JBQ8/WCxbXe8Ura6ZheSRK45E0p21M3xeqedX90= github.com/kubescape/k8s-interface v0.0.214 h1:j7KP0/5VvYOoQdBGV2+gRM3qnR8PWLAGF8RM/k/DmJ0= github.com/kubescape/k8s-interface v0.0.214/go.mod h1:WNYUG93aZ5kDmuaRKFLtVhp18Yc6EfaHdD1gLYtVTN4= -github.com/kubescape/storage v0.0.303 h1:0nXI6E07lbWsg7iEH04vR4kwiekj//uCQl/La+8j4aM= -github.com/kubescape/storage v0.0.303/go.mod h1:d/1hqWPda2clsjx2wmQgysnB5dThIo3rDKP7RWx+v+M= github.com/kubescape/syft v1.32.0-ks.2 h1:xdUksUmKEyyVKsTfJDYW8Z5HawVJtelsUolPOsWtDx0= github.com/kubescape/syft v1.32.0-ks.2/go.mod h1:E6Kd4iBM2ljUOUQvSt7hVK6vBwaHkMXwcvBZmGMSY5o= github.com/kubescape/workerpool v0.0.0-20250526074519-0e4a4e7f44cf h1:hI0jVwrB6fT4GJWvuUjzObfci1CUknrZdRHfnRVtKM0= diff --git a/pkg/config/config.go b/pkg/config/config.go index cec9f41ab6..61094f8318 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -83,6 +83,7 @@ type Config struct { EnableMalwareDetection bool `mapstructure:"malwareDetectionEnabled"` EnableNetworkStreaming bool `mapstructure:"networkStreamingEnabled"` EnableNetworkTracing bool `mapstructure:"networkServiceEnabled"` + EnableNetworkServiceResolution bool `mapstructure:"networkServiceResolutionEnabled"` EnableNodeProfile bool `mapstructure:"nodeProfileServiceEnabled"` EnablePartialProfileGeneration bool `mapstructure:"partialProfileGenerationEnabled"` EnableMetricsExporter bool `mapstructure:"prometheusExporterEnabled"` @@ -185,6 +186,7 @@ func LoadConfigOptional(path string, errNotFound bool) (Config, error) { viper.SetDefault("podName", os.Getenv(PodNameEnvVar)) viper.SetDefault("fimEnabled", false) viper.SetDefault("networkStreamingEnabled", false) + viper.SetDefault("networkServiceResolutionEnabled", true) viper.SetDefault("kubernetesMode", true) viper.SetDefault("networkStreamingInterval", 2*time.Minute) viper.SetDefault("workerPoolSize", 3000) diff --git a/pkg/config/config_test.go b/pkg/config/config_test.go index 754b342279..7624e5e2ce 100644 --- a/pkg/config/config_test.go +++ b/pkg/config/config_test.go @@ -36,6 +36,7 @@ func TestLoadConfig(t *testing.T) { EnableHttpDetection: false, EnableFIM: true, EnableNetworkStreaming: false, + EnableNetworkServiceResolution: true, EnableEmbeddedSboms: false, EnableHostSensor: true, HostSensorInterval: 1 * time.Minute, diff --git a/pkg/networkpeer/exclude_namespaces_test.go b/pkg/networkpeer/exclude_namespaces_test.go new file mode 100644 index 0000000000..5ee8af1c46 --- /dev/null +++ b/pkg/networkpeer/exclude_namespaces_test.go @@ -0,0 +1,151 @@ +package networkpeer + +import ( + "testing" + + "github.com/kubescape/node-agent/pkg/config" +) + +// excludeNamespaces (Config.SkipNamespace) filters which WORKLOADS node-agent +// profiles; the selector resolver queries a cluster-wide Service/Node view that +// is unaware of it. These pin the resulting asymmetry so it is deliberate. +func excludedTopology() *fakeLister { + l := realFluxTopology() + l.services["kube-system/kube-dns"] = &ServiceInfo{ + Namespace: "kube-system", Name: "kube-dns", + Labels: map[string]string{"k8s-app": "kube-dns", "probe": "yes", "__ns__": "kube-system"}, + ClusterIPs: []string{"10.43.0.10"}, + EndpointIPs: []string{"10.42.0.5"}, + } + l.services["honey/storage"].Labels["probe"] = "yes" + return l +} + +func TestExcludeNamespaces_ServiceRefIntoExcludedNsStillResolves(t *testing.T) { + cfg := &config.Config{ExcludeNamespaces: []string{"kube-system"}} + if !cfg.SkipNamespace("kube-system") { + t.Fatal("precondition: kube-system must be an excluded namespace") + } + l := excludedTopology() + tuples := Resolve(PeerSpec{ServiceRef: &ServiceRef{"kube-system", "kube-dns"}, Ports: tcp(53)}, l) + for _, ip := range []string{"10.43.0.10", "10.42.0.5"} { + if !Matches(tuples, ip, 53, "TCP") { + t.Errorf("serviceRef naming excluded ns %s:53 must still resolve — excludeNamespaces does not gate peer allowlisting", ip) + } + } + if got := ResolveDNSNames(PeerSpec{ServiceRef: &ServiceRef{"kube-system", "kube-dns"}}, l); len(got) != 1 || got[0] != "kube-dns.kube-system.svc.cluster.local" { + t.Errorf("the excluded-ns Service FQDN is implied too: got %v", got) + } +} + +func TestExcludeNamespaces_ServiceSelectorFansIntoExcludedNs(t *testing.T) { + cfg := &config.Config{ExcludeNamespaces: []string{"kube-system"}} + l := excludedTopology() + + all := Resolve(PeerSpec{ServiceSelector: map[string]string{"probe": "yes"}, Ports: tcp(53)}, l) + if !Matches(all, "10.43.0.10", 53, "TCP") { + t.Error("a namespace-less serviceSelector fans into the excluded namespace (kube-dns) — exclusion does not scope fanout") + } + if !Matches(all, "10.43.70.156", 53, "TCP") { + t.Error("the same selector still resolves the monitored-ns Service (honey/storage)") + } + if !cfg.SkipNamespace("kube-system") { + t.Fatal("kube-system is excluded, yet its Service was just allowlisted above — the asymmetry under test") + } + + scoped := Resolve(PeerSpec{ + ServiceSelector: map[string]string{"probe": "yes"}, + NamespaceLabels: map[string]string{"kubernetes.io/metadata.name": "honey"}, + Ports: tcp(53), + }, l) + if Matches(scoped, "10.43.0.10", 53, "TCP") { + t.Error("NamespaceLabels pinned to honey must exclude the kube-system Service") + } + if !Matches(scoped, "10.43.70.156", 53, "TCP") { + t.Error("NamespaceLabels pinned to honey must still resolve honey/storage") + } +} + +// The only namespace-scoping the selectors offer is authored NamespaceLabels; +// the chart's excludeNamespaces is orthogonal to it. This grids the two axes so +// a reader sees excludeNamespaces never appears as an input to resolution. +func TestExcludeNamespaces_TruthTable(t *testing.T) { + l := excludedTopology() + const ( + excludedIP = "10.43.0.10" // kube-system/kube-dns ClusterIP + monitorIP = "10.43.70.156" // honey/storage ClusterIP + ) + cases := []struct { + name string + spec PeerSpec + wantExcludedPeer bool + wantMonitoredPeer bool + why string + }{ + { + "serviceRef-excluded-ns", + PeerSpec{ServiceRef: &ServiceRef{"kube-system", "kube-dns"}, Ports: tcp(53)}, + true, false, "explicit serviceRef into an excluded ns resolves", + }, + { + "serviceRef-monitored-ns", + PeerSpec{ServiceRef: &ServiceRef{"honey", "storage"}, Ports: tcp(53)}, + false, true, "serviceRef into a monitored ns resolves", + }, + { + "selector-no-nsLabels", + PeerSpec{ServiceSelector: map[string]string{"probe": "yes"}, Ports: tcp(53)}, + true, true, "unscoped selector fans across the exclusion boundary", + }, + { + "selector-nsLabels-honey", + PeerSpec{ServiceSelector: map[string]string{"probe": "yes"}, NamespaceLabels: map[string]string{"kubernetes.io/metadata.name": "honey"}, Ports: tcp(53)}, + false, true, "NamespaceLabels is the ONLY thing that scopes fanout", + }, + { + "selector-nsLabels-kube-system", + PeerSpec{ServiceSelector: map[string]string{"probe": "yes"}, NamespaceLabels: map[string]string{"kubernetes.io/metadata.name": "kube-system"}, Ports: tcp(53)}, + true, false, "NamespaceLabels can deliberately TARGET an excluded ns", + }, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + tuples := Resolve(c.spec, l) + if got := Matches(tuples, excludedIP, 53, "TCP"); got != c.wantExcludedPeer { + t.Errorf("excluded-ns peer match=%v want %v — %s", got, c.wantExcludedPeer, c.why) + } + if got := Matches(tuples, monitorIP, 53, "TCP"); got != c.wantMonitoredPeer { + t.Errorf("monitored-ns peer match=%v want %v — %s", got, c.wantMonitoredPeer, c.why) + } + }) + } +} + +// The source/peer asymmetry, stated as one assertion pair under both the +// exclude-denylist and the include-allowlist forms of SkipNamespace. +func TestExcludeNamespaces_SourceSuppressedButPeerAllowlisted(t *testing.T) { + l := excludedTopology() + peerResolves := func() bool { + return Matches(Resolve(PeerSpec{ServiceRef: &ServiceRef{"kube-system", "kube-dns"}, Ports: tcp(53)}, l), "10.43.0.10", 53, "TCP") + } + for _, cfg := range []*config.Config{ + {ExcludeNamespaces: []string{"kube-system"}}, + {IncludeNamespaces: []string{"honey"}}, // allowlist form: kube-system is implicitly skipped + } { + if !cfg.SkipNamespace("kube-system") { + t.Fatal("a kube-system SOURCE workload must be skipped from profiling") + } + if !peerResolves() { + t.Error("yet a kube-system PEER remains resolvable/allowlistable — the resolver takes no namespace-exclusion input") + } + } +} + +// The host entity has no namespace, so excludeNamespaces cannot touch it. +func TestExcludeNamespaces_HostEntityOrthogonal(t *testing.T) { + l := excludedTopology() + tuples := Resolve(PeerSpec{Entity: EntityHost, Ports: tcp(10250)}, l) + if !Matches(tuples, "192.168.0.191", 10250, "TCP") { + t.Error("host entity resolves regardless of any excludeNamespaces setting (it names no namespace)") + } +} diff --git a/pkg/networkpeer/expand.go b/pkg/networkpeer/expand.go new file mode 100644 index 0000000000..fb4afea23d --- /dev/null +++ b/pkg/networkpeer/expand.go @@ -0,0 +1,157 @@ +package networkpeer + +import ( + "strings" + + "github.com/kubescape/storage/pkg/apis/softwarecomposition/v1beta1" +) + +// ExpandServiceNeighbors resolves every serviceRef / serviceSelector / entity +// neighbor in the list against the cluster view and returns equivalent +// synthesized ipAddresses neighbors (one per source neighbor, carrying the +// resolved IPs and the source neighbor's own ports). +// +// The synthesized neighbors are ordinary selector-free ipAddresses entries +// (plus, for Service-backed specs, the Service's cluster FQDN as a dnsName so a +// client dialling it by name is allowlisted too), so the existing +// port-sensitive address matcher and DNS matcher handle them with no further +// change — a serviceRef/host neighbor becomes exactly the narrow, resolved +// entry it stands for. Neighbors that resolve to nothing (unknown +// Service, selector matching nothing, unknown entity) contribute nothing — +// never a match-all. Callers append the result to the same direction (egress +// or ingress) before projecting the profile. +func ExpandServiceNeighbors(neighbors []v1beta1.NetworkNeighbor, l Lister) []v1beta1.NetworkNeighbor { + if l == nil { + return nil + } + var out []v1beta1.NetworkNeighbor + for i := range neighbors { + n := &neighbors[i] + spec, ok := specFromNeighbor(n) + if !ok { + continue + } + ips := ResolveIPs(spec, l) + dnsNames := ResolveDNSNames(spec, l) + if len(ips) == 0 && len(dnsNames) == 0 { + continue + } + out = append(out, v1beta1.NetworkNeighbor{ + Identifier: n.Identifier + "-resolved", + Type: n.Type, + IPAddresses: ips, + DNSNames: dnsNames, + Ports: n.Ports, + }) + } + return out +} + +// WithResolvedServiceNeighbors returns cp with every serviceRef/serviceSelector/ +// entity neighbor expanded into equivalent selector-free ipAddresses neighbors +// (appended to the same direction), so the projection's existing address +// surface enforces them. It is a no-op — returning cp unchanged — when the +// lister is nil or nothing resolves, and it never mutates the input: a copy is +// made only when there is something to add. Call it immediately before +// projecting a ContainerProfile. +func WithResolvedServiceNeighbors(cp *v1beta1.ContainerProfile, l Lister) *v1beta1.ContainerProfile { + if cp == nil || l == nil { + return cp + } + egExtra := ExpandServiceNeighbors(cp.Spec.Egress, l) + inExtra := ExpandServiceNeighbors(cp.Spec.Ingress, l) + if len(egExtra) == 0 && len(inExtra) == 0 { + return cp + } + out := cp.DeepCopy() + out.Spec.Egress = append(out.Spec.Egress, egExtra...) + out.Spec.Ingress = append(out.Spec.Ingress, inExtra...) + return out +} + +// HasServiceNeighbors reports whether any egress/ingress neighbor declares a +// serviceRef / serviceSelector / entity — i.e. whether this profile's +// projection depends on the live cluster view (Service/EndpointSlice/Node) and +// must be re-projected when that view changes. Keyed on the raw fields, not on +// whether they currently resolve, so a profile projected before the informers +// synced is still marked and re-projects once they do. +func HasServiceNeighbors(cp *v1beta1.ContainerProfile) bool { + if cp == nil { + return false + } + for i := range cp.Spec.Egress { + if hasServiceFields(&cp.Spec.Egress[i]) { + return true + } + } + for i := range cp.Spec.Ingress { + if hasServiceFields(&cp.Spec.Ingress[i]) { + return true + } + } + return false +} + +// Must mirror specFromNeighbor's gate: ServiceRefNamespace alone is not a serviceRef. +func hasServiceFields(n *v1beta1.NetworkNeighbor) bool { + return n.ServiceRefName != "" || n.ServiceSelector != nil || n.Entity != "" +} + +// specFromNeighbor extracts a PeerSpec from a NetworkNeighbor, reporting false +// if the neighbor declares none of the service/entity selectors (a plain +// ipAddresses / dnsNames / podSelector neighbor is left untouched). +func specFromNeighbor(n *v1beta1.NetworkNeighbor) (PeerSpec, bool) { + // Cheap-reject a plain ipAddresses/dnsNames neighbor before allocating a + // []PortProto it would only discard (hot on every projection's non-service + // neighbors). + if n.Entity == "" && n.ServiceRefName == "" && n.ServiceSelector == nil { + return PeerSpec{}, false + } + spec := PeerSpec{Ports: portsFromNeighbor(n.Ports)} + switch { + case n.Entity != "": + spec.Entity = n.Entity + case n.ServiceRefName != "": + spec.ServiceRef = &ServiceRef{Namespace: n.ServiceRefNamespace, Name: n.ServiceRefName} + case n.ServiceSelector != nil: + // Only equality (matchLabels) is honored. A MatchExpressions clause or + // an empty matchLabels would either be silently ignored (broadening the + // match) or resolve to every Service — fail closed instead. + if len(n.ServiceSelector.MatchExpressions) > 0 || len(n.ServiceSelector.MatchLabels) == 0 { + return PeerSpec{}, false + } + spec.ServiceSelector = n.ServiceSelector.MatchLabels + // A namespaceSelector is honored only as the single equality + // kubernetes.io/metadata.name= (the only key the lister scopes on). + // Any other form — MatchExpressions, extra keys, or a different key — + // would be silently dropped and broaden the match cluster-wide, so fail + // closed. A nil namespaceSelector is cluster-wide by design. + if n.NamespaceSelector != nil { + nsl := n.NamespaceSelector + if len(nsl.MatchExpressions) > 0 || len(nsl.MatchLabels) != 1 || + nsl.MatchLabels["kubernetes.io/metadata.name"] == "" { + return PeerSpec{}, false + } + spec.NamespaceLabels = nsl.MatchLabels + } + default: + return PeerSpec{}, false + } + return spec, true +} + +func portsFromNeighbor(ports []v1beta1.NetworkPort) []PortProto { + if len(ports) == 0 { + return nil + } + out := make([]PortProto, 0, len(ports)) + for i := range ports { + p := &ports[i] + var port int32 + if p.Port != nil { + port = *p.Port + } + out = append(out, PortProto{Port: port, Protocol: strings.ToUpper(string(p.Protocol))}) + } + return out +} diff --git a/pkg/networkpeer/expand_test.go b/pkg/networkpeer/expand_test.go new file mode 100644 index 0000000000..be7680c59d --- /dev/null +++ b/pkg/networkpeer/expand_test.go @@ -0,0 +1,222 @@ +package networkpeer + +import ( + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/kubescape/storage/pkg/apis/softwarecomposition/v1beta1" +) + +func port(name string, p int32) v1beta1.NetworkPort { + return v1beta1.NetworkPort{Name: name, Protocol: "TCP", Port: &p} +} + +// TestExpandServiceNeighbors_Egress: a serviceRef (alertmanager) + a host +// entity neighbor expand into selector-free ipAddresses neighbors carrying the +// resolved IPs and the original ports; a plain ipAddresses neighbor and an +// unresolvable serviceRef contribute nothing. +func TestExpandServiceNeighbors_Egress(t *testing.T) { + l := realFluxTopology() + in := []v1beta1.NetworkNeighbor{ + {Identifier: "am", Type: "internal", ServiceRefNamespace: "honey", ServiceRefName: "alertmanager", Ports: []v1beta1.NetworkPort{port("TCP-9093", 9093)}}, + {Identifier: "plain", Type: "internal", IPAddresses: []string{"10.43.0.0/16"}, Ports: []v1beta1.NetworkPort{port("TCP-443", 443)}}, + {Identifier: "ghost", Type: "internal", ServiceRefNamespace: "honey", ServiceRefName: "missing", Ports: []v1beta1.NetworkPort{port("TCP-1", 1)}}, + } + out := ExpandServiceNeighbors(in, l) + + if len(out) != 1 { + t.Fatalf("expected 1 synthesized neighbor (alertmanager only), got %d", len(out)) + } + got := out[0] + if got.Identifier != "am-resolved" { + t.Errorf("identifier: got %q", got.Identifier) + } + // ClusterIP + both endpoints, port carried over. + wantIPs := map[string]bool{"10.43.54.190": false, "10.42.0.44": false, "10.42.0.84": false} + for _, ip := range got.IPAddresses { + if _, ok := wantIPs[ip]; !ok { + t.Errorf("unexpected resolved IP %s", ip) + } + wantIPs[ip] = true + } + for ip, seen := range wantIPs { + if !seen { + t.Errorf("missing resolved IP %s", ip) + } + } + if len(got.Ports) != 1 || got.Ports[0].Port == nil || *got.Ports[0].Port != 9093 { + t.Errorf("ports not carried over: %+v", got.Ports) + } + // serviceRef implies the Service cluster FQDN as a dnsName (R0005 suppression). + if len(got.DNSNames) != 1 || got.DNSNames[0] != "alertmanager.honey.svc.cluster.local" { + t.Errorf("serviceRef FQDN not emitted: %v", got.DNSNames) + } +} + +// TestExpandServiceNeighbors_HostEntity: fromEntity host resolves to node + +// gateway IPs on the health port. +func TestExpandServiceNeighbors_HostEntity(t *testing.T) { + l := realFluxTopology() + in := []v1beta1.NetworkNeighbor{ + {Identifier: "probes", Type: "internal", Entity: "host", Ports: []v1beta1.NetworkPort{port("TCP-9440", 9440)}}, + } + out := ExpandServiceNeighbors(in, l) + if len(out) != 1 { + t.Fatalf("expected 1 synthesized host neighbor, got %d", len(out)) + } + // host entity is not a Service: no FQDN. + if len(out[0].DNSNames) != 0 { + t.Errorf("host entity must not emit a dnsName: %v", out[0].DNSNames) + } + // Feed the synthesized entry through the address matcher the same way the + // projection would, to prove end-to-end intent. + tuples := Resolve(PeerSpec{Entity: "host", Ports: []PortProto{{Port: 9440, Protocol: "TCP"}}}, l) + if !Matches(tuples, "10.42.0.1", 9440, "TCP") { + t.Errorf("gateway kubelet probe should match") + } + if Matches(tuples, "10.42.0.1", 9090, "TCP") { + t.Errorf("wrong port must not match") + } +} + +// TestExpandServiceNeighbors_Selector: serviceSelector expands across all +// matching Services in the scoped namespace. +func TestExpandServiceNeighbors_Selector(t *testing.T) { + l := realFluxTopology() + in := []v1beta1.NetworkNeighbor{{ + Identifier: "guestbooks", + Type: "internal", + ServiceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "guestbook"}}, + NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/metadata.name": "gitops-demo"}}, + Ports: []v1beta1.NetworkPort{port("TCP-80", 80)}, + }} + out := ExpandServiceNeighbors(in, l) + if len(out) != 1 { + t.Fatalf("expected 1 synthesized neighbor, got %d", len(out)) + } + if len(out[0].IPAddresses) != 2 { + t.Errorf("expected 2 guestbook ClusterIPs, got %v", out[0].IPAddresses) + } +} + +// TestExpandServiceNeighbors_NilLister: no cluster view, no expansion. +func TestExpandServiceNeighbors_NilLister(t *testing.T) { + in := []v1beta1.NetworkNeighbor{{Identifier: "am", Entity: "host"}} + if out := ExpandServiceNeighbors(in, nil); out != nil { + t.Errorf("nil lister must expand to nil, got %v", out) + } +} + +// TestExpandServiceNeighbors_SelectorFailClosed: a serviceSelector carrying +// MatchExpressions (unsupported) or an empty matchLabels must NOT broaden the +// allowlist — it resolves to nothing. +func TestExpandServiceNeighbors_SelectorFailClosed(t *testing.T) { + l := realFluxTopology() + cases := []v1beta1.NetworkNeighbor{ + {Identifier: "expr", ServiceSelector: &metav1.LabelSelector{ + MatchLabels: map[string]string{"app": "guestbook"}, + MatchExpressions: []metav1.LabelSelectorRequirement{{Key: "tier", Operator: metav1.LabelSelectorOpExists}}, + }, Ports: []v1beta1.NetworkPort{port("TCP-80", 80)}}, + {Identifier: "empty", ServiceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{}}, Ports: []v1beta1.NetworkPort{port("TCP-80", 80)}}, + } + for _, n := range cases { + if out := ExpandServiceNeighbors([]v1beta1.NetworkNeighbor{n}, l); len(out) != 0 { + t.Errorf("%s: selector must fail closed, got %d", n.Identifier, len(out)) + } + } +} + +// TestWithResolvedServiceNeighbors: the CP-level wrapper appends resolved +// neighbors without mutating the input, and is a no-op when nothing resolves. +func TestWithResolvedServiceNeighbors(t *testing.T) { + l := realFluxTopology() + cp := &v1beta1.ContainerProfile{} + cp.Spec.Egress = []v1beta1.NetworkNeighbor{ + {Identifier: "am", ServiceRefNamespace: "honey", ServiceRefName: "alertmanager", Ports: []v1beta1.NetworkPort{port("TCP-9093", 9093)}}, + } + cp.Spec.Ingress = []v1beta1.NetworkNeighbor{ + {Identifier: "probes", Entity: "host", Ports: []v1beta1.NetworkPort{port("TCP-9440", 9440)}}, + } + out := WithResolvedServiceNeighbors(cp, l) + + if len(cp.Spec.Egress) != 1 || len(cp.Spec.Ingress) != 1 { + t.Fatalf("input CP must not be mutated: eg=%d in=%d", len(cp.Spec.Egress), len(cp.Spec.Ingress)) + } + if len(out.Spec.Egress) != 2 { + t.Errorf("egress: want original + 1 resolved, got %d", len(out.Spec.Egress)) + } + if len(out.Spec.Ingress) != 2 { + t.Errorf("ingress: want original + 1 resolved, got %d", len(out.Spec.Ingress)) + } + + // No-op cases. + if got := WithResolvedServiceNeighbors(cp, nil); got != cp { + t.Error("nil lister must return the same CP unchanged") + } + plain := &v1beta1.ContainerProfile{} + plain.Spec.Egress = []v1beta1.NetworkNeighbor{{Identifier: "ip", IPAddresses: []string{"10.0.0.0/8"}}} + if got := WithResolvedServiceNeighbors(plain, l); got != plain { + t.Error("a CP with no service/entity neighbors must return unchanged (same pointer)") + } +} + +// TestExpandServiceNeighbors_NamespaceSelectorFailClosed: a namespaceSelector +// is honored only as the single equality kubernetes.io/metadata.name=; any +// other form must fail closed rather than silently broaden cluster-wide. +func TestExpandServiceNeighbors_NamespaceSelectorFailClosed(t *testing.T) { + l := realFluxTopology() + withNS := func(nsSel *metav1.LabelSelector) v1beta1.NetworkNeighbor { + return v1beta1.NetworkNeighbor{ + Identifier: "svc", + ServiceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "guestbook"}}, + NamespaceSelector: nsSel, + Ports: []v1beta1.NetworkPort{port("TCP-80", 80)}, + } + } + bad := []*metav1.LabelSelector{ + {MatchExpressions: []metav1.LabelSelectorRequirement{{Key: "kubernetes.io/metadata.name", Operator: metav1.LabelSelectorOpExists}}}, + {MatchLabels: map[string]string{"env": "prod"}}, // wrong key + {MatchLabels: map[string]string{"kubernetes.io/metadata.name": "gitops-demo", "x": "y"}}, // extra key + {MatchLabels: map[string]string{}}, // empty + } + for i, ns := range bad { + if out := ExpandServiceNeighbors([]v1beta1.NetworkNeighbor{withNS(ns)}, l); len(out) != 0 { + t.Errorf("bad namespaceSelector[%d] must fail closed, got %d", i, len(out)) + } + } + good := withNS(&metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/metadata.name": "gitops-demo"}}) + if out := ExpandServiceNeighbors([]v1beta1.NetworkNeighbor{good}, l); len(out) != 1 { + t.Errorf("metadata.name namespaceSelector should resolve, got %d", len(out)) + } +} + +// TestHasServiceNeighbors: a profile with a serviceRef/serviceSelector/entity +// neighbor is flagged as depending on the live cluster view; a plain +// ipAddresses/dnsNames profile is not. +func TestHasServiceNeighbors(t *testing.T) { + if HasServiceNeighbors(nil) { + t.Error("nil CP must be false") + } + plain := &v1beta1.ContainerProfile{} + plain.Spec.Egress = []v1beta1.NetworkNeighbor{{Identifier: "ip", IPAddresses: []string{"10.0.0.0/8"}}} + if HasServiceNeighbors(plain) { + t.Error("plain ipAddresses profile must not use service resolution") + } + for _, n := range []v1beta1.NetworkNeighbor{ + {ServiceRefName: "alertmanager", ServiceRefNamespace: "honey"}, + {ServiceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "x"}}}, + {Entity: "host"}, + } { + cp := &v1beta1.ContainerProfile{} + cp.Spec.Ingress = []v1beta1.NetworkNeighbor{n} + if !HasServiceNeighbors(cp) { + t.Errorf("neighbor %+v should be flagged", n) + } + } + nsOnly := &v1beta1.ContainerProfile{} + nsOnly.Spec.Ingress = []v1beta1.NetworkNeighbor{{ServiceRefNamespace: "honey"}} + if HasServiceNeighbors(nsOnly) { + t.Error("ServiceRefNamespace without ServiceRefName must not be flagged (specFromNeighbor rejects it)") + } +} diff --git a/pkg/networkpeer/lister.go b/pkg/networkpeer/lister.go new file mode 100644 index 0000000000..5c88c7d7ba --- /dev/null +++ b/pkg/networkpeer/lister.go @@ -0,0 +1,192 @@ +package networkpeer + +import ( + "net" + "sync/atomic" + + corev1 "k8s.io/api/core/v1" + discoveryv1 "k8s.io/api/discovery/v1" + "k8s.io/apimachinery/pkg/labels" + corelisters "k8s.io/client-go/listers/core/v1" + discoverylisters "k8s.io/client-go/listers/discovery/v1" +) + +// InformerLister is the production Lister, backed by Service / EndpointSlice / +// Node informer listers. It resolves a serviceRef/serviceSelector to the +// Service's ClusterIP(s) ∪ its EndpointSlice addresses, and the "host" entity +// to every node's InternalIP(s) plus the CNI gateway derived from its PodCIDR. +type InformerLister struct { + services corelisters.ServiceLister + slices discoverylisters.EndpointSliceLister + nodes corelisters.NodeLister + // nodeName scopes the "host" entity to the local node. Empty means every + // node (used by tests); production passes the agent's own node so kubelet + // probes from this node's gateway match without broadening "host" to the + // whole cluster. + nodeName string + // generation advances on every observed Service/EndpointSlice/Node change + // (bumped from informer event handlers wired in cmd/main.go). + generation atomic.Int64 +} + +// Generation returns the current cluster-view generation. +func (l *InformerLister) Generation() int64 { return l.generation.Load() } + +// Bump advances the generation; wire it to the informer event handlers. +func (l *InformerLister) Bump() { l.generation.Add(1) } + +func NewInformerLister(services corelisters.ServiceLister, slices discoverylisters.EndpointSliceLister, nodes corelisters.NodeLister, nodeName string) *InformerLister { + return &InformerLister{services: services, slices: slices, nodes: nodes, nodeName: nodeName} +} + +var _ Lister = (*InformerLister)(nil) + +func (l *InformerLister) ServiceByName(namespace, name string) (*ServiceInfo, bool) { + svc, err := l.services.Services(namespace).Get(name) + if err != nil { + return nil, false + } + return l.serviceInfo(svc), true +} + +func (l *InformerLister) ServicesByLabels(serviceSelector, namespaceLabels map[string]string) []*ServiceInfo { + // Never resolve an empty selector to labels.Everything() — that would + // allowlist every Service in the cluster. Fail closed. + if len(serviceSelector) == 0 { + return nil + } + svcs, err := l.services.List(labels.SelectorFromSet(serviceSelector)) + if err != nil { + return nil + } + wantNS := "" + if namespaceLabels != nil { + wantNS = namespaceLabels["kubernetes.io/metadata.name"] + } + var out []*ServiceInfo + for _, svc := range svcs { + if wantNS != "" && svc.Namespace != wantNS { + continue + } + out = append(out, l.serviceInfo(svc)) + } + return out +} + +func (l *InformerLister) HostIPs() []string { + nodes, err := l.nodes.List(labels.Everything()) + if err != nil { + return nil + } + var ips []string + for _, n := range nodes { + if l.nodeName != "" && n.Name != l.nodeName { + continue + } + for _, addr := range n.Status.Addresses { + if addr.Type == corev1.NodeInternalIP { + ips = append(ips, addr.Address) + } + } + for _, cidr := range podCIDRs(n) { + if gw := gatewayIP(cidr); gw != "" { + ips = append(ips, gw) + } + } + } + return dedupe(ips) +} + +func (l *InformerLister) serviceInfo(svc *corev1.Service) *ServiceInfo { + info := &ServiceInfo{Namespace: svc.Namespace, Name: svc.Name, Labels: svc.Labels} + for _, ip := range svc.Spec.ClusterIPs { + if ip != "" && ip != corev1.ClusterIPNone { + info.ClusterIPs = append(info.ClusterIPs, ip) + } + } + if len(info.ClusterIPs) == 0 && svc.Spec.ClusterIP != "" && svc.Spec.ClusterIP != corev1.ClusterIPNone { + info.ClusterIPs = append(info.ClusterIPs, svc.Spec.ClusterIP) + } + info.EndpointIPs = l.endpointIPs(svc.Namespace, svc.Name) + return info +} + +func (l *InformerLister) endpointIPs(namespace, service string) []string { + sel := labels.SelectorFromSet(labels.Set{discoveryv1.LabelServiceName: service}) + slices, err := l.slices.EndpointSlices(namespace).List(sel) + if err != nil { + return nil + } + var ips []string + for _, es := range slices { + for i := range es.Endpoints { + ips = append(ips, es.Endpoints[i].Addresses...) + } + } + return dedupe(ips) +} + +// TrimService and TrimEndpointSlice are informer TransformFuncs that drop the +// bulk the resolver never reads — managedFields and annotations (1–4 KiB per +// real object), and for EndpointSlices every per-endpoint field but Addresses — +// before objects enter the cluster-wide cache. Wire via Informer().SetTransform +// so a DaemonSet's per-node Service/EndpointSlice cache stays small. Identity +// and resourceVersion are preserved so listing/indexing is unaffected. +func TrimService(obj interface{}) (interface{}, error) { + if svc, ok := obj.(*corev1.Service); ok { + svc.ManagedFields = nil + svc.Annotations = nil + } + return obj, nil +} + +func TrimEndpointSlice(obj interface{}) (interface{}, error) { + if es, ok := obj.(*discoveryv1.EndpointSlice); ok { + es.ManagedFields = nil + es.Annotations = nil + for i := range es.Endpoints { + es.Endpoints[i] = discoveryv1.Endpoint{Addresses: es.Endpoints[i].Addresses} + } + } + return obj, nil +} + +func podCIDRs(n *corev1.Node) []string { + if len(n.Spec.PodCIDRs) > 0 { + return n.Spec.PodCIDRs + } + if n.Spec.PodCIDR != "" { + return []string{n.Spec.PodCIDR} + } + return nil +} + +// gatewayIP returns the conventional CNI gateway for a pod CIDR: the network +// address + 1 (e.g. 10.42.0.0/24 -> 10.42.0.1). Masqueraded node-sourced +// traffic (kubelet health probes) appears from this address. IPv6 CIDRs yield +// no gateway (the .1 convention is IPv4). +func gatewayIP(cidr string) string { + _, ipNet, err := net.ParseCIDR(cidr) + if err != nil { + return "" + } + ip := ipNet.IP.To4() + if ip == nil { + return "" + } + gw := make(net.IP, len(ip)) + copy(gw, ip) + for i := len(gw) - 1; i >= 0; i-- { + gw[i]++ + if gw[i] != 0 { + break + } + } + // A /31 or /32 (or a network address ending in .255 that overflows) yields a + // gateway outside the CIDR — never allowlist an IP the pod network doesn't + // actually contain. + if !ipNet.Contains(gw) { + return "" + } + return gw.String() +} diff --git a/pkg/networkpeer/lister_test.go b/pkg/networkpeer/lister_test.go new file mode 100644 index 0000000000..98d0143b10 --- /dev/null +++ b/pkg/networkpeer/lister_test.go @@ -0,0 +1,147 @@ +package networkpeer + +import ( + "testing" + + corev1 "k8s.io/api/core/v1" + discoveryv1 "k8s.io/api/discovery/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes/fake" +) + +// newTestLister builds an InformerLister over a fake cluster seeded from the +// real flux/kubescape topology, exercising the production Service/EndpointSlice/ +// Node -> Lister path (not the hand-written fake used by the resolver tests). +func newTestLister(t *testing.T) *InformerLister { + t.Helper() + client := fake.NewClientset( + &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{Namespace: "honey", Name: "alertmanager", Labels: map[string]string{"app": "alertmanager"}}, + Spec: corev1.ServiceSpec{ClusterIP: "10.43.54.190", ClusterIPs: []string{"10.43.54.190"}}, + }, + &discoveryv1.EndpointSlice{ + ObjectMeta: metav1.ObjectMeta{Namespace: "honey", Name: "alertmanager-x1", Labels: map[string]string{discoveryv1.LabelServiceName: "alertmanager"}}, + Endpoints: []discoveryv1.Endpoint{{Addresses: []string{"10.42.0.44", "10.42.0.84"}}}, + }, + &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{Namespace: "gitops-demo", Name: "guestbook-ui", Labels: map[string]string{"app": "guestbook"}}, + Spec: corev1.ServiceSpec{ClusterIP: "10.43.111.192", ClusterIPs: []string{"10.43.111.192"}}, + }, + &corev1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: "tanzee"}, + Spec: corev1.NodeSpec{PodCIDR: "10.42.0.0/24", PodCIDRs: []string{"10.42.0.0/24"}}, + Status: corev1.NodeStatus{Addresses: []corev1.NodeAddress{{Type: corev1.NodeInternalIP, Address: "192.168.0.191"}}}, + }, + &corev1.Node{ // a second node whose IPs must NOT leak into this node's "host" + ObjectMeta: metav1.ObjectMeta{Name: "other"}, + Spec: corev1.NodeSpec{PodCIDR: "10.42.9.0/24", PodCIDRs: []string{"10.42.9.0/24"}}, + Status: corev1.NodeStatus{Addresses: []corev1.NodeAddress{{Type: corev1.NodeInternalIP, Address: "192.168.0.99"}}}, + }, + ) + factory := informers.NewSharedInformerFactory(client, 0) + l := NewInformerLister( + factory.Core().V1().Services().Lister(), + factory.Discovery().V1().EndpointSlices().Lister(), + factory.Core().V1().Nodes().Lister(), + "tanzee", + ) + stop := make(chan struct{}) + t.Cleanup(func() { close(stop) }) + factory.Start(stop) + factory.WaitForCacheSync(stop) + return l +} + +func TestInformerLister_ServiceByName(t *testing.T) { + l := newTestLister(t) + svc, ok := l.ServiceByName("honey", "alertmanager") + if !ok { + t.Fatal("alertmanager Service should resolve") + } + if len(svc.ClusterIPs) != 1 || svc.ClusterIPs[0] != "10.43.54.190" { + t.Errorf("ClusterIPs: got %v", svc.ClusterIPs) + } + if len(svc.EndpointIPs) != 2 { + t.Errorf("EndpointIPs: got %v (want 2 from the EndpointSlice)", svc.EndpointIPs) + } + // End-to-end through the resolver, like the projection will. + tuples := Resolve(PeerSpec{ServiceRef: &ServiceRef{"honey", "alertmanager"}, Ports: []PortProto{{Port: 9093, Protocol: "TCP"}}}, l) + if !Matches(tuples, "10.43.54.190", 9093, "TCP") || !Matches(tuples, "10.42.0.44", 9093, "TCP") { + t.Errorf("resolver over informer lister should match ClusterIP + endpoints") + } + if _, ok := l.ServiceByName("honey", "nope"); ok { + t.Error("unknown Service must not resolve") + } +} + +func TestInformerLister_HostIPs(t *testing.T) { + l := newTestLister(t) + ips := l.HostIPs() + want := map[string]bool{"192.168.0.191": false, "10.42.0.1": false} + for _, ip := range ips { + if _, ok := want[ip]; ok { + want[ip] = true + } + } + for ip, seen := range want { + if !seen { + t.Errorf("HostIPs missing %s (got %v)", ip, ips) + } + } + for _, ip := range ips { + if ip == "192.168.0.99" || ip == "10.42.9.1" { + t.Errorf("HostIPs must be scoped to the local node; leaked other node's %s", ip) + } + } +} + +// TestInformerLister_EmptySelectorFailsClosed: an empty serviceSelector must +// NOT resolve to every Service in the cluster. +func TestInformerLister_EmptySelectorFailsClosed(t *testing.T) { + l := newTestLister(t) + if got := l.ServicesByLabels(map[string]string{}, nil); len(got) != 0 { + t.Errorf("empty selector must fail closed, got %d services", len(got)) + } + if got := Resolve(PeerSpec{ServiceSelector: map[string]string{}, Ports: tcp(80)}, l); len(got) != 0 { + t.Errorf("Resolve with empty selector must yield nothing, got %v", got) + } +} + +func TestInformerLister_ServicesByLabels(t *testing.T) { + l := newTestLister(t) + svcs := l.ServicesByLabels(map[string]string{"app": "guestbook"}, map[string]string{"kubernetes.io/metadata.name": "gitops-demo"}) + if len(svcs) != 1 || svcs[0].Name != "guestbook-ui" { + t.Fatalf("expected guestbook-ui, got %v", svcs) + } + // Namespace scoping excludes a same-label service elsewhere (none here) and + // a wrong-namespace filter yields nothing. + if got := l.ServicesByLabels(map[string]string{"app": "guestbook"}, map[string]string{"kubernetes.io/metadata.name": "other"}); len(got) != 0 { + t.Errorf("namespace filter should exclude, got %v", got) + } +} + +func TestGatewayIP(t *testing.T) { + cases := map[string]string{ + "10.42.0.0/24": "10.42.0.1", + "10.244.5.0/24": "10.244.5.1", + "2001:db8::/64": "", + "10.42.0.5/32": "", // /32 host: incremented gateway is outside the CIDR + "255.255.255.255/32": "", // overflow to 0.0.0.0, out of CIDR + } + for cidr, want := range cases { + if got := gatewayIP(cidr); got != want { + t.Errorf("gatewayIP(%s)=%q want %q", cidr, got, want) + } + } +} + +func TestInformerLister_Generation(t *testing.T) { + l := newTestLister(t) + g0 := l.Generation() + l.Bump() + l.Bump() + if l.Generation() != g0+2 { + t.Errorf("Generation must advance on Bump: got %d want %d", l.Generation(), g0+2) + } +} diff --git a/pkg/networkpeer/perf_bench_test.go b/pkg/networkpeer/perf_bench_test.go new file mode 100644 index 0000000000..e3343214b8 --- /dev/null +++ b/pkg/networkpeer/perf_bench_test.go @@ -0,0 +1,283 @@ +package networkpeer + +import ( + "fmt" + "runtime" + "testing" + + "github.com/kubescape/storage/pkg/apis/softwarecomposition/v1beta1" + corev1 "k8s.io/api/core/v1" + discoveryv1 "k8s.io/api/discovery/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + corelisters "k8s.io/client-go/listers/core/v1" + discoverylisters "k8s.io/client-go/listers/discovery/v1" + "k8s.io/client-go/tools/cache" +) + +func benchService(i int) *corev1.Service { + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: fmt.Sprintf("ns-%d", i%50), + Name: fmt.Sprintf("svc-%d", i), + Labels: map[string]string{"app": fmt.Sprintf("app-%d", i), "team": fmt.Sprintf("team-%d", i%20)}, + }, + Spec: corev1.ServiceSpec{ + ClusterIP: fmt.Sprintf("10.43.%d.%d", i/256, i%256), + ClusterIPs: []string{fmt.Sprintf("10.43.%d.%d", i/256, i%256)}, + Ports: []corev1.ServicePort{{Name: "http", Port: 8080, Protocol: corev1.ProtocolTCP}}, + }, + } +} + +func benchSlice(svcIdx, sliceIdx, endpoints int) *discoveryv1.EndpointSlice { + eps := make([]discoveryv1.Endpoint, 0, endpoints) + ready := true + for e := 0; e < endpoints; e++ { + eps = append(eps, discoveryv1.Endpoint{ + Addresses: []string{fmt.Sprintf("10.42.%d.%d", (svcIdx*7+e)%256, (sliceIdx*31+e)%256)}, + Conditions: discoveryv1.EndpointConditions{Ready: &ready}, + TargetRef: &corev1.ObjectReference{Kind: "Pod", Namespace: fmt.Sprintf("ns-%d", svcIdx%50), Name: fmt.Sprintf("pod-%d-%d-%d", svcIdx, sliceIdx, e)}, + NodeName: ptrTo(fmt.Sprintf("node-%d", e%10)), + }) + } + return &discoveryv1.EndpointSlice{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: fmt.Sprintf("ns-%d", svcIdx%50), + Name: fmt.Sprintf("svc-%d-%d", svcIdx, sliceIdx), + Labels: map[string]string{discoveryv1.LabelServiceName: fmt.Sprintf("svc-%d", svcIdx)}, + }, + AddressType: discoveryv1.AddressTypeIPv4, + Endpoints: eps, + Ports: []discoveryv1.EndpointPort{{Name: ptrTo("http"), Port: ptrTo(int32(8080)), Protocol: &[]corev1.Protocol{corev1.ProtocolTCP}[0]}}, + } +} + +func ptrTo[T any](v T) *T { return &v } + +// buildBenchLister backs an InformerLister with plain cache indexers (the same +// store type a SharedInformer uses) so benchmarks measure lister/resolution +// cost without fake-clientset watch machinery. +func buildBenchLister(tb testing.TB, nServices, slicesPerSvc, endpointsPerSlice int) *InformerLister { + tb.Helper() + svcIdx := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + sliceIdx := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + nodeIdx := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{}) + for i := 0; i < nServices; i++ { + if err := svcIdx.Add(benchService(i)); err != nil { + tb.Fatal(err) + } + for s := 0; s < slicesPerSvc; s++ { + if err := sliceIdx.Add(benchSlice(i, s, endpointsPerSlice)); err != nil { + tb.Fatal(err) + } + } + } + if err := nodeIdx.Add(&corev1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: "bench-node"}, + Spec: corev1.NodeSpec{PodCIDR: "10.42.0.0/24", PodCIDRs: []string{"10.42.0.0/24"}}, + Status: corev1.NodeStatus{Addresses: []corev1.NodeAddress{{Type: corev1.NodeInternalIP, Address: "192.168.0.191"}}}, + }); err != nil { + tb.Fatal(err) + } + return NewInformerLister( + corelisters.NewServiceLister(svcIdx), + discoverylisters.NewEndpointSliceLister(sliceIdx), + corelisters.NewNodeLister(nodeIdx), + "bench-node", + ) +} + +// benchProfile builds a ContainerProfile with nPlain ordinary ipAddresses +// egress neighbors, nRefs serviceRef neighbors, and some opens/execs bulk so +// DeepCopy cost is realistic. +func benchProfile(nPlain, nRefs, nOpens int) *v1beta1.ContainerProfile { + cp := &v1beta1.ContainerProfile{} + cp.Name = "bench-cp" + port := int32(8080) + for i := 0; i < nPlain; i++ { + cp.Spec.Egress = append(cp.Spec.Egress, v1beta1.NetworkNeighbor{ + Identifier: fmt.Sprintf("plain-%d", i), + Type: "external", + IPAddresses: []string{fmt.Sprintf("52.216.%d.%d", i/256, i%256)}, + Ports: []v1beta1.NetworkPort{{Name: "TCP-8080", Protocol: "TCP", Port: &port}}, + }) + } + for i := 0; i < nRefs; i++ { + cp.Spec.Egress = append(cp.Spec.Egress, v1beta1.NetworkNeighbor{ + Identifier: fmt.Sprintf("ref-%d", i), + Type: "internal", + ServiceRefNamespace: fmt.Sprintf("ns-%d", i%50), + ServiceRefName: fmt.Sprintf("svc-%d", i), + Ports: []v1beta1.NetworkPort{{Name: "TCP-8080", Protocol: "TCP", Port: &port}}, + }) + } + for i := 0; i < nOpens; i++ { + cp.Spec.Opens = append(cp.Spec.Opens, v1beta1.OpenCalls{ + Path: fmt.Sprintf("/usr/lib/x86_64-linux-gnu/lib-%d.so.%d", i, i%9), + Flags: []string{"O_RDONLY", "O_CLOEXEC"}, + }) + cp.Spec.Execs = append(cp.Spec.Execs, v1beta1.ExecCalls{ + Path: fmt.Sprintf("/usr/bin/tool-%d", i), + Args: []string{fmt.Sprintf("--flag-%d", i)}, + }) + } + return cp +} + +func BenchmarkServiceByName_1kSvc_5kSlices(b *testing.B) { + l := buildBenchLister(b, 1000, 5, 10) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if _, ok := l.ServiceByName("ns-7", "svc-7"); !ok { + b.Fatal("service must resolve") + } + } +} + +func BenchmarkServicesByLabels_1kSvc(b *testing.B) { + l := buildBenchLister(b, 1000, 5, 10) + sel := map[string]string{"app": "app-7"} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if out := l.ServicesByLabels(sel, nil); len(out) != 1 { + b.Fatalf("want 1 service, got %d", len(out)) + } + } +} + +func BenchmarkResolveIPs_ServiceRef(b *testing.B) { + l := buildBenchLister(b, 1000, 5, 10) + spec := PeerSpec{ServiceRef: &ServiceRef{Namespace: "ns-7", Name: "svc-7"}} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if ips := ResolveIPs(spec, l); len(ips) == 0 { + b.Fatal("must resolve") + } + } +} + +func BenchmarkResolveIPs_Entity_Host(b *testing.B) { + l := buildBenchLister(b, 10, 1, 2) + spec := PeerSpec{Entity: "host"} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if ips := ResolveIPs(spec, l); len(ips) == 0 { + b.Fatal("must resolve") + } + } +} + +// BenchmarkWithResolvedServiceNeighbors_NoServiceFields is the 99% case: a +// profile with only plain ipAddresses neighbors. The function documents itself +// as a no-op then — this measures whether the no-op is actually free. +func BenchmarkWithResolvedServiceNeighbors_NoServiceFields(b *testing.B) { + l := buildBenchLister(b, 1000, 5, 10) + for _, n := range []int{100, 1000} { + cp := benchProfile(n, 0, 0) + b.Run(fmt.Sprintf("plainNeighbors=%d", n), func(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + out := WithResolvedServiceNeighbors(cp, l) + if out != cp { + b.Fatal("no-op path must return the same pointer") + } + } + }) + } +} + +// BenchmarkWithResolvedServiceNeighbors_Resolving measures the full expansion: +// resolution + DeepCopy of the whole profile (opens/execs bulk included). +func BenchmarkWithResolvedServiceNeighbors_Resolving(b *testing.B) { + l := buildBenchLister(b, 1000, 5, 10) + for _, tc := range []struct{ plain, refs, opens int }{ + {plain: 20, refs: 1, opens: 200}, + {plain: 20, refs: 8, opens: 200}, + {plain: 20, refs: 8, opens: 2000}, + } { + cp := benchProfile(tc.plain, tc.refs, tc.opens) + b.Run(fmt.Sprintf("plain=%d/refs=%d/opens=%d", tc.plain, tc.refs, tc.opens), func(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + out := WithResolvedServiceNeighbors(cp, l) + if out == cp { + b.Fatal("resolving path must copy") + } + } + }) + } +} + +func BenchmarkHasServiceNeighbors_1kPlain(b *testing.B) { + cp := benchProfile(1000, 0, 0) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + if HasServiceNeighbors(cp) { + b.Fatal("plain profile must not report service neighbors") + } + } +} + +// TestInformerCacheMemoryEstimate approximates the heap retained by a +// cluster-wide Service + EndpointSlice informer cache at 1k Services / 5k +// EndpointSlices (10 endpoints each), vs Services alone. Run with -run +// InformerCacheMemoryEstimate -v. +func TestInformerCacheMemoryEstimate(t *testing.T) { + measure := func(build func() []interface{}) uint64 { + runtime.GC() + var before, after runtime.MemStats + runtime.ReadMemStats(&before) + objs := build() + runtime.GC() + runtime.ReadMemStats(&after) + runtime.KeepAlive(objs) + if after.HeapAlloc < before.HeapAlloc { + return 0 + } + return after.HeapAlloc - before.HeapAlloc + } + + svcBytes := measure(func() []interface{} { + idx := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + for i := 0; i < 1000; i++ { + _ = idx.Add(benchService(i)) + } + return []interface{}{idx} + }) + sliceBytes := measure(func() []interface{} { + idx := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + for i := 0; i < 1000; i++ { + for s := 0; s < 5; s++ { + _ = idx.Add(benchSlice(i, s, 10)) + } + } + return []interface{}{idx} + }) + strippedBytes := measure(func() []interface{} { + idx := cache.NewIndexer(cache.MetaNamespaceKeyFunc, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}) + for i := 0; i < 1000; i++ { + for s := 0; s < 5; s++ { + full := benchSlice(i, s, 10) + stripped := &discoveryv1.EndpointSlice{ + ObjectMeta: metav1.ObjectMeta{Namespace: full.Namespace, Name: full.Name, Labels: full.Labels}, + AddressType: full.AddressType, + } + for _, ep := range full.Endpoints { + stripped.Endpoints = append(stripped.Endpoints, discoveryv1.Endpoint{Addresses: ep.Addresses}) + } + _ = idx.Add(stripped) + } + } + return []interface{}{idx} + }) + t.Logf("1000 Services in indexer: ~%d KiB total, ~%d B/object", svcBytes/1024, svcBytes/1000) + t.Logf("5000 EndpointSlices (10 endpoints each) in indexer: ~%d KiB total, ~%d B/object", sliceBytes/1024, sliceBytes/5000) + t.Logf("5000 STRIPPED EndpointSlices (addresses+labels only, SetTransform mitigation): ~%d KiB total, ~%d B/object", strippedBytes/1024, strippedBytes/5000) +} diff --git a/pkg/networkpeer/resolve.go b/pkg/networkpeer/resolve.go new file mode 100644 index 0000000000..de8c1375d7 --- /dev/null +++ b/pkg/networkpeer/resolve.go @@ -0,0 +1,228 @@ +// Package networkpeer resolves Kubernetes-native network-neighbor selectors +// (a Service reference, a Service label selector, or a reserved entity such as +// "host") into the concrete (IP, port, protocol) tuples an egress/ingress +// allowlist should match. +// +// It exists so a ContainerProfile can express cluster-infrastructure peers — +// Service ClusterIPs (alertmanager, kube-apiserver via default/kubernetes, +// kube-dns, ...) and host/kubelet traffic — portably (by name, resolved +// per-cluster) and narrowly (only the referenced Service/entity), instead of a +// broad ipAddresses CIDR over the whole service network. See +// k8sstormcenter/node-agent#92. +// +// Resolution is intentionally decoupled from live matching: callers resolve a +// PeerSpec to []AllowTuple once (e.g. at projection time) via a Lister backed +// by Service/EndpointSlice/Node informers, then match observed connections +// against the tuples with Matches. The Lister interface keeps the resolver +// unit-testable against a fake cluster view. +package networkpeer + +import "strings" + +// EntityHost is the reserved entity naming the local node: its InternalIP(s) +// and the CNI gateway address. It is the one peer class no Service object can +// represent (kubelet health probes, node-sourced / masqueraded traffic). +const EntityHost = "host" + +// clusterDNSSuffix is the in-cluster Service DNS zone. A serviceRef / +// serviceSelector implies the cluster FQDN ..svc. so a +// client dialling the Service by DNS is allowlisted without a parallel +// dnsNames entry. Kubernetes' default zone; overridden per-cluster is rare. +const clusterDNSSuffix = "svc.cluster.local" + +// PortProto is a single allowed destination port/protocol. Protocol is +// upper-case ("TCP"/"UDP"); an empty Protocol matches any protocol. +type PortProto struct { + Port int32 + Protocol string +} + +// ServiceRef names a single Service by namespace and name. +type ServiceRef struct { + Namespace string + Name string +} + +// PeerSpec is the storage-agnostic form of one serviceRef / serviceSelector / +// entity network-neighbor entry. Exactly one of ServiceRef, ServiceSelector, +// or Entity is expected to be set; Ports scopes the resolved tuples. +type PeerSpec struct { + ServiceRef *ServiceRef + ServiceSelector map[string]string + NamespaceLabels map[string]string + Entity string + Ports []PortProto +} + +// ServiceInfo is the resolver's view of one Service. +type ServiceInfo struct { + Namespace string + Name string + Labels map[string]string + ClusterIPs []string + EndpointIPs []string +} + +// Lister is the read-only cluster view the resolver needs. Production wires it +// to Service/EndpointSlice/Node informer listers; tests use a fake. +type Lister interface { + ServiceByName(namespace, name string) (*ServiceInfo, bool) + ServicesByLabels(serviceSelector, namespaceLabels map[string]string) []*ServiceInfo + HostIPs() []string + // Generation increments whenever the underlying cluster view changes (any + // Service/EndpointSlice/Node event). Callers store it alongside a projected + // profile and re-project when it advances, so resolved IPs don't go stale on + // endpoint churn or caches that filled after projection. + Generation() int64 +} + +// AllowTuple is one concrete (IP, port, protocol) a resolved PeerSpec permits. +type AllowTuple struct { + IP string + Port int32 + Protocol string +} + +// Resolve expands spec into the concrete tuples it authorises, using l for the +// current cluster view. A spec with no resolvable target (unknown Service, +// selector matching nothing, unknown entity) yields no tuples — never a +// match-all. A spec with no Ports yields one tuple per IP with Port 0 / +// Protocol "" (any-port), so callers that ignore ports still work; callers +// that enforce ports should treat Port 0 as "unspecified". +func Resolve(spec PeerSpec, l Lister) []AllowTuple { + if l == nil { + return nil + } + ips := resolveIPs(spec, l) + if len(ips) == 0 { + return nil + } + return expand(ips, spec.Ports) +} + +// ResolveIPs returns just the IPs a spec resolves to, ignoring ports. Used by +// the projection-time expansion, which pairs them with the neighbor's own +// ports. +func ResolveIPs(spec PeerSpec, l Lister) []string { + if l == nil { + return nil + } + return resolveIPs(spec, l) +} + +func resolveIPs(spec PeerSpec, l Lister) []string { + if spec.Entity != "" { + if strings.EqualFold(spec.Entity, EntityHost) { + return dedupe(l.HostIPs()) + } + return nil + } + var ips []string + for _, svc := range resolveServices(spec, l) { + ips = append(ips, serviceIPs(svc)...) + } + return dedupe(ips) +} + +// resolveServices returns the Services a serviceRef / serviceSelector spec +// matches. An entity spec matches no Service and returns nil; an empty +// serviceSelector fails closed (never every Service). +func resolveServices(spec PeerSpec, l Lister) []*ServiceInfo { + switch { + case spec.ServiceRef != nil: + svc, ok := l.ServiceByName(spec.ServiceRef.Namespace, spec.ServiceRef.Name) + if !ok || svc == nil { + return nil + } + return []*ServiceInfo{svc} + case spec.ServiceSelector != nil: + if len(spec.ServiceSelector) == 0 { + return nil + } + return l.ServicesByLabels(spec.ServiceSelector, spec.NamespaceLabels) + default: + return nil + } +} + +// ResolveDNSNames returns the cluster FQDN(s) — ..svc. — +// of the Services a serviceRef / serviceSelector spec matches, so a client +// dialling the Service by DNS is allowlisted alongside its IPs. Entity specs +// and unresolvable selectors yield nothing. +func ResolveDNSNames(spec PeerSpec, l Lister) []string { + if l == nil { + return nil + } + var out []string + for _, svc := range resolveServices(spec, l) { + if svc == nil || svc.Namespace == "" || svc.Name == "" { + continue + } + out = append(out, svc.Name+"."+svc.Namespace+"."+clusterDNSSuffix) + } + return dedupe(out) +} + +func serviceIPs(svc *ServiceInfo) []string { + out := make([]string, 0, len(svc.ClusterIPs)+len(svc.EndpointIPs)) + out = append(out, svc.ClusterIPs...) + out = append(out, svc.EndpointIPs...) + return dedupe(out) +} + +func expand(ips []string, ports []PortProto) []AllowTuple { + if len(ports) == 0 { + out := make([]AllowTuple, 0, len(ips)) + for _, ip := range ips { + out = append(out, AllowTuple{IP: ip}) + } + return out + } + out := make([]AllowTuple, 0, len(ips)*len(ports)) + for _, ip := range ips { + for _, p := range ports { + out = append(out, AllowTuple{IP: ip, Port: p.Port, Protocol: strings.ToUpper(p.Protocol)}) + } + } + return out +} + +// Matches reports whether the observed (ip, port, protocol) connection is +// permitted by any tuple. Matching is port-sensitive: a tuple with Port 0 +// (any-port) matches any observed port; otherwise the port must be equal. An +// empty tuple Protocol matches any protocol. +func Matches(tuples []AllowTuple, ip string, port int32, protocol string) bool { + protocol = strings.ToUpper(protocol) + for _, t := range tuples { + if t.IP != ip { + continue + } + if t.Port != 0 && t.Port != port { + continue + } + if t.Protocol != "" && t.Protocol != protocol { + continue + } + return true + } + return false +} + +func dedupe(in []string) []string { + if len(in) == 0 { + return nil + } + seen := make(map[string]struct{}, len(in)) + out := make([]string, 0, len(in)) + for _, s := range in { + if s == "" { + continue + } + if _, ok := seen[s]; ok { + continue + } + seen[s] = struct{}{} + out = append(out, s) + } + return out +} diff --git a/pkg/networkpeer/resolve_test.go b/pkg/networkpeer/resolve_test.go new file mode 100644 index 0000000000..9a16e0a828 --- /dev/null +++ b/pkg/networkpeer/resolve_test.go @@ -0,0 +1,359 @@ +package networkpeer + +import ( + "sort" + "testing" +) + +// fakeLister is a static cluster view seeded from the real Flux/kubescape +// topology observed on the k3s dev cluster (issue #92). It lets the resolver +// tests assert against genuine ClusterIPs/endpoints without a live cluster. +type fakeLister struct { + services map[string]*ServiceInfo // key "ns/name" + hostIPs []string +} + +func (f *fakeLister) ServiceByName(ns, name string) (*ServiceInfo, bool) { + s, ok := f.services[ns+"/"+name] + return s, ok +} + +func (f *fakeLister) ServicesByLabels(sel, nsLabels map[string]string) []*ServiceInfo { + var out []*ServiceInfo + for _, s := range f.services { + if nsLabels != nil { + if s.Labels["__ns__"] != nsLabels["kubernetes.io/metadata.name"] { + continue + } + } + if labelsSubset(sel, s.Labels) { + out = append(out, s) + } + } + sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name }) + return out +} + +func (f *fakeLister) HostIPs() []string { return f.hostIPs } + +func (f *fakeLister) Generation() int64 { return 0 } + +func labelsSubset(want, have map[string]string) bool { + for k, v := range want { + if have[k] != v { + return false + } + } + return true +} + +// realFluxTopology mirrors what `kubectl get svc/endpoints` returned live. +func realFluxTopology() *fakeLister { + return &fakeLister{ + hostIPs: []string{"192.168.0.191", "10.42.0.1"}, // node InternalIP + CNI gateway + services: map[string]*ServiceInfo{ + "honey/alertmanager": { + Namespace: "honey", Name: "alertmanager", + Labels: map[string]string{"app": "alertmanager", "__ns__": "honey"}, + ClusterIPs: []string{"10.43.54.190"}, + EndpointIPs: []string{"10.42.0.44", "10.42.0.84"}, + }, + "honey/storage": { + Namespace: "honey", Name: "storage", + Labels: map[string]string{"app": "storage", "__ns__": "honey"}, + ClusterIPs: []string{"10.43.70.156"}, + }, + "default/kubernetes": { // k3s: apiserver endpoint is the node IP (Kind: Host) + Namespace: "default", Name: "kubernetes", + Labels: map[string]string{"__ns__": "default"}, + ClusterIPs: []string{"10.43.0.1"}, + EndpointIPs: []string{"192.168.0.191"}, + }, + "argocd/argocd-server": { + Namespace: "argocd", Name: "argocd-server", + Labels: map[string]string{"app.kubernetes.io/name": "argocd-server", "__ns__": "argocd"}, + ClusterIPs: []string{"10.43.173.14"}, + }, + "gitops-demo/guestbook-ui": { + Namespace: "gitops-demo", Name: "guestbook-ui", + Labels: map[string]string{"app": "guestbook", "__ns__": "gitops-demo"}, + ClusterIPs: []string{"10.43.111.192"}, + }, + "gitops-demo/helm-guestbook": { + Namespace: "gitops-demo", Name: "helm-guestbook", + Labels: map[string]string{"app": "guestbook", "__ns__": "gitops-demo"}, + ClusterIPs: []string{"10.43.3.63"}, + }, + }, + } +} + +func tcp(port int32) []PortProto { return []PortProto{{Port: port, Protocol: "TCP"}} } + +// Test A1 — serviceRef egress (alertmanager) is narrow AND port-sensitive: +// matches its ClusterIP and endpoints on 9093 only; a sibling service on the +// same port stays visible to R0011 (the whole point vs a /16). +func TestServiceRef_Alertmanager(t *testing.T) { + l := realFluxTopology() + tuples := Resolve(PeerSpec{ServiceRef: &ServiceRef{"honey", "alertmanager"}, Ports: tcp(9093)}, l) + + cases := []struct { + ip string + port int32 + proto string + want bool + why string + }{ + {"10.43.54.190", 9093, "TCP", true, "ClusterIP + port match"}, + {"10.42.0.44", 9093, "TCP", true, "backing endpoint IP"}, + {"10.42.0.84", 9093, "TCP", true, "backing endpoint IP"}, + {"10.43.54.190", 8080, "TCP", false, "wrong port (port-sensitive)"}, + {"10.43.54.190", 9093, "UDP", false, "wrong protocol"}, + {"10.43.173.14", 9093, "TCP", false, "argocd-server — different service, detection preserved"}, + {"10.43.70.156", 9093, "TCP", false, "storage — different service, detection preserved"}, + } + for _, c := range cases { + if got := Matches(tuples, c.ip, c.port, c.proto); got != c.want { + t.Errorf("Matches(%s:%d/%s)=%v want %v (%s)", c.ip, c.port, c.proto, got, c.want, c.why) + } + } +} + +// Test A2 — kube-apiserver needs no dedicated entity: toService default/kubernetes +// resolves to the ClusterIP AND the node-IP endpoint (k3s embeds the apiserver). +func TestServiceRef_KubeApiserver(t *testing.T) { + l := realFluxTopology() + tuples := Resolve(PeerSpec{ServiceRef: &ServiceRef{"default", "kubernetes"}, Ports: tcp(443)}, l) + for _, ip := range []string{"10.43.0.1", "192.168.0.191"} { + if !Matches(tuples, ip, 443, "TCP") { + t.Errorf("apiserver egress %s:443 should match via default/kubernetes", ip) + } + } + if Matches(tuples, "10.43.0.1", 6443, "TCP") { + t.Errorf("apiserver :6443 must not match (port 443 only)") + } +} + +// Test A3 — host entity: kubelet probe from the node/gateway matches on the +// health port only; a pod-CIDR source or wrong port does not. +func TestEntityHost(t *testing.T) { + l := realFluxTopology() + tuples := Resolve(PeerSpec{Entity: EntityHost, Ports: tcp(9440)}, l) + if !Matches(tuples, "10.42.0.1", 9440, "TCP") { + t.Errorf("kubelet probe 10.42.0.1:9440 should match fromEntity host") + } + if !Matches(tuples, "192.168.0.191", 9440, "TCP") { + t.Errorf("node InternalIP :9440 should match fromEntity host") + } + if Matches(tuples, "10.42.0.1", 9090, "TCP") { + t.Errorf("host :9090 must not match (port 9440 only)") + } + if Matches(tuples, "10.42.0.55", 9440, "TCP") { + t.Errorf("a pod IP must not match fromEntity host") + } +} + +// Test A4 — serviceSelector fans out across all matching Services (the two +// gitops-demo guestbook services share app=guestbook), scoped by namespace. +func TestServiceSelector_GuestbookFanout(t *testing.T) { + l := realFluxTopology() + tuples := Resolve(PeerSpec{ + ServiceSelector: map[string]string{"app": "guestbook"}, + NamespaceLabels: map[string]string{"kubernetes.io/metadata.name": "gitops-demo"}, + Ports: tcp(80), + }, l) + for _, ip := range []string{"10.43.111.192", "10.43.3.63"} { + if !Matches(tuples, ip, 80, "TCP") { + t.Errorf("guestbook service %s:80 should match app=guestbook selector", ip) + } + } + if Matches(tuples, "10.43.173.14", 80, "TCP") { + t.Errorf("argocd-server must not match app=guestbook selector") + } +} + +// Test A5 — no accidental match-all: unknown service, unknown entity, and a +// selector matching nothing all resolve to zero tuples. +func TestResolve_NoMatchAll(t *testing.T) { + l := realFluxTopology() + specs := []PeerSpec{ + {ServiceRef: &ServiceRef{"honey", "does-not-exist"}, Ports: tcp(443)}, + {Entity: "world", Ports: tcp(443)}, + {ServiceSelector: map[string]string{"app": "nope"}, Ports: tcp(443)}, + } + for i, s := range specs { + if tuples := Resolve(s, l); len(tuples) != 0 { + t.Errorf("spec[%d] should resolve to no tuples, got %d", i, len(tuples)) + } + } + if Matches(nil, "10.43.0.1", 443, "TCP") { + t.Errorf("nil tuples must never match") + } +} + +// Test A7 — serviceRef/serviceSelector imply the Service cluster FQDN(s); +// entity and unresolvable specs imply none. +func TestResolveDNSNames(t *testing.T) { + l := realFluxTopology() + one := ResolveDNSNames(PeerSpec{ServiceRef: &ServiceRef{"honey", "alertmanager"}}, l) + if len(one) != 1 || one[0] != "alertmanager.honey.svc.cluster.local" { + t.Errorf("serviceRef FQDN: got %v", one) + } + fan := ResolveDNSNames(PeerSpec{ + ServiceSelector: map[string]string{"app": "guestbook"}, + NamespaceLabels: map[string]string{"kubernetes.io/metadata.name": "gitops-demo"}, + }, l) + want := map[string]bool{"guestbook-ui.gitops-demo.svc.cluster.local": true, "helm-guestbook.gitops-demo.svc.cluster.local": true} + if len(fan) != 2 || !want[fan[0]] || !want[fan[1]] { + t.Errorf("selector FQDN fanout: got %v", fan) + } + if got := ResolveDNSNames(PeerSpec{Entity: EntityHost}, l); got != nil { + t.Errorf("host entity implies no FQDN, got %v", got) + } + if got := ResolveDNSNames(PeerSpec{ServiceRef: &ServiceRef{"honey", "nope"}}, l); got != nil { + t.Errorf("unresolvable serviceRef implies no FQDN, got %v", got) + } +} + +// Test A6 — a nil Lister and any-port (no Ports) behave safely. +func TestResolve_Edges(t *testing.T) { + if got := Resolve(PeerSpec{Entity: EntityHost}, nil); got != nil { + t.Errorf("nil lister must resolve to nil, got %v", got) + } + l := realFluxTopology() + anyPort := Resolve(PeerSpec{ServiceRef: &ServiceRef{"honey", "storage"}}, l) + if !Matches(anyPort, "10.43.70.156", 443, "TCP") || !Matches(anyPort, "10.43.70.156", 8443, "TCP") { + t.Errorf("a serviceRef with no Ports should match any observed port on its IP") + } +} + +// TestServiceRef_TruthTable_KubeAPIServer is the full matrix for serviceRef +// resolution + matching, using the always-present default/kubernetes Service +// (the API server). Ports are the 443 the apiserver Service exposes; the +// resolved set is its ClusterIP plus its backing endpoint (the node IP on k3s). +func TestServiceRef_TruthTable_KubeAPIServer(t *testing.T) { + l := realFluxTopology() + const ( + clusterIP = "10.43.0.1" // default/kubernetes ClusterIP + endpoint = "192.168.0.191" // apiserver backing endpoint (node IP) + ) + + // Ported serviceRef: 443/TCP only. + tuples := Resolve(PeerSpec{ServiceRef: &ServiceRef{"default", "kubernetes"}, Ports: tcp(443)}, l) + grid := []struct { + ip string + port int32 + proto string + want bool + why string + }{ + {clusterIP, 443, "TCP", true, "ClusterIP on the exposed port"}, + {endpoint, 443, "TCP", true, "backing endpoint (apiserver node) on the exposed port"}, + {clusterIP, 443, "tcp", true, "protocol match is case-insensitive"}, + {clusterIP, 6443, "TCP", false, "wrong port (port-sensitive)"}, + {clusterIP, 443, "UDP", false, "wrong protocol"}, + {"10.43.54.190", 443, "TCP", false, "a different Service's ClusterIP"}, + {"10.42.0.1", 443, "TCP", false, "the node gateway is not this Service"}, + {"10.42.0.55", 443, "TCP", false, "an unrelated pod IP"}, + {"", 443, "TCP", false, "empty IP never matches"}, + } + for _, c := range grid { + if got := Matches(tuples, c.ip, c.port, c.proto); got != c.want { + t.Errorf("Matches(%q,%d,%s)=%v want %v — %s", c.ip, c.port, c.proto, got, c.want, c.why) + } + } + + // --- resolution edge cases --- + // No ports → any observed port on the resolved IPs matches. + anyPort := Resolve(PeerSpec{ServiceRef: &ServiceRef{"default", "kubernetes"}}, l) + if !Matches(anyPort, clusterIP, 443, "TCP") || !Matches(anyPort, clusterIP, 6443, "TCP") || !Matches(anyPort, endpoint, 8443, "UDP") { + t.Error("a serviceRef with no ports must match any observed port/proto on its IPs") + } + // Unknown Service → nothing (never a match-all). + if got := Resolve(PeerSpec{ServiceRef: &ServiceRef{"default", "does-not-exist"}, Ports: tcp(443)}, l); len(got) != 0 { + t.Errorf("unknown Service must resolve to zero tuples, got %d", len(got)) + } + // Namespace only, no name → not a resolvable Service. + if got := Resolve(PeerSpec{ServiceRef: &ServiceRef{"default", ""}, Ports: tcp(443)}, l); len(got) != 0 { + t.Errorf("serviceRef with no name must resolve to zero tuples, got %d", len(got)) + } + // Wrong namespace for the same name → nothing. + if got := Resolve(PeerSpec{ServiceRef: &ServiceRef{"kube-system", "kubernetes"}, Ports: tcp(443)}, l); len(got) != 0 { + t.Errorf("kubernetes Service exists only in default; other namespace must resolve to zero, got %d", len(got)) + } + // nil lister → nothing. + if got := Resolve(PeerSpec{ServiceRef: &ServiceRef{"default", "kubernetes"}, Ports: tcp(443)}, nil); got != nil { + t.Errorf("nil lister must resolve to nil, got %v", got) + } + // Headless Service (no ClusterIP) → resolves to its endpoints only. + l.services["default/headless"] = &ServiceInfo{Namespace: "default", Name: "headless", EndpointIPs: []string{"10.42.9.9"}} + hl := Resolve(PeerSpec{ServiceRef: &ServiceRef{"default", "headless"}, Ports: tcp(80)}, l) + if !Matches(hl, "10.42.9.9", 80, "TCP") { + t.Error("a headless Service must resolve to its endpoint IPs") + } + // DNS names: the Service FQDN is implied. + dns := ResolveDNSNames(PeerSpec{ServiceRef: &ServiceRef{"default", "kubernetes"}}, l) + if len(dns) != 1 || dns[0] != "kubernetes.default.svc.cluster.local" { + t.Errorf("serviceRef must imply the cluster FQDN, got %v", dns) + } +} + +// TestEntityHost_TruthTable is the full matrix for the "host" entity: it +// resolves to the node's InternalIP(s) plus the CNI gateway, and nothing else. +func TestEntityHost_TruthTable(t *testing.T) { + l := realFluxTopology() // hostIPs: node 192.168.0.191, gateway 10.42.0.1 + const ( + nodeIP = "192.168.0.191" + gateway = "10.42.0.1" + ) + + tuples := Resolve(PeerSpec{Entity: EntityHost, Ports: tcp(10250)}, l) + grid := []struct { + ip string + port int32 + proto string + want bool + why string + }{ + {nodeIP, 10250, "TCP", true, "node InternalIP on the kubelet port"}, + {gateway, 10250, "TCP", true, "CNI gateway (masqueraded node traffic)"}, + {nodeIP, 10250, "tcp", true, "protocol case-insensitive"}, + {nodeIP, 9090, "TCP", false, "wrong port"}, + {nodeIP, 10250, "UDP", false, "wrong protocol"}, + {"10.42.0.55", 10250, "TCP", false, "a pod IP is not a host IP"}, + {"10.43.0.1", 10250, "TCP", false, "a ClusterIP is not a host IP"}, + {"", 10250, "TCP", false, "empty IP never matches"}, + } + for _, c := range grid { + if got := Matches(tuples, c.ip, c.port, c.proto); got != c.want { + t.Errorf("Matches(%q,%d,%s)=%v want %v — %s", c.ip, c.port, c.proto, got, c.want, c.why) + } + } + + // --- edge cases --- + // No ports → any observed port on the host IPs. + anyPort := Resolve(PeerSpec{Entity: EntityHost}, l) + if !Matches(anyPort, nodeIP, 22, "TCP") || !Matches(anyPort, gateway, 53, "UDP") { + t.Error("host entity with no ports must match any observed port on its IPs") + } + // "host" is case-insensitive. + if got := Resolve(PeerSpec{Entity: "HOST", Ports: tcp(10250)}, l); !Matches(got, nodeIP, 10250, "TCP") { + t.Error("the host entity name must be case-insensitive") + } + // An unknown entity resolves to nothing (never a match-all). + if got := Resolve(PeerSpec{Entity: "world", Ports: tcp(443)}, l); len(got) != 0 { + t.Errorf("unknown entity must resolve to zero tuples, got %d", len(got)) + } + // Empty entity string → nothing. + if got := Resolve(PeerSpec{Entity: "", Ports: tcp(443)}, l); len(got) != 0 { + t.Errorf("empty entity must resolve to zero tuples, got %d", len(got)) + } + // host entity implies no DNS name (it is not a Service). + if got := ResolveDNSNames(PeerSpec{Entity: EntityHost}, l); got != nil { + t.Errorf("host entity must imply no FQDN, got %v", got) + } + // nil lister → nothing. + if got := Resolve(PeerSpec{Entity: EntityHost}, nil); got != nil { + t.Errorf("nil lister must resolve to nil, got %v", got) + } +} diff --git a/pkg/objectcache/addr_ports_test.go b/pkg/objectcache/addr_ports_test.go new file mode 100644 index 0000000000..d1dd2ee88d --- /dev/null +++ b/pkg/objectcache/addr_ports_test.go @@ -0,0 +1,43 @@ +package objectcache + +import ( + "testing" + + "github.com/kubescape/storage/pkg/apis/softwarecomposition/v1beta1" + "github.com/stretchr/testify/assert" + "k8s.io/utils/ptr" +) + +func np(proto string, p int32) v1beta1.NetworkPort { + return v1beta1.NetworkPort{Protocol: v1beta1.Protocol(proto), Port: ptr.To(p)} +} + +func TestExtractAddrPorts_ZeroPortIsALiteralNotAWildcard(t *testing.T) { + groups := ExtractAddrPorts([]v1beta1.NetworkNeighbor{ + {IPAddresses: []string{"10.0.5.9"}, Ports: []v1beta1.NetworkPort{np("TCP", 0), np("UDP", 53)}}, + }) + assert.Len(t, groups, 1) + assert.NotNil(t, groups[0].Ports, "a zero-port entry must not collapse the entry to fully open") + assert.Contains(t, groups[0].Ports, PortKey("TCP", 0)) + assert.Contains(t, groups[0].Ports, PortKey("UDP", 53)) + assert.Len(t, groups[0].Ports, 2) +} + +func TestExtractAddrPorts_AbsentStanzaIsTheOnlyWildcard(t *testing.T) { + groups := ExtractAddrPorts([]v1beta1.NetworkNeighbor{ + {IPAddresses: []string{"93.184.216.34"}}, + }) + assert.Len(t, groups, 1) + assert.Nil(t, groups[0].Ports) +} + +func TestExtractAddrPorts_NilPortEntryContributesNothing(t *testing.T) { + groups := ExtractAddrPorts([]v1beta1.NetworkNeighbor{ + {IPAddresses: []string{"10.1.2.3"}, Ports: []v1beta1.NetworkPort{{Protocol: "TCP", Port: nil}, np("UDP", 53)}}, + }) + assert.Len(t, groups, 1) + assert.NotNil(t, groups[0].Ports) + assert.NotContains(t, groups[0].Ports, PortKey("TCP", 0)) + assert.Contains(t, groups[0].Ports, PortKey("UDP", 53)) + assert.Len(t, groups[0].Ports, 1) +} diff --git a/pkg/objectcache/containerprofilecache/containerprofilecache.go b/pkg/objectcache/containerprofilecache/containerprofilecache.go index 2a5394d18e..0c2b1574a3 100644 --- a/pkg/objectcache/containerprofilecache/containerprofilecache.go +++ b/pkg/objectcache/containerprofilecache/containerprofilecache.go @@ -16,6 +16,7 @@ import ( helpersv1 "github.com/kubescape/k8s-interface/instanceidhandler/v1/helpers" "github.com/kubescape/node-agent/pkg/config" "github.com/kubescape/node-agent/pkg/metricsmanager" + "github.com/kubescape/node-agent/pkg/networkpeer" "github.com/kubescape/node-agent/pkg/objectcache" "github.com/kubescape/node-agent/pkg/objectcache/callstackcache" "github.com/kubescape/node-agent/pkg/resourcelocks" @@ -58,6 +59,15 @@ type CachedContainerProfile struct { State *objectcache.ProfileState CallStackTree *callstackcache.CallStackSearchTree + // UsesServiceResolution is true when the profile declares serviceRef/ + // serviceSelector/entity neighbors, so its projected addresses depend on the + // live cluster view. ListerGen is the service-lister generation captured at + // projection time; the reconciler re-projects such entries when the + // generation advances (endpoint churn, or caches that filled after the + // initial projection). Non-resolving profiles keep the RV/spec fast-skip. + UsesServiceResolution bool + ListerGen int64 + ContainerName string PodName string Namespace string @@ -111,6 +121,10 @@ type ContainerProfileCacheImpl struct { containerLocks *resourcelocks.ResourceLocks storageClient storage.ProfileClient k8sObjectCache objectcache.K8sObjectCache + // serviceLister resolves serviceRef/serviceSelector/entity network + // neighbors to concrete IPs at projection time. nil = feature off (the + // profile projects unchanged), which is what every unit test leaves it as. + serviceLister networkpeer.Lister metricsManager metricsmanager.MetricsManager reconcileEvery time.Duration @@ -174,6 +188,22 @@ func NewContainerProfileCache(cfg config.Config, storageClient storage.ProfileCl return c } +// SetServiceLister installs the cluster view used to resolve +// serviceRef/serviceSelector/entity network neighbors at projection time. It +// is optional: when unset the projection leaves such neighbors unresolved. +func (c *ContainerProfileCacheImpl) SetServiceLister(l networkpeer.Lister) { + c.serviceLister = l +} + +// listerGen returns the current service-lister generation, or 0 when no lister +// is installed (unit tests, or the feature is off). +func (c *ContainerProfileCacheImpl) listerGen() int64 { + if c.serviceLister == nil { + return 0 + } + return c.serviceLister.Generation() +} + // refreshRPC calls fn with a context bounded by c.rpcBudget, enforcing a // per-call SLO so a slow API server cannot stall a full reconciler burst. func (c *ContainerProfileCacheImpl) refreshRPC(ctx context.Context, fn func(context.Context) error) error { @@ -579,9 +609,15 @@ func (c *ContainerProfileCacheImpl) buildEntry( } entry.CallStackTree = tree - // Project under the current spec. + // Project under the current spec, resolving any serviceRef/entity network + // neighbors to concrete IPs first. Record whether this profile depends on + // the live cluster view and the lister generation it was resolved against, + // so the reconciler re-projects it when that view changes. spec := c.snapshotSpec() - projected := Apply(spec, userMerged, tree) + entry.UsesServiceResolution = networkpeer.HasServiceNeighbors(userMerged) + entry.ListerGen = c.listerGen() + projected := Apply(spec, networkpeer.WithResolvedServiceNeighbors(userMerged, c.serviceLister), tree) + projected.ResolvedGen = entry.ListerGen entry.Projected = projected entry.SpecHash = projected.SpecHash diff --git a/pkg/objectcache/containerprofilecache/projection_apply.go b/pkg/objectcache/containerprofilecache/projection_apply.go index f0de872a52..60e7fb32bf 100644 --- a/pkg/objectcache/containerprofilecache/projection_apply.go +++ b/pkg/objectcache/containerprofilecache/projection_apply.go @@ -51,6 +51,10 @@ func Apply(spec *objectcache.RuleProjectionSpec, cp *v1beta1.ContainerProfile, c pcp.Execs = projectField(s.Execs, execsPaths, true) pcp.ExecsByPath = extractExecsByPath(cp) + pcp.Namespace = cp.Namespace + pcp.IngressPeers = extractIngressPeers(cp) + pcp.EgressPeers = extractEgressPeers(cp) + endpointPaths := extractEndpointPaths(cp) pcp.Endpoints = projectField(s.Endpoints, endpointPaths, true) @@ -63,6 +67,9 @@ func Apply(spec *objectcache.RuleProjectionSpec, cp *v1beta1.ContainerProfile, c pcp.IngressDomains = projectField(s.IngressDomains, extractIngressDomains(cp), false) pcp.IngressAddresses = projectField(s.IngressAddresses, extractIngressAddresses(cp), false) + pcp.EgressAddrPorts = objectcache.ExtractAddrPorts(cp.Spec.Egress) + pcp.IngressAddrPorts = objectcache.ExtractAddrPorts(cp.Spec.Ingress) + return pcp } @@ -260,3 +267,29 @@ func extractIngressAddresses(cp *v1beta1.ContainerProfile) []string { } return addrs } + +// extractIngressPeers / extractEgressPeers carry the label selectors of each +// network-neighbor entry so cp.was_selector_in_{ingress,egress} can match a +// peer by identity. Only entries that actually declare a podSelector are kept. +func extractIngressPeers(cp *v1beta1.ContainerProfile) []objectcache.PeerSelector { + return extractPeers(cp.Spec.Ingress) +} + +func extractEgressPeers(cp *v1beta1.ContainerProfile) []objectcache.PeerSelector { + return extractPeers(cp.Spec.Egress) +} + +func extractPeers(neighbors []v1beta1.NetworkNeighbor) []objectcache.PeerSelector { + var peers []objectcache.PeerSelector + for i := range neighbors { + n := &neighbors[i] + if n.PodSelector == nil { + continue + } + peers = append(peers, objectcache.PeerSelector{ + PodSelector: n.PodSelector, + NamespaceSelector: n.NamespaceSelector, + }) + } + return peers +} diff --git a/pkg/objectcache/containerprofilecache/projection_golden_test.go b/pkg/objectcache/containerprofilecache/projection_golden_test.go index 750d120bcb..1450907e29 100644 --- a/pkg/objectcache/containerprofilecache/projection_golden_test.go +++ b/pkg/objectcache/containerprofilecache/projection_golden_test.go @@ -65,6 +65,8 @@ type projectionGolden struct { EgressAddresses objectcache.ProjectedField `json:"egressAddresses"` IngressDomains objectcache.ProjectedField `json:"ingressDomains"` IngressAddresses objectcache.ProjectedField `json:"ingressAddresses"` + IngressPeers []objectcache.PeerSelector `json:"ingressPeers"` + EgressPeers []objectcache.PeerSelector `json:"egressPeers"` ExecsByPath map[string][][]string `json:"execsByPath"` PolicyByRuleId map[string]v1beta1.RulePolicy `json:"policyByRuleId"` CallStacks []callStackSummary `json:"callStacks"` @@ -93,6 +95,8 @@ func toGolden(pcp *objectcache.ProjectedContainerProfile, tree *callstackcache.C EgressAddresses: pcp.EgressAddresses, IngressDomains: pcp.IngressDomains, IngressAddresses: pcp.IngressAddresses, + IngressPeers: pcp.IngressPeers, + EgressPeers: pcp.EgressPeers, ExecsByPath: pcp.ExecsByPath, PolicyByRuleId: pcp.PolicyByRuleId, } @@ -236,9 +240,12 @@ func networkProfile() *v1beta1.ContainerProfile { Ingress: []v1beta1.NetworkNeighbor{ {Identifier: "in-a", DNS: "old.internal", DNSNames: []string{"a.internal", "b.internal"}, IPAddresses: []string{"192.168.1.10", "192.168.0.0/16"}}, {Identifier: "in-b", IPAddresses: []string{wild}}, + {Identifier: "in-c", PodSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "redis-client"}}}, + {Identifier: "in-d", PodSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "probe"}}, NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/metadata.name": "monitoring"}}}, }, Egress: []v1beta1.NetworkNeighbor{ {Identifier: "eg-a", DNSNames: []string{"c.example.com"}, IPAddress: "203.0.113.7", IPAddresses: []string{"203.0.113.0/24", wild}}, + {Identifier: "eg-b", PodSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"app": "upstream"}}}, }, }, } diff --git a/pkg/objectcache/containerprofilecache/reconciler.go b/pkg/objectcache/containerprofilecache/reconciler.go index bd7517c728..d394b0c61c 100644 --- a/pkg/objectcache/containerprofilecache/reconciler.go +++ b/pkg/objectcache/containerprofilecache/reconciler.go @@ -23,6 +23,7 @@ import ( "github.com/kubescape/go-logger" "github.com/kubescape/go-logger/helpers" helpersv1 "github.com/kubescape/k8s-interface/instanceidhandler/v1/helpers" + "github.com/kubescape/node-agent/pkg/networkpeer" "github.com/kubescape/node-agent/pkg/objectcache" "github.com/kubescape/node-agent/pkg/objectcache/callstackcache" "github.com/kubescape/node-agent/pkg/utils" @@ -418,9 +419,14 @@ func (c *ContainerProfileCacheImpl) refreshOneEntry(ctx context.Context, id stri if spec := c.snapshotSpec(); spec != nil { currentSpecHash = spec.Hash } + // serviceRef/entity profiles must also re-project when the cluster view + // changed since they were resolved (endpoint churn, or caches that filled + // after projection). Non-resolving profiles ignore the lister generation and + // keep the cheap RV/spec fast-skip. if rvsMatchCP(cp, e.RV) && rvsMatchCP(userDefinedCP, e.UserCPRV) && - e.SpecHash == currentSpecHash { + e.SpecHash == currentSpecHash && + (!e.UsesServiceResolution || e.ListerGen == c.listerGen()) { return } @@ -489,28 +495,33 @@ func (c *ContainerProfileCacheImpl) rebuildEntryFromSources( // Project under the current spec. spec := c.snapshotSpec() + // Read gen BEFORE resolving so a concurrent Bump() invalidates this projection. + gen := c.listerGen() applyStart := time.Now() - projectedCP := Apply(spec, projected, tree) + projectedCP := Apply(spec, networkpeer.WithResolvedServiceNeighbors(projected, c.serviceLister), tree) + projectedCP.ResolvedGen = gen if c.cfg.ProfileProjection.DetailedMetricsEnabled { c.metricsManager.ObserveProjectionApplyDuration(time.Since(applyStart)) c.observeMemoryMetrics(projected, projectedCP) } newEntry := &CachedContainerProfile{ - Projected: projectedCP, - SpecHash: projectedCP.SpecHash, - State: &objectcache.ProfileState{Completion: effectiveCP.Annotations[helpersv1.CompletionMetadataKey], Status: effectiveCP.Annotations[helpersv1.StatusMetadataKey], Name: effectiveCP.Name}, - CallStackTree: tree, - ContainerName: prev.ContainerName, - PodName: prev.PodName, - Namespace: prev.Namespace, - PodUID: podUID, - WorkloadID: prev.WorkloadID, - CPName: prev.CPName, - WorkloadName: prev.WorkloadName, - RV: rvOfCP(cp), - UserCPRV: rvOfCP(userDefinedCP), - terminatedSeenAt: prev.terminatedSeenAt, + Projected: projectedCP, + SpecHash: projectedCP.SpecHash, + UsesServiceResolution: networkpeer.HasServiceNeighbors(projected), + ListerGen: gen, + State: &objectcache.ProfileState{Completion: effectiveCP.Annotations[helpersv1.CompletionMetadataKey], Status: effectiveCP.Annotations[helpersv1.StatusMetadataKey], Name: effectiveCP.Name}, + CallStackTree: tree, + ContainerName: prev.ContainerName, + PodName: prev.PodName, + Namespace: prev.Namespace, + PodUID: podUID, + WorkloadID: prev.WorkloadID, + CPName: prev.CPName, + WorkloadName: prev.WorkloadName, + RV: rvOfCP(cp), + UserCPRV: rvOfCP(userDefinedCP), + terminatedSeenAt: prev.terminatedSeenAt, } if userDefinedCP != nil { // The user-authored CP is authoritative and complete by definition (no diff --git a/pkg/objectcache/containerprofilecache/resolvedgen_test.go b/pkg/objectcache/containerprofilecache/resolvedgen_test.go new file mode 100644 index 0000000000..7abdc844d1 --- /dev/null +++ b/pkg/objectcache/containerprofilecache/resolvedgen_test.go @@ -0,0 +1,48 @@ +package containerprofilecache + +import ( + "strconv" + "testing" + + "github.com/kubescape/node-agent/pkg/networkpeer" + "github.com/kubescape/node-agent/pkg/objectcache" +) + +// genLister is a cluster view whose generation the test drives directly. +type genLister struct{ gen int64 } + +func (g *genLister) ServiceByName(string, string) (*networkpeer.ServiceInfo, bool) { return nil, false } +func (g *genLister) ServicesByLabels(map[string]string, map[string]string) []*networkpeer.ServiceInfo { + return nil +} +func (g *genLister) HostIPs() []string { return nil } +func (g *genLister) Generation() int64 { return g.gen } + +// TestProjectedResolvedGenFeedsCacheKey: the CEL result cache keys on the +// projected profile's SpecHash+SyncChecksum+ResolvedGen. Re-resolving against a +// moved cluster view changes neither of the first two — an authored profile +// carries no SyncChecksum at all — so without ResolvedGen a result computed +// before the informers filled (e.g. "this address is not in egress") would be +// served from cache forever, and the profile's own re-projection would never +// take effect. +func TestProjectedResolvedGenFeedsCacheKey(t *testing.T) { + l := &genLister{gen: 7} + c := &ContainerProfileCacheImpl{} + c.SetServiceLister(l) + + if got := c.listerGen(); got != 7 { + t.Fatalf("listerGen: got %d want 7", got) + } + + key := func(p *objectcache.ProjectedContainerProfile) string { + return p.SpecHash + "|" + p.SyncChecksum + "|" + strconv.FormatInt(p.ResolvedGen, 10) + } + before := &objectcache.ProjectedContainerProfile{SpecHash: "spec", ResolvedGen: c.listerGen()} + + l.gen = 8 + after := &objectcache.ProjectedContainerProfile{SpecHash: "spec", ResolvedGen: c.listerGen()} + + if key(before) == key(after) { + t.Errorf("cache key must change when the profile is re-resolved against a moved cluster view") + } +} diff --git a/pkg/objectcache/containerprofilecache/testdata/golden/network_all.json b/pkg/objectcache/containerprofilecache/testdata/golden/network_all.json index 3b833c5cfd..8607a33268 100644 --- a/pkg/objectcache/containerprofilecache/testdata/golden/network_all.json +++ b/pkg/objectcache/containerprofilecache/testdata/golden/network_all.json @@ -78,6 +78,38 @@ "PrefixHits": {}, "SuffixHits": {} }, + "ingressPeers": [ + { + "PodSelector": { + "matchLabels": { + "app": "redis-client" + } + }, + "NamespaceSelector": null + }, + { + "PodSelector": { + "matchLabels": { + "app": "probe" + } + }, + "NamespaceSelector": { + "matchLabels": { + "kubernetes.io/metadata.name": "monitoring" + } + } + } + ], + "egressPeers": [ + { + "PodSelector": { + "matchLabels": { + "app": "upstream" + } + }, + "NamespaceSelector": null + } + ], "execsByPath": null, "policyByRuleId": null, "callStacks": null diff --git a/pkg/objectcache/containerprofilecache/testdata/golden/rich_filtered.json b/pkg/objectcache/containerprofilecache/testdata/golden/rich_filtered.json index dccf5126c1..71fb76abb4 100644 --- a/pkg/objectcache/containerprofilecache/testdata/golden/rich_filtered.json +++ b/pkg/objectcache/containerprofilecache/testdata/golden/rich_filtered.json @@ -103,6 +103,8 @@ "PrefixHits": {}, "SuffixHits": {} }, + "ingressPeers": null, + "egressPeers": null, "execsByPath": { "/bin/curl": [ [ diff --git a/pkg/objectcache/containerprofilecache/testdata/golden/rich_passthrough.json b/pkg/objectcache/containerprofilecache/testdata/golden/rich_passthrough.json index a343f73b67..13eef20a98 100644 --- a/pkg/objectcache/containerprofilecache/testdata/golden/rich_passthrough.json +++ b/pkg/objectcache/containerprofilecache/testdata/golden/rich_passthrough.json @@ -97,6 +97,8 @@ "PrefixHits": {}, "SuffixHits": {} }, + "ingressPeers": null, + "egressPeers": null, "execsByPath": { "/bin/curl": [ [ diff --git a/pkg/objectcache/projection_types.go b/pkg/objectcache/projection_types.go index 8d452c7a36..b5ea376a9a 100644 --- a/pkg/objectcache/projection_types.go +++ b/pkg/objectcache/projection_types.go @@ -1,10 +1,25 @@ package objectcache import ( + "strconv" + "strings" + "github.com/kubescape/node-agent/pkg/objectcache/callstackcache" "github.com/kubescape/storage/pkg/apis/softwarecomposition/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +// PeerSelector carries a single network-neighbor entry's identity selectors +// (podSelector + namespaceSelector) through the projection so the +// cp.was_selector_in_{ingress,egress} CEL helpers can resolve a runtime peer +// IP to a pod and match it by LABEL rather than by (volatile) IP. The address +// surfaces (Ingress/EgressAddresses) still carry the ipAddress/CIDR form for +// the was_address_in_* helpers; these are complementary. +type PeerSelector struct { + PodSelector *metav1.LabelSelector + NamespaceSelector *metav1.LabelSelector +} + // PathMatcher is implemented by the trie-based matchers in containerprofilecache. type PathMatcher interface { HasMatch(s string) bool @@ -41,9 +56,57 @@ type FieldSpec struct { SuffixMatcher PathMatcher } +// AddrPortGroup pairs one neighbor entry's addresses with its allowed ports. +// Ports == nil means the neighbor declared no ports stanza (indistinguishable +// from an empty one after a storage round-trip) and matches any port; a +// non-empty map matches only its literal (protocol, port) keys. +type AddrPortGroup struct { + Addrs []string + Ports map[string]struct{} +} + +func PortKey(protocol string, port int32) string { + return strings.ToUpper(protocol) + "-" + strconv.Itoa(int(port)) +} + +func ExtractAddrPorts(neighbors []v1beta1.NetworkNeighbor) []AddrPortGroup { + var groups []AddrPortGroup + for i := range neighbors { + n := &neighbors[i] + var addrs []string + if n.IPAddress != "" { + addrs = append(addrs, n.IPAddress) + } + addrs = append(addrs, n.IPAddresses...) + if len(addrs) == 0 { + continue + } + // The only port wildcard is an absent (or empty — protobuf cannot tell + // them apart) ports stanza. A listed entry always restricts: an explicit + // port (0 included) is a literal, a nil port contributes nothing. + ports := make(map[string]struct{}, len(n.Ports)) + for _, p := range n.Ports { + if p.Port == nil { + continue + } + ports[PortKey(string(p.Protocol), *p.Port)] = struct{}{} + } + if len(n.Ports) == 0 { + ports = nil + } + groups = append(groups, AddrPortGroup{Addrs: addrs, Ports: ports}) + } + return groups +} + // ProjectedContainerProfile is the cache-resident compact form. Pure node-agent // internal type; never serialized. Replaces *v1beta1.ContainerProfile in the cache. type ProjectedContainerProfile struct { + // Namespace is the profiled workload's own namespace; a peer entry whose + // NamespaceSelector is nil matches only peers in this namespace (the learned + // encoding and the NetworkPolicyPeer semantic for an absent namespaceSelector). + Namespace string + Opens ProjectedField Execs ProjectedField Endpoints ProjectedField @@ -54,6 +117,18 @@ type ProjectedContainerProfile struct { IngressDomains ProjectedField IngressAddresses ProjectedField + // IngressPeers / EgressPeers carry the podSelector+namespaceSelector of each + // network-neighbor entry (dropped by the address/domain projection) so the + // cp.was_selector_in_{ingress,egress} helpers can match a runtime peer by + // label. Always projected in full (not gated by a rule surface) since they + // are small and only populated when the profile actually declares selectors. + IngressPeers []PeerSelector + EgressPeers []PeerSelector + + // IngressAddrPorts / EgressAddrPorts group each neighbor's addresses with its ports for was_address_port_protocol_in_*. + IngressAddrPorts []AddrPortGroup + EgressAddrPorts []AddrPortGroup + // ExecsByPath carries the per-Path Args slices from cp.Spec.Execs so // downstream consumers (e.g. dynamicpathdetector.CompareExecArgs used // by R0040 in node-agent#807) can run wildcard-aware argv matching @@ -67,7 +142,14 @@ type ProjectedContainerProfile struct { // constraint" (back-compat for pre-projection profiles). ExecsByPath map[string][][]string - SpecHash string + SpecHash string + // ResolvedGen is the cluster-view generation the profile's serviceRef/ + // serviceSelector/entity neighbors were resolved against. It participates in + // the CEL result-cache key: re-projecting against a moved cluster view + // changes the projected addresses without touching SpecHash or SyncChecksum, + // so a result cached before the informers filled would otherwise be served + // forever. Zero for profiles that resolve nothing. + ResolvedGen int64 SyncChecksum string PolicyByRuleId map[string]v1beta1.RulePolicy CallStackTree *callstackcache.CallStackSearchTree diff --git a/pkg/objectcache/v1/mock.go b/pkg/objectcache/v1/mock.go index 789eccb9ec..5066b933cc 100644 --- a/pkg/objectcache/v1/mock.go +++ b/pkg/objectcache/v1/mock.go @@ -193,6 +193,9 @@ func (r *RuleObjectCacheMock) GetProjectedContainerProfile(containerID string) * } } + pcp.EgressAddrPorts = objectcache.ExtractAddrPorts(cp.Spec.Egress) + pcp.IngressAddrPorts = objectcache.ExtractAddrPorts(cp.Spec.Ingress) + return pcp } diff --git a/pkg/rulemanager/cel/libraries/cache/function_cache.go b/pkg/rulemanager/cel/libraries/cache/function_cache.go index ba07eafcd3..22af6beed6 100644 --- a/pkg/rulemanager/cel/libraries/cache/function_cache.go +++ b/pkg/rulemanager/cel/libraries/cache/function_cache.go @@ -2,6 +2,7 @@ package cache import ( "fmt" + "strconv" "strings" "time" @@ -101,8 +102,10 @@ func HashForContainerProfile(oc objectcache.ObjectCache) func([]ref.Val) string } // Include SyncChecksum so the key changes when profile content is updated // under the same projection spec, preventing stale cached results after - // the profile learns new paths/execs/etc. - return pcp.SpecHash + "|" + pcp.SyncChecksum + // the profile learns new paths/execs/etc. ResolvedGen covers the same + // hazard for serviceRef/entity neighbors, whose projected addresses move + // with the cluster view while both other components stay put. + return pcp.SpecHash + "|" + pcp.SyncChecksum + "|" + strconv.FormatInt(pcp.ResolvedGen, 10) } } diff --git a/pkg/rulemanager/cel/libraries/containerprofilenetwork/containerprofilenetwork.go b/pkg/rulemanager/cel/libraries/containerprofilenetwork/containerprofilenetwork.go index 58058c2aed..d9c7a2938d 100644 --- a/pkg/rulemanager/cel/libraries/containerprofilenetwork/containerprofilenetwork.go +++ b/pkg/rulemanager/cel/libraries/containerprofilenetwork/containerprofilenetwork.go @@ -83,6 +83,9 @@ type containerProfileNetworkFuncSpec struct { arity int // call invokes the shared implementation method on l. call func(l *containerProfileNetworkLibrary, args []ref.Val) ref.Val + // noCache bypasses the functionCache: a map argument has no stable scalar + // cache key, and the selector match is cheap (O(selectors)). + noCache bool } var containerProfileNetworkFuncSpecs = []containerProfileNetworkFuncSpec{ @@ -140,6 +143,26 @@ var containerProfileNetworkFuncSpecs = []containerProfileNetworkFuncSpec{ return l.wasAddressPortProtocolInIngress(a[0], a[1], a[2], a[3]) }, }, + { + name: "was_selector_in_egress", + argTypes: []*cel.Type{cel.StringType, cel.StringType, cel.MapType(cel.StringType, cel.StringType)}, + resultType: cel.BoolType, + arity: 3, + call: func(l *containerProfileNetworkLibrary, a []ref.Val) ref.Val { + return l.wasSelectorInEgress(a[0], a[1], a[2]) + }, + noCache: true, + }, + { + name: "was_selector_in_ingress", + argTypes: []*cel.Type{cel.StringType, cel.StringType, cel.MapType(cel.StringType, cel.StringType)}, + resultType: cel.BoolType, + arity: 3, + call: func(l *containerProfileNetworkLibrary, a []ref.Val) ref.Val { + return l.wasSelectorInIngress(a[0], a[1], a[2]) + }, + noCache: true, + }, } // declarationsWithPrefix builds the cel.FunctionOpt map for every function in @@ -165,6 +188,9 @@ func (l *containerProfileNetworkLibrary) declarationsWithPrefix(namePrefix, over if l.detailedMetrics && l.metrics != nil { l.metrics.IncHelperCall(fullName) } + if spec.noCache { + return cache.ConvertProfileNotAvailableErrToBool(spec.call(l, values), false) + } wrapperFunc := func(args ...ref.Val) ref.Val { return spec.call(l, args) } @@ -270,6 +296,9 @@ func (e *containerProfileNetworkCostEstimator) EstimateCallCost(function, overlo case "cp.is_domain_in_egress", "cp.is_domain_in_ingress": // Cache lookup + O(n) list iteration + O(m) slice.Contains on DNS names per entry cost = 35 + case "cp.was_selector_in_egress", "cp.was_selector_in_ingress": + // O(selectors) label-set match per peer entry + cost = 30 case "cp.was_address_port_protocol_in_egress", "cp.was_address_port_protocol_in_ingress": // Cache lookup + O(n) address search + O(p) nested port/protocol matching cost = 45 diff --git a/pkg/rulemanager/cel/libraries/containerprofilenetwork/integration_test.go b/pkg/rulemanager/cel/libraries/containerprofilenetwork/integration_test.go index e515a5fd73..81143c0133 100644 --- a/pkg/rulemanager/cel/libraries/containerprofilenetwork/integration_test.go +++ b/pkg/rulemanager/cel/libraries/containerprofilenetwork/integration_test.go @@ -209,28 +209,24 @@ func TestIntegrationWithAllNetworkFunctions(t *testing.T) { expectedResult: true, }, { - // v1 degradation: port/protocol projection is out of scope; address IS in profile → true. name: "Check non-existent egress address with port and protocol", expression: `cp.was_address_port_protocol_in_egress(containerID, "192.168.1.100", 9999, "TCP")`, - expectedResult: true, + expectedResult: false, }, { - // v1 degradation: port/protocol projection is out of scope; address IS in profile → true. name: "Check non-existent ingress address with port and protocol", expression: `cp.was_address_port_protocol_in_ingress(containerID, "172.16.0.10", 9999, "TCP")`, - expectedResult: true, + expectedResult: false, }, { - // v1 degradation: port/protocol projection is out of scope; address IS in profile → true. name: "Check wrong protocol for existing address and port", expression: `cp.was_address_port_protocol_in_egress(containerID, "192.168.1.100", 80, "UDP")`, - expectedResult: true, + expectedResult: false, }, { - // v1 degradation: port/protocol projection is out of scope; address IS in profile → true. name: "Check wrong protocol for existing ingress address and port", expression: `cp.was_address_port_protocol_in_ingress(containerID, "172.16.0.10", 8080, "UDP")`, - expectedResult: true, + expectedResult: false, }, { name: "Complex network check with port and protocol - egress", @@ -243,10 +239,9 @@ func TestIntegrationWithAllNetworkFunctions(t *testing.T) { expectedResult: true, }, { - // v1 degradation: both sides match on address only → true. name: "Mixed valid and invalid port protocol checks", expression: `cp.was_address_port_protocol_in_egress(containerID, "192.168.1.100", 80, "TCP") && cp.was_address_port_protocol_in_egress(containerID, "192.168.1.100", 9999, "TCP")`, - expectedResult: true, + expectedResult: false, }, } diff --git a/pkg/rulemanager/cel/libraries/containerprofilenetwork/network.go b/pkg/rulemanager/cel/libraries/containerprofilenetwork/network.go index 3d97e85f2e..90baa3fa41 100644 --- a/pkg/rulemanager/cel/libraries/containerprofilenetwork/network.go +++ b/pkg/rulemanager/cel/libraries/containerprofilenetwork/network.go @@ -2,6 +2,7 @@ package containerprofilenetwork import ( "net" + "reflect" "strings" "github.com/google/cel-go/common/types" @@ -10,6 +11,8 @@ import ( "github.com/kubescape/node-agent/pkg/rulemanager/cel/libraries/cache" "github.com/kubescape/node-agent/pkg/rulemanager/profilehelper" "github.com/kubescape/storage/pkg/registry/file/networkmatch" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" ) // matchIPField is the wildcard-aware adapter from the projection layer's @@ -59,6 +62,29 @@ func matchIPField(field *objectcache.ProjectedField, observed string) bool { return networkmatch.MatchIP(entries, observed) } +// matchAddrPort reports whether observed (address, protocol, port) falls within +// any single neighbor entry: its addresses match AND the entry allows the port +// (nil Ports = no ports stanza = any port; a populated map matches literal keys only). +func matchAddrPort(groups []objectcache.AddrPortGroup, address, protocol string, port int32) bool { + if address == "" { + return false + } + key := objectcache.PortKey(protocol, port) + for i := range groups { + g := &groups[i] + if !networkmatch.MatchIP(g.Addrs, address) { + continue + } + if g.Ports == nil { + return true + } + if _, ok := g.Ports[key]; ok { + return true + } + } + return false +} + func matchDNSField(field *objectcache.ProjectedField, observed string) bool { if observed == "" || field == nil { return false @@ -171,9 +197,6 @@ func (l *containerProfileNetworkLibrary) wasAddressPortProtocolInEgress(containe if !ok { return types.MaybeNoSuchOverloadErr(address) } - // port/protocol projection (AddressPortsByAddr) is out of scope for the - // projection-v1 layer upstream landed; matchers degrade to address-only. - // Wildcards remain enforced via matchIPField. portInt, ok := port.Value().(int64) if !ok { return types.MaybeNoSuchOverloadErr(port) @@ -181,14 +204,15 @@ func (l *containerProfileNetworkLibrary) wasAddressPortProtocolInEgress(containe if portInt < 0 || portInt > 65535 { return types.Bool(false) } - if _, ok := protocol.Value().(string); !ok { + protocolStr, ok := protocol.Value().(string) + if !ok { return types.MaybeNoSuchOverloadErr(protocol) } cp, _, err := profilehelper.GetProjectedContainerProfile(l.objectCache, containerIDStr) if err != nil { return cache.NewProfileNotAvailableErr("%v", err) } - return types.Bool(matchIPField(&cp.EgressAddresses, addressStr)) + return types.Bool(matchAddrPort(cp.EgressAddrPorts, addressStr, protocolStr, int32(portInt))) } func (l *containerProfileNetworkLibrary) wasAddressPortProtocolInIngress(containerID, address, port, protocol ref.Val) ref.Val { @@ -210,12 +234,121 @@ func (l *containerProfileNetworkLibrary) wasAddressPortProtocolInIngress(contain if portInt < 0 || portInt > 65535 { return types.Bool(false) } - if _, ok := protocol.Value().(string); !ok { + protocolStr, ok := protocol.Value().(string) + if !ok { return types.MaybeNoSuchOverloadErr(protocol) } cp, _, err := profilehelper.GetProjectedContainerProfile(l.objectCache, containerIDStr) if err != nil { return cache.NewProfileNotAvailableErr("%v", err) } - return types.Bool(matchIPField(&cp.IngressAddresses, addressStr)) + return types.Bool(matchAddrPort(cp.IngressAddrPorts, addressStr, protocolStr, int32(portInt))) +} + +// namespaceSelectorMatches matches a namespaceSelector against the peer's +// namespace via the implicit kubernetes.io/metadata.name label every namespace +// carries (the form these profiles use). A nil selector matches only the +// profiled workload's own namespace: the learned generator omits the selector +// exactly for same-namespace peers, and NetworkPolicyPeer gives an absent +// namespaceSelector the same meaning. Selectors keyed on other namespace +// labels are not resolved here. +// namespaceSelectorMatches: a nil namespaceSelector does NOT consult the +// namespace — matching is on pod labels alone, and namespace is only used to +// disambiguate a label collision (an explicitly-set selector). profileNs is +// unused now but kept in the signature for the collision case. +func namespaceSelectorMatches(sel *metav1.LabelSelector, ns, profileNs string) bool { + if sel == nil { + return true + } + s, err := metav1.LabelSelectorAsSelector(sel) + if err != nil { + return false + } + return s.Matches(labels.Set{"kubernetes.io/metadata.name": ns}) +} + +// wasSelectorInPeers reports whether the peer identified by (podLabels, ns) +// matches any peer entry's podSelector AND its namespaceSelector. An empty +// podSelector matches NOTHING (fail closed → the peer alerts), the opposite of +// NetworkPolicy's match-all: an allowlist entry must name what it permits. +func wasSelectorInPeers(peers []objectcache.PeerSelector, podLabels labels.Set, ns, profileNs string) bool { + for i := range peers { + peer := &peers[i] + if peer.PodSelector == nil || + (len(peer.PodSelector.MatchLabels) == 0 && len(peer.PodSelector.MatchExpressions) == 0) { + continue + } + ps, err := metav1.LabelSelectorAsSelector(peer.PodSelector) + if err != nil { + continue + } + if ps.Matches(podLabels) && namespaceSelectorMatches(peer.NamespaceSelector, ns, profileNs) { + return true + } + } + return false +} + +func (l *containerProfileNetworkLibrary) wasSelectorInIngress(containerID, namespace, podLabels ref.Val) ref.Val { + return l.wasSelectorIn(containerID, namespace, podLabels, true) +} + +func (l *containerProfileNetworkLibrary) wasSelectorInEgress(containerID, namespace, podLabels ref.Val) ref.Val { + return l.wasSelectorIn(containerID, namespace, podLabels, false) +} + +// wasSelectorIn reports whether the runtime peer — identified by the namespace +// and pod labels that Inspektor Gadget's kubeipresolver stamps onto the network +// event — matches any of the profile's ingress-or-egress peer selectors. +// +// Matching on the peer's identity (namespace + labels) rather than its IP is the +// whole point: it is stable across pod IP churn AND works across nodes, because +// kubeipresolver resolves the peer against a cluster-wide pod inventory before +// the event ever reaches CEL. There is deliberately no IP→pod lookup here — that +// would reintroduce a dependency on node-agent's node-local pod cache, which is +// exactly what breaks cross-node peers. +func (l *containerProfileNetworkLibrary) wasSelectorIn(containerID, namespace, podLabels ref.Val, ingress bool) ref.Val { + if l.objectCache == nil { + return types.NewErr("objectCache is nil") + } + containerIDStr, ok := containerID.Value().(string) + if !ok { + return types.MaybeNoSuchOverloadErr(containerID) + } + nsStr, ok := namespace.Value().(string) + if !ok { + return types.MaybeNoSuchOverloadErr(namespace) + } + if nsStr == "" { + // The peer did not resolve to a pod (external IP, or the resolver had no + // inventory entry): a nil peer never satisfies a selector — it alerts. + return types.Bool(false) + } + peerLabels := refValToStringMap(podLabels) + cp, _, err := profilehelper.GetProjectedContainerProfile(l.objectCache, containerIDStr) + if err != nil { + return cache.NewProfileNotAvailableErr("%v", err) + } + peers := cp.EgressPeers + if ingress { + peers = cp.IngressPeers + } + if len(peers) == 0 { + return types.Bool(false) + } + return types.Bool(wasSelectorInPeers(peers, labels.Set(peerLabels), nsStr, cp.Namespace)) +} + +// refValToStringMap converts a CEL map argument to a Go map[string]string. A nil +// or non-map value yields nil (treated as "peer has no labels"). +func refValToStringMap(v ref.Val) map[string]string { + if v == nil { + return nil + } + native, err := v.ConvertToNative(reflect.TypeOf(map[string]string(nil))) + if err != nil { + return nil + } + m, _ := native.(map[string]string) + return m } diff --git a/pkg/rulemanager/cel/libraries/containerprofilenetwork/network_test.go b/pkg/rulemanager/cel/libraries/containerprofilenetwork/network_test.go index 10321073cc..e94ac732e7 100644 --- a/pkg/rulemanager/cel/libraries/containerprofilenetwork/network_test.go +++ b/pkg/rulemanager/cel/libraries/containerprofilenetwork/network_test.go @@ -99,22 +99,20 @@ func TestWasAddressPortProtocolInEgress(t *testing.T) { expectedResult: true, }, { - // v1 degradation: port/protocol projection is out of scope; address-only matching. name: "Invalid port", containerID: "test-container-id", address: "192.168.1.100", port: 9999, protocol: "TCP", - expectedResult: true, + expectedResult: false, }, { - // v1 degradation: port/protocol projection is out of scope; address-only matching. name: "Invalid protocol", containerID: "test-container-id", address: "192.168.1.100", port: 80, protocol: "UDP", - expectedResult: true, + expectedResult: false, }, { name: "Invalid address", @@ -235,22 +233,20 @@ func TestWasAddressPortProtocolInIngress(t *testing.T) { expectedResult: true, }, { - // v1 degradation: port/protocol projection is out of scope; address-only matching. name: "Invalid port", containerID: "test-container-id", address: "172.16.0.10", port: 9999, protocol: "TCP", - expectedResult: true, + expectedResult: false, }, { - // v1 degradation: port/protocol projection is out of scope; address-only matching. name: "Invalid protocol", containerID: "test-container-id", address: "172.16.0.10", port: 8080, protocol: "UDP", - expectedResult: true, + expectedResult: false, }, { name: "Invalid address", @@ -405,14 +401,15 @@ func TestWasAddressPortProtocolWithNilPort(t *testing.T) { functionCache: cache.NewFunctionCache(cache.DefaultFunctionCacheConfig()), } - // v1 degradation: address-only matching; nil port in profile no longer checked. + // A listed entry with a nil port contributes nothing: the only port + // wildcard is an ABSENT ports stanza, so these addresses stay restricted. result := lib.wasAddressPortProtocolInEgress( types.String("test-container-id"), types.String("192.168.1.100"), types.Int(80), types.String("TCP"), ) - assert.Equal(t, types.Bool(true), result) + assert.Equal(t, types.Bool(false), result) result = lib.wasAddressPortProtocolInIngress( types.String("test-container-id"), @@ -420,5 +417,5 @@ func TestWasAddressPortProtocolWithNilPort(t *testing.T) { types.Int(8080), types.String("TCP"), ) - assert.Equal(t, types.Bool(true), result) + assert.Equal(t, types.Bool(false), result) } diff --git a/pkg/rulemanager/cel/libraries/containerprofilenetwork/port_protocol_test.go b/pkg/rulemanager/cel/libraries/containerprofilenetwork/port_protocol_test.go new file mode 100644 index 0000000000..20773b1b83 --- /dev/null +++ b/pkg/rulemanager/cel/libraries/containerprofilenetwork/port_protocol_test.go @@ -0,0 +1,74 @@ +package containerprofilenetwork + +import ( + "testing" + + "github.com/google/cel-go/common/types" + "github.com/kubescape/node-agent/pkg/rulemanager/cel/libraries/cache" + "github.com/kubescape/storage/pkg/apis/softwarecomposition/v1beta1" + "github.com/stretchr/testify/assert" + "k8s.io/utils/ptr" +) + +func port(proto string, p int32) v1beta1.NetworkPort { + return v1beta1.NetworkPort{Protocol: v1beta1.Protocol(proto), Port: ptr.To(p)} +} + +func evalEgressPort(lib *containerProfileNetworkLibrary, addr string, p int64, proto string) types.Bool { + res := lib.wasAddressPortProtocolInEgress(types.String("cid"), types.String(addr), types.Int(p), types.String(proto)) + return cache.ConvertProfileNotAvailableErrToBool(res, false).(types.Bool) +} + +func evalIngressPort(lib *containerProfileNetworkLibrary, addr string, p int64, proto string) types.Bool { + res := lib.wasAddressPortProtocolInIngress(types.String("cid"), types.String(addr), types.Int(p), types.String(proto)) + return cache.ConvertProfileNotAvailableErrToBool(res, false).(types.Bool) +} + +func TestWasAddressPortProtocolInEgress_PortWildcard(t *testing.T) { + noPorts := buildLibWithContainer(t, []v1beta1.NetworkNeighbor{ + {IPAddresses: []string{"93.184.216.34"}}, + }, nil) + assert.Equal(t, types.Bool(true), evalEgressPort(noPorts, "93.184.216.34", 8080, "TCP")) + assert.Equal(t, types.Bool(false), evalEgressPort(noPorts, "1.1.1.1", 8080, "TCP")) + + zeroPort := buildLibWithContainer(t, []v1beta1.NetworkNeighbor{ + {IPAddresses: []string{"93.184.216.34"}, Ports: []v1beta1.NetworkPort{port("TCP", 0)}}, + }, nil) + assert.Equal(t, types.Bool(false), evalEgressPort(zeroPort, "93.184.216.34", 8080, "TCP"), + "an explicit port 0 is a literal, not a wildcard: only an absent ports stanza opens the entry") +} + +func TestWasAddressPortProtocolInIngress_Symmetric(t *testing.T) { + lib := buildLibWithContainer(t, nil, []v1beta1.NetworkNeighbor{ + {IPAddresses: []string{"172.16.0.0/12"}, Ports: []v1beta1.NetworkPort{port("TCP", 6379)}}, + }) + assert.Equal(t, types.Bool(true), evalIngressPort(lib, "172.16.5.9", 6379, "TCP")) + assert.Equal(t, types.Bool(false), evalIngressPort(lib, "172.16.5.9", 5432, "TCP")) + assert.Equal(t, types.Bool(false), evalIngressPort(lib, "10.0.0.1", 6379, "TCP")) +} + +func TestWasAddressPortProtocolInEgress_ZeroPortIsNotAWildcard(t *testing.T) { + lib := buildLibWithContainer(t, []v1beta1.NetworkNeighbor{ + {IPAddresses: []string{"93.184.216.34"}, Ports: []v1beta1.NetworkPort{port("TCP", 0)}}, + }, nil) + assert.Equal(t, types.Bool(false), evalEgressPort(lib, "93.184.216.34", 8080, "TCP")) + assert.Equal(t, types.Bool(false), evalEgressPort(lib, "93.184.216.34", 53, "UDP")) +} + +func TestWasAddressPortProtocolInEgress_MixedZeroPortKeepsEveryProtocolRestricted(t *testing.T) { + lib := buildLibWithContainer(t, []v1beta1.NetworkNeighbor{ + {IPAddresses: []string{"10.0.5.9"}, Ports: []v1beta1.NetworkPort{port("TCP", 0), port("UDP", 53)}}, + }, nil) + assert.Equal(t, types.Bool(false), evalEgressPort(lib, "10.0.5.9", 9999, "TCP"), + "an explicit {TCP,0} entry no longer opens TCP: wildcard is expressed only by omitting the ports stanza") + assert.Equal(t, types.Bool(true), evalEgressPort(lib, "10.0.5.9", 53, "UDP")) + assert.Equal(t, types.Bool(false), evalEgressPort(lib, "10.0.5.9", 54, "UDP")) +} + +func TestWasAddressPortProtocolInEgress_NilPortEntryContributesNothing(t *testing.T) { + lib := buildLibWithContainer(t, []v1beta1.NetworkNeighbor{ + {IPAddresses: []string{"10.0.5.9"}, Ports: []v1beta1.NetworkPort{{Protocol: "TCP"}, port("UDP", 53)}}, + }, nil) + assert.Equal(t, types.Bool(false), evalEgressPort(lib, "10.0.5.9", 8080, "TCP")) + assert.Equal(t, types.Bool(true), evalEgressPort(lib, "10.0.5.9", 53, "UDP")) +} diff --git a/pkg/rulemanager/cel/libraries/containerprofilenetwork/selector_test.go b/pkg/rulemanager/cel/libraries/containerprofilenetwork/selector_test.go new file mode 100644 index 0000000000..77f349bc82 --- /dev/null +++ b/pkg/rulemanager/cel/libraries/containerprofilenetwork/selector_test.go @@ -0,0 +1,106 @@ +package containerprofilenetwork + +import ( + "testing" + + "github.com/kubescape/node-agent/pkg/objectcache" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/labels" +) + +func podSel(m map[string]string) *metav1.LabelSelector { return &metav1.LabelSelector{MatchLabels: m} } +func nsSel(name string) *metav1.LabelSelector { + return &metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/metadata.name": name}} +} + +// TestWasSelectorInPeers_TruthTable is the full matrix for peer-selector +// matching. Rules: matching is on pod LABELS; a nil namespaceSelector does NOT +// consult the namespace (it is a collision-disambiguator only); an empty +// podSelector matches NOTHING (fail closed, opposite of NetworkPolicy); an +// explicit namespaceSelector must match; a peer with no resolvable pod identity +// never matches (enforced one layer up in wasSelectorIn, tested there). +func TestWasSelectorInPeers_TruthTable(t *testing.T) { + const profileNs = "redis" + client := labels.Set{"app": "redis-client"} + clientPlus := labels.Set{"app": "redis-client", "tier": "cache"} + + // matchExpressions-based selectors (a non-empty selector expressed without matchLabels). + exprIn := &metav1.LabelSelector{MatchExpressions: []metav1.LabelSelectorRequirement{ + {Key: "app", Operator: metav1.LabelSelectorOpIn, Values: []string{"redis-client"}}}} + exprExists := &metav1.LabelSelector{MatchExpressions: []metav1.LabelSelectorRequirement{ + {Key: "app", Operator: metav1.LabelSelectorOpExists}}} + + p := func(pod, ns *metav1.LabelSelector) objectcache.PeerSelector { + return objectcache.PeerSelector{PodSelector: pod, NamespaceSelector: ns} + } + + cases := []struct { + name string + peers []objectcache.PeerSelector + labels labels.Set + peerNs string + want bool + }{ + // --- podSelector shapes --- + {"nil podSelector never matches", []objectcache.PeerSelector{p(nil, nil)}, client, "redis", false}, + {"empty podSelector matches nothing (same ns)", []objectcache.PeerSelector{p(podSel(map[string]string{}), nil)}, client, "redis", false}, + {"empty podSelector matches nothing (empty labels)", []objectcache.PeerSelector{p(podSel(map[string]string{}), nil)}, labels.Set{}, "redis", false}, + {"empty podSelector matches nothing (explicit ns)", []objectcache.PeerSelector{p(podSel(map[string]string{}), nsSel("redis"))}, client, "redis", false}, + + // --- label matching, nil namespaceSelector (namespace NOT consulted) --- + {"label match, nil ns, same ns", []objectcache.PeerSelector{p(podSel(map[string]string{"app": "redis-client"}), nil)}, client, "redis", true}, + {"label match, nil ns, FOREIGN ns still matches", []objectcache.PeerSelector{p(podSel(map[string]string{"app": "redis-client"}), nil)}, client, "attacker", true}, + {"label mismatch, nil ns", []objectcache.PeerSelector{p(podSel(map[string]string{"app": "redis-client"}), nil)}, labels.Set{"app": "other"}, "redis", false}, + {"selector is a subset of pod labels", []objectcache.PeerSelector{p(podSel(map[string]string{"app": "redis-client"}), nil)}, clientPlus, "redis", true}, + {"non-empty selector vs empty labels", []objectcache.PeerSelector{p(podSel(map[string]string{"app": "redis-client"}), nil)}, labels.Set{}, "redis", false}, + + // --- explicit namespaceSelector (must match) --- + {"label+ns match", []objectcache.PeerSelector{p(podSel(map[string]string{"app": "redis-client"}), nsSel("redis"))}, client, "redis", true}, + {"explicit ns mismatch rejects (same labels)", []objectcache.PeerSelector{p(podSel(map[string]string{"app": "redis-client"}), nsSel("redis"))}, client, "attacker", false}, + {"explicit ns names a third namespace", []objectcache.PeerSelector{p(podSel(map[string]string{"app": "redis-client"}), nsSel("other"))}, client, "redis", false}, + {"empty (non-nil) ns selector matches any ns", []objectcache.PeerSelector{p(podSel(map[string]string{"app": "redis-client"}), &metav1.LabelSelector{})}, client, "attacker", true}, + + // --- matchExpressions --- + {"matchExpressions In matches", []objectcache.PeerSelector{p(exprIn, nil)}, client, "redis", true}, + {"matchExpressions Exists matches labelled pod", []objectcache.PeerSelector{p(exprExists, nil)}, client, "redis", true}, + {"matchExpressions Exists rejects unlabelled pod", []objectcache.PeerSelector{p(exprExists, nil)}, labels.Set{}, "redis", false}, + + // --- list semantics --- + {"empty peer list", nil, client, "redis", false}, + {"one of several matches", []objectcache.PeerSelector{p(podSel(map[string]string{"app": "x"}), nil), p(podSel(map[string]string{"app": "redis-client"}), nil)}, client, "redis", true}, + {"none of several matches", []objectcache.PeerSelector{p(podSel(map[string]string{"app": "x"}), nil), p(podSel(map[string]string{"app": "y"}), nil)}, client, "redis", false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := wasSelectorInPeers(tc.peers, tc.labels, tc.peerNs, profileNs); got != tc.want { + t.Fatalf("wasSelectorInPeers = %v, want %v", got, tc.want) + } + }) + } +} + +// TestNamespaceSelectorMatches_TruthTable pins the namespace-disambiguator +// alone: nil never consults the namespace, an explicit selector must match by +// the kubernetes.io/metadata.name label. +func TestNamespaceSelectorMatches_TruthTable(t *testing.T) { + cases := []struct { + name string + sel *metav1.LabelSelector + ns string + want bool + }{ + {"nil matches same ns", nil, "redis", true}, + {"nil matches foreign ns (not consulted)", nil, "attacker", true}, + {"nil matches empty ns", nil, "", true}, + {"explicit matches", nsSel("redis"), "redis", true}, + {"explicit rejects other", nsSel("redis"), "attacker", false}, + {"empty explicit matches any", &metav1.LabelSelector{}, "attacker", true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := namespaceSelectorMatches(tc.sel, tc.ns, "redis"); got != tc.want { + t.Fatalf("namespaceSelectorMatches = %v, want %v", got, tc.want) + } + }) + } +} diff --git a/pkg/rulemanager/cel/libraries/containerprofilenetwork/wildcard_test.go b/pkg/rulemanager/cel/libraries/containerprofilenetwork/wildcard_test.go index e0a16c2299..bca5171f51 100644 --- a/pkg/rulemanager/cel/libraries/containerprofilenetwork/wildcard_test.go +++ b/pkg/rulemanager/cel/libraries/containerprofilenetwork/wildcard_test.go @@ -272,20 +272,16 @@ func TestWasAddressPortProtocolInEgress_PortWrapRejected(t *testing.T) { }, }, nil) - // See TestWasAddressPortProtocolInEgress_WithCIDR for the - // port/protocol regression note. The port-range guard ([0, 65535]) - // still applies — what's gone is port-specific matching: any in-range - // port matches if the address matches. cases := []struct { name string port int64 want bool }{ {"in-range hit", 443, true}, - {"in-range miss", 444, true}, // was: false (port mismatch). Now matches: address-only after projection-v1. - {"wrap-to-443 rejected", 4294967739, false}, // (1<<32)+443 — range guard fires - {"negative rejected", -1, false}, // range guard fires - {"too-large rejected", 65536, false}, // range guard fires + {"in-range miss", 444, false}, + {"wrap-to-443 rejected", 4294967739, false}, + {"negative rejected", -1, false}, + {"too-large rejected", 65536, false}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { @@ -350,27 +346,17 @@ func TestWasAddressPortProtocolInEgress_WithCIDR(t *testing.T) { }, }, nil) - // NOTE: upstream's projection-v1 (PR #799) explicitly drops port/protocol - // granularity from the address surface — the comment in network.go reads - // "port/protocol projection (AddressPortsByAddr) is out of scope for v1; - // degrade to address-only matching". So the matcher now only checks IP. - // - // Spec §4.7 still says ports[] is per-neighbor; the runtime gap is a - // known limitation flagged in the rebase commit. Test expectations - // updated to match runtime reality. Bringing port/protocol back is a - // follow-up: would need projection_apply to surface a per-address - // (port, protocol) set into ProjectedContainerProfile and the CEL - // helper to consult it. cases := []struct { observed string port int64 proto string want bool }{ - {"10.1.2.3", 443, "TCP", true}, // CIDR match (port/proto not enforced) - {"10.1.2.3", 80, "TCP", true}, // was: wrong port — now matches address-only - {"10.1.2.3", 443, "UDP", true}, // was: wrong protocol — now matches address-only - {"11.0.0.1", 443, "TCP", false}, // outside CIDR — still rejected + {"10.1.2.3", 443, "TCP", true}, + {"10.1.2.3", 80, "TCP", false}, + {"10.1.2.3", 443, "UDP", false}, + {"11.0.0.1", 443, "TCP", false}, + {"10.1.2.3", 443, "tcp", true}, } for _, tc := range cases { t.Run(tc.observed, func(t *testing.T) { diff --git a/pkg/rulemanager/cel/selector_compile_test.go b/pkg/rulemanager/cel/selector_compile_test.go new file mode 100644 index 0000000000..fbcb225d5b --- /dev/null +++ b/pkg/rulemanager/cel/selector_compile_test.go @@ -0,0 +1,41 @@ +package cel + +import ( + "testing" + "time" + + "github.com/goradd/maps" + "github.com/kubescape/node-agent/pkg/config" + "github.com/kubescape/node-agent/pkg/objectcache" + objectcachev1 "github.com/kubescape/node-agent/pkg/objectcache/v1" + "github.com/kubescape/node-agent/pkg/rulemanager/cel/libraries/cache" +) + +// TestCompileSelectorRules pins that the selector rules type-check against the +// real event object type: event.dstPodLabels is declared as a generic CEL map +// and must remain assignable to the was_selector_in_{ingress,egress} map param. +// Regression guard for the R0012 ingress rule that consumes the IG-enriched +// peer namespace + labels. +func TestCompileSelectorRules(t *testing.T) { + objCache := &objectcachev1.RuleObjectCacheMock{ + ContainerIDToSharedData: maps.NewSafeMap[string, *objectcache.WatchedContainerData](), + } + c, err := NewCEL(objCache, config.Config{ + CelConfigCache: cache.FunctionCacheConfig{MaxSize: 1000, TTL: time.Minute}, + }) + if err != nil { + t.Fatalf("NewCEL: %v", err) + } + + exprs := []string{ + `cp.was_selector_in_ingress(event.containerId, event.dstNamespace, event.dstPodLabels)`, + `cp.was_selector_in_egress(event.containerId, event.dstNamespace, event.dstPodLabels)`, + // The full R0012 ingress expression as bound in default-rules.yaml. + `event.pktType == 'HOST' && !event.dstAddr.startsWith('127.') && !cp.was_address_in_ingress(event.containerId, event.dstAddr) && !cp.was_selector_in_ingress(event.containerId, event.dstNamespace, event.dstPodLabels)`, + } + for _, e := range exprs { + if err := c.registerExpression(e); err != nil { + t.Fatalf("expression failed to compile: %q\n%v", e, err) + } + } +} diff --git a/pkg/utils/cel.go b/pkg/utils/cel.go index 39c3ad40f2..b95cc1c115 100644 --- a/pkg/utils/cel.go +++ b/pkg/utils/cel.go @@ -202,6 +202,35 @@ var CelFields = map[string]*celtypes.FieldType{ return celtypes.Int(x.Raw.GetDstPort()), nil }), }, + // dstNamespace / dstPodLabels carry the peer identity that IG's + // kubeipresolver resolves cluster-wide (independent of node-agent's + // node-local pod cache), so selector rules can match a peer on any node. + "dstNamespace": { + Type: celtypes.StringType, + IsSet: isSet, + GetFrom: ref.FieldGetter(func(target any) (any, error) { + x := target.(*xcel.Object[CelEvent]) + if x.Raw == nil { + return nil, errCelObjectNil + } + return celtypes.String(x.Raw.GetDstEndpoint().Namespace), nil + }), + }, + "dstPodLabels": { + Type: celtypes.MapType, + IsSet: isSet, + GetFrom: ref.FieldGetter(func(target any) (any, error) { + x := target.(*xcel.Object[CelEvent]) + if x.Raw == nil { + return nil, errCelObjectNil + } + pl := x.Raw.GetDstEndpoint().PodLabels + if pl == nil { + pl = map[string]string{} + } + return pl, nil + }), + }, "exepath": { Type: celtypes.StringType, IsSet: isSet, diff --git a/tests/chart/templates/node-agent/clusterrole.yaml b/tests/chart/templates/node-agent/clusterrole.yaml index 03d5137555..7d4096a3fe 100644 --- a/tests/chart/templates/node-agent/clusterrole.yaml +++ b/tests/chart/templates/node-agent/clusterrole.yaml @@ -11,6 +11,9 @@ rules: - apiGroups: [""] resources: ["events"] verbs: ["list", "watch", "create"] +- apiGroups: ["discovery.k8s.io"] + resources: ["endpointslices"] + verbs: ["watch", "list"] - apiGroups: ["batch"] resources: ["jobs", "cronjobs"] verbs: ["get", "watch", "list"] diff --git a/tests/chart/templates/node-agent/configmap.yaml b/tests/chart/templates/node-agent/configmap.yaml index 523b5bbac6..b64a63024b 100644 --- a/tests/chart/templates/node-agent/configmap.yaml +++ b/tests/chart/templates/node-agent/configmap.yaml @@ -14,6 +14,7 @@ data: "prometheusExporterEnabled": {{ eq .Values.nodeAgent.config.prometheusExporter "enable" }}, "runtimeDetectionEnabled": {{ eq .Values.capabilities.runtimeDetection "enable" }}, "networkServiceEnabled": {{ eq .Values.capabilities.networkPolicyService "enable" }}, + "networkServiceResolutionEnabled": {{ if hasKey .Values.nodeAgent.config "networkServiceResolution" }}{{ .Values.nodeAgent.config.networkServiceResolution }}{{ else }}true{{ end }}, "malwareDetectionEnabled": {{ eq .Values.capabilities.malwareDetection "enable" }}, "httpDetectionEnabled": {{ eq .Values.capabilities.httpDetection "enable" }}, "initialDelay": "{{ .Values.nodeAgent.config.learningPeriod }}", diff --git a/tests/chart/templates/node-agent/default-rule-binding.yaml b/tests/chart/templates/node-agent/default-rule-binding.yaml index 3d8f7847b4..755bd39055 100644 --- a/tests/chart/templates/node-agent/default-rule-binding.yaml +++ b/tests/chart/templates/node-agent/default-rule-binding.yaml @@ -39,5 +39,6 @@ spec: - ruleName: "Exec to pod" - ruleName: "Port forward to pod" - ruleName: "Unexpected Egress Network Traffic" + - ruleName: "Unexpected Ingress Network Traffic" - ruleName: "Unexpected Ptrace Syscall Usage" - ruleName: "Unexpected io_uring Operation Detected" diff --git a/tests/chart/templates/node-agent/default-rules.yaml b/tests/chart/templates/node-agent/default-rules.yaml index 512b4d9ec8..909e9f3f3f 100644 --- a/tests/chart/templates/node-agent/default-rules.yaml +++ b/tests/chart/templates/node-agent/default-rules.yaml @@ -313,7 +313,7 @@ spec: uniqueId: "event.dstAddr + '_' + string(event.dstPort) + '_' + event.proto" ruleExpression: - eventType: "network" - expression: "event.pktType == 'OUTGOING' && !net.is_private_ip(event.dstAddr) && !cp.was_address_in_egress(event.containerId, event.dstAddr)" + expression: "event.pktType == 'OUTGOING' && !event.dstAddr.startsWith('127.') && event.dstAddr != '::1' && !cp.was_address_port_protocol_in_egress(event.containerId, event.dstAddr, event.dstPort, event.proto) && !cp.was_selector_in_egress(event.containerId, event.dstNamespace, event.dstPodLabels)" profileDependency: 0 profileDataRequired: egressAddresses: all @@ -329,6 +329,31 @@ spec: - "network" - "anomaly" - "networkprofile" + - name: "Unexpected Ingress Network Traffic" + enabled: true + id: "R0012" + description: "Detecting unexpected ingress network traffic that is not allowlisted by application profile. Symmetric twin of R0011: internal and external peers alike, only loopback excluded; internal peers are allowlisted narrowly via addresses or resolved serviceRef/serviceSelector entries rather than a serviceCIDR that blinds detection to lateral movement." + expressions: + message: "'Unexpected ingress network communication from: ' + event.dstAddr + ':' + string(event.dstPort) + ' using ' + event.proto + ' to: ' + event.containerName" + uniqueId: "event.dstAddr + '_' + string(event.dstPort) + '_' + event.proto" + ruleExpression: + - eventType: "network" + expression: "event.pktType == 'HOST' && !event.dstAddr.startsWith('127.') && event.dstAddr != '::1' && !cp.was_address_port_protocol_in_ingress(event.containerId, event.dstAddr, event.dstPort, event.proto) && !cp.was_selector_in_ingress(event.containerId, event.dstNamespace, event.dstPodLabels)" + profileDependency: 0 + profileDataRequired: + ingressAddresses: all + severity: 5 # Medium + supportPolicy: false + isTriggerAlert: false + mitreTactic: "TA0008" + mitreTechnique: "T1210" + tags: + - "context:kubernetes" + - "context:container" + - "whitelisted" + - "network" + - "anomaly" + - "networkprofile" - name: "Unexpected process arguments" enabled: true id: "R0040" diff --git a/tests/chart/values.yaml b/tests/chart/values.yaml index 1aea3a150f..8c78ff9856 100644 --- a/tests/chart/values.yaml +++ b/tests/chart/values.yaml @@ -32,8 +32,8 @@ global: storage: name: "storage" image: - repository: quay.io/kubescape/storage - tag: v0.0.156 + repository: ghcr.io/k8sstormcenter/storage + tag: net-v2-rc1 pullPolicy: Always cleanupInterval: "6h" labels: @@ -58,6 +58,7 @@ nodeAgent: maxLearningPeriod: 2m learningPeriod: 1m updatePeriod: 30s + networkServiceResolution: true maxDelaySeconds: 1 prometheusExporter: enable httpExporterConfig: {} diff --git a/tests/component_test.go b/tests/component_test.go index 154e441e8b..35ec4f27bb 100644 --- a/tests/component_test.go +++ b/tests/component_test.go @@ -1192,6 +1192,19 @@ func Test_21_AlertOnPartialThenLearnNetworkTest(t *testing.T) { fusioncoreIP = "162.0.217.171" ) port80 := int32(80) + port53 := int32(53) + // R0011 excludes only loopback (maximally noisy by design), so authored + // profiles must allow the pod's own DNS egress to cluster DNS or every + // nslookup mints an R0011 that skews the before/after counts. + clusterDNS := v1beta1.NetworkNeighbor{ + Identifier: "cluster-dns", + Type: v1beta1.CommunicationTypeEgress, + IPAddresses: []string{"10.96.0.0/12"}, + Ports: []v1beta1.NetworkPort{ + {Name: "UDP-53", Protocol: v1beta1.ProtocolUDP, Port: &port53}, + {Name: "TCP-53", Protocol: v1beta1.ProtocolTCP, Port: &port53}, + }, + } ns := testutils.NewRandomNamespace() k8sClient := k8sinterface.NewKubernetesApi() @@ -1220,6 +1233,7 @@ func Test_21_AlertOnPartialThenLearnNetworkTest(t *testing.T) { IPAddress: fusioncoreIP, Ports: []v1beta1.NetworkPort{{Name: "TCP-80", Protocol: v1beta1.ProtocolTCP, Port: &port80}}, }, + clusterDNS, }, }, } @@ -1291,6 +1305,7 @@ func Test_21_AlertOnPartialThenLearnNetworkTest(t *testing.T) { IPAddress: subjectIP, Ports: []v1beta1.NetworkPort{{Name: "TCP-80", Protocol: v1beta1.ProtocolTCP, Port: &port80}}, }, + clusterDNS, } _, err = storageClient.ContainerProfiles(ns.Name).Update(context.Background(), cur, metav1.UpdateOptions{}) require.NoError(t, err, "update CP: add subject IP, remove canary domain") @@ -2718,6 +2733,52 @@ func Test_28_UserDefinedNetworkNeighborhood(t *testing.T) { "fusioncore.ai IP is in NN — should NOT fire R0011") }) + // 162.0.217.171 is allowed on TCP/80 only; :443 is a port violation → R0011. + t.Run("port_violation_different_port_R0011", func(t *testing.T) { + wl := setup(t) + stdout, stderr, err := wl.ExecIntoPod([]string{"curl", "-sm5", "-k", "https://162.0.217.171"}, "curl") + t.Logf("curl https://162.0.217.171 → err=%v stdout=%q stderr=%q", err, stdout, stderr) + alerts := waitAlerts(t, wl.Namespace) + logAlerts(t, alerts) + assert.GreaterOrEqual(t, countByRule(alerts, "R0011"), 1, + "egress to allowed IP 162.0.217.171 on non-allowed port 443 must fire R0011") + }) + + // 9.9.9.9 is listed with an explicit port 0 — a literal, NOT a wildcard: + // the only port wildcard is an absent ports stanza, so :80/:443 violate. + t.Run("port_zero_is_literal_not_wildcard", func(t *testing.T) { + wl := setup(t) + wl.ExecIntoPod([]string{"curl", "-sm5", "http://9.9.9.9"}, "curl") + wl.ExecIntoPod([]string{"curl", "-sm5", "-k", "https://9.9.9.9"}, "curl") + alerts := waitAlerts(t, wl.Namespace) + logAlerts(t, alerts) + assert.GreaterOrEqual(t, countByRule(alerts, "R0011"), 1, + "an explicit port-0 entry must not open 9.9.9.9 on other ports") + }) + + // 208.67.222.222 is allowlisted with no ports stanza (ANY); no port fires R0011. + t.Run("port_wildcard_empty_stanza_allows_any", func(t *testing.T) { + wl := setup(t) + wl.ExecIntoPod([]string{"curl", "-sm5", "http://208.67.222.222"}, "curl") + wl.ExecIntoPod([]string{"curl", "-sm5", "-k", "https://208.67.222.222"}, "curl") + alerts := waitAlerts(t, wl.Namespace) + logAlerts(t, alerts) + assert.Equal(t, 0, countByRule(alerts, "R0011"), + "208.67.222.222 allowlisted with empty ports stanza (any) must not fire R0011 on any port") + }) + + // Internal peer 10.96.0.1 (kube-api) is allowlisted on TCP/443 only; :80 is a port violation → R0011. + t.Run("internal_port_violation_R0011", func(t *testing.T) { + wl := setup(t) + wl.ExecIntoPod([]string{"curl", "-sm5", "-k", "https://10.96.0.1"}, "curl") + stdout, stderr, err := wl.ExecIntoPod([]string{"curl", "-sm5", "http://10.96.0.1"}, "curl") + t.Logf("curl http://10.96.0.1:80 → err=%v stdout=%q stderr=%q", err, stdout, stderr) + alerts := waitAlerts(t, wl.Namespace) + logAlerts(t, alerts) + assert.GreaterOrEqual(t, countByRule(alerts, "R0011"), 1, + "egress to internal IP 10.96.0.1 on non-allowed port 80 must fire R0011") + }) + // --------------------------------------------------------------- // 28b. Unknown domains — domains NOT in the NN → R0005. // Uses both nslookup (pure DNS) and curl (DNS + TCP). @@ -3906,3 +3967,261 @@ func Test_49_EphemeralContainerFullTreatment(t *testing.T) { return countRuleAlerts(t, ns.Name, "R0001", "ephcon", "id") > 0 }, 2*time.Minute, 10*time.Second, "id was not in the ephemeral container's learned profile — it must fire R0001 (detected + alerted like any other container)") } + +// Test_50_ServiceRefNetworkNeighbor validates serviceRef/serviceSelector end to +// end (k8sstormcenter/node-agent#92) against REAL GitOps traffic: a Flux +// source-controller reconciling HelmRepository CRs. Its egress is authored +// purely as Kubernetes-native selectors — serviceRef default/kubernetes for the +// apiserver, serviceRef kube-system/kube-dns for name resolution, and a +// serviceSelector role=helm-repo fanning across the two repo Services it is +// allowed to fetch. Nothing is exec'd and no address is hardcoded; the +// controller's own reconcile loop generates every connection. +// +// The negative is the lateral move a broad serviceCIDR entry would hide: the +// HelmRepository URL is repointed at decoy-repo — a sibling Service on the same +// port, backed by its own pod, carrying none of the selector's labels — and the +// controller itself fetches it. R0011 MUST fire for that and MUST NOT fire for +// the allowlisted Services. +func Test_50_ServiceRefNetworkNeighbor(t *testing.T) { + start := time.Now() + defer tearDownTest(t, start) + + const ( + cpName = "serviceref-flux-cp" + containerName = "manager" + ) + port80, port443, port53 := int32(80), int32(443), int32(53) + + ns := testutils.NewRandomNamespace() + k8sClient := k8sinterface.NewKubernetesApi() + storageClient := spdxv1beta1client.NewForConfigOrDie(k8sClient.K8SConfig) + + // Authored profile: NETWORK ONLY. No syscalls/execs — they are irrelevant to + // a network test and only add false-positive surface. Every peer is named, + // never addressed. + cp := &v1beta1.ContainerProfile{ + ObjectMeta: metav1.ObjectMeta{Name: cpName, Namespace: ns.Name}, + Spec: v1beta1.ContainerProfileSpec{ + LabelSelector: metav1.LabelSelector{MatchLabels: map[string]string{"app": "source-controller"}}, + Egress: []v1beta1.NetworkNeighbor{ + { + Identifier: "apiserver", + Type: v1beta1.CommunicationTypeEgress, + ServiceRefNamespace: "default", + ServiceRefName: "kubernetes", + Ports: []v1beta1.NetworkPort{{Name: "TCP-443", Protocol: v1beta1.ProtocolTCP, Port: &port443}}, + }, + { + Identifier: "cluster-dns", + Type: v1beta1.CommunicationTypeEgress, + ServiceRefNamespace: "kube-system", + ServiceRefName: "kube-dns", + Ports: []v1beta1.NetworkPort{ + {Name: "UDP-53", Protocol: v1beta1.ProtocolUDP, Port: &port53}, + {Name: "TCP-53", Protocol: v1beta1.ProtocolTCP, Port: &port53}, + }, + }, + { + Identifier: "helm-repos", + Type: v1beta1.CommunicationTypeEgress, + ServiceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"role": "helm-repo"}}, + NamespaceSelector: &metav1.LabelSelector{MatchLabels: map[string]string{"kubernetes.io/metadata.name": ns.Name}}, + Ports: []v1beta1.NetworkPort{{Name: "TCP-80", Protocol: v1beta1.ProtocolTCP, Port: &port80}}, + }, + }, + }, + } + _, err := storageClient.ContainerProfiles(ns.Name).Create(context.Background(), cp, metav1.CreateOptions{}) + require.NoError(t, err, "create authored ContainerProfile") + require.Eventually(t, func() bool { + _, e := storageClient.ContainerProfiles(ns.Name).Get(context.Background(), cpName, v1.GetOptions{}) + return e == nil + }, 30*time.Second, time.Second, "authored CP must be in storage before pod deploy") + + require.NoError(t, testutils.ApplyMultiDocDir(ns.Name, path.Join(utils.CurrentDir(), "resources/serviceref-suite")), + "apply flux source-controller + helm repo suite") + + waitDeploy := func(name string) { + t.Helper() + require.Eventually(t, func() bool { + d, e := k8sClient.KubernetesClient.AppsV1().Deployments(ns.Name).Get(context.TODO(), name, metav1.GetOptions{}) + return e == nil && d.Status.ReadyReplicas > 0 + }, 3*time.Minute, 5*time.Second, "%s must become ready", name) + } + waitDeploy("helm-repo") + waitDeploy("decoy-repo") + waitDeploy("source-controller") + + countRule := func(ruleID string) int { + alerts, _ := testutils.GetAlerts(ns.Name) + n := 0 + for _, a := range alerts { + if a.Labels["rule_id"] == ruleID && a.Labels["container_name"] == containerName { + n++ + } + } + return n + } + + helmRepoGVR := schema.GroupVersionResource{Group: "source.toolkit.fluxcd.io", Version: "v1", Resource: "helmrepositories"} + repoClient := k8sClient.DynamicClient.Resource(helmRepoGVR).Namespace(ns.Name) + newRepo := func(name, svc string) *unstructured.Unstructured { + return &unstructured.Unstructured{Object: map[string]interface{}{ + "apiVersion": "source.toolkit.fluxcd.io/v1", + "kind": "HelmRepository", + "metadata": map[string]interface{}{"name": name, "namespace": ns.Name}, + // Trailing-dot FQDN: absolute, so the resolver does not walk the + // search list and emit extra lookups. + "spec": map[string]interface{}{ + "interval": "30s", + "url": fmt.Sprintf("http://%s.%s.svc.cluster.local./", svc, ns.Name), + }, + }} + } + repoReady := func(name string) bool { + obj, e := repoClient.Get(context.TODO(), name, metav1.GetOptions{}) + if e != nil { + return false + } + conds, _, _ := unstructured.NestedSlice(obj.Object, "status", "conditions") + for _, c := range conds { + m, ok := c.(map[string]interface{}) + if ok && m["type"] == "Ready" && m["status"] == "True" { + return true + } + } + return false + } + + // Let node-agent bind the profile and fill its Service/EndpointSlice caches + // before any reconcile traffic is judged. + time.Sleep(40 * time.Second) + + // Phase 1 — the controller reconciles both allowlisted repo Services while + // continuously talking to the apiserver and cluster DNS. Every one of those + // peers is named by the profile, so no egress alert may fire. + t.Run("selector_allowed_no_alert", func(t *testing.T) { + for _, r := range []struct{ name, svc string }{{"primary", "helm-primary"}, {"mirror", "helm-mirror"}} { + _, e := repoClient.Create(context.TODO(), newRepo(r.name, r.svc), metav1.CreateOptions{}) + require.NoError(t, e, "create HelmRepository %s", r.name) + } + for _, n := range []string{"primary", "mirror"} { + require.Eventually(t, func() bool { return repoReady(n) }, 3*time.Minute, 10*time.Second, + "HelmRepository %s must reconcile (real fetch through an allowlisted Service)", n) + } + // Two further reconcile intervals of steady-state traffic. + time.Sleep(90 * time.Second) + assert.Equal(t, 0, countRule("R0011"), + "apiserver/DNS/helm-repo egress is fully named by serviceRef+serviceSelector — R0011 may not fire") + }) + + // Phase 2 — the GitOps source of truth is tampered with: primary is + // repointed at decoy-repo, a sibling Service on the same port that the + // role=helm-repo selector does not cover. source-controller fetches it on + // its own next reconcile. This is the lateral move a serviceCIDR entry hides. + t.Run("sibling_service_pivot_fires_alert", func(t *testing.T) { + before := countRule("R0011") + patch := []byte(fmt.Sprintf(`{"spec":{"url":"http://decoy-repo.%s.svc.cluster.local./"}}`, ns.Name)) + _, e := repoClient.Patch(context.TODO(), "primary", types.MergePatchType, patch, metav1.PatchOptions{}) + require.NoError(t, e, "repoint HelmRepository at the decoy Service") + require.Eventually(t, func() bool { + return countRule("R0011") > before + }, 4*time.Minute, 15*time.Second, + "egress to an unlisted sibling Service MUST fire R0011 — the selector is narrow, not a blanket") + }) +} + +// Test_51_ServiceRefIngressR0012 is the ingress twin of Test_50: nginx serves +// two real clients; the profile names one of them (ingress serviceRef, whose +// resolution covers the client Service's ClusterIP and pod endpoint IPs), and +// R0012 must stay silent for it while firing for the unlisted one (k6). +func Test_51_ServiceRefIngressR0012(t *testing.T) { + start := time.Now() + defer tearDownTest(t, start) + + const cpName = "serviceref-ingress-cp" + port80 := int32(80) + + ns := testutils.NewRandomNamespace() + k8sClient := k8sinterface.NewKubernetesApi() + storageClient := spdxv1beta1client.NewForConfigOrDie(k8sClient.K8SConfig) + + cp := &v1beta1.ContainerProfile{ + ObjectMeta: metav1.ObjectMeta{Name: cpName, Namespace: ns.Name}, + Spec: v1beta1.ContainerProfileSpec{ + LabelSelector: metav1.LabelSelector{MatchLabels: map[string]string{"app": "helm-repo"}}, + Ingress: []v1beta1.NetworkNeighbor{ + { + Identifier: "gitops-clients", + Type: v1beta1.CommunicationTypeIngress, + ServiceRefNamespace: ns.Name, + ServiceRefName: "source-controller", + Ports: []v1beta1.NetworkPort{{Name: "TCP-80", Protocol: v1beta1.ProtocolTCP, Port: &port80}}, + }, + }, + }, + } + _, err := storageClient.ContainerProfiles(ns.Name).Create(context.Background(), cp, metav1.CreateOptions{}) + require.NoError(t, err, "create authored ContainerProfile") + + require.NoError(t, testutils.ApplyMultiDocDir(ns.Name, path.Join(utils.CurrentDir(), "resources/serviceref-suite")), + "apply flux source-controller + helm repo suite") + + patch := []byte(fmt.Sprintf(`{"spec":{"template":{"metadata":{"labels":{"kubescape.io/user-defined-profile":%q}}}}}`, cpName)) + _, err = k8sClient.KubernetesClient.AppsV1().Deployments(ns.Name).Patch(context.TODO(), "helm-repo", types.StrategicMergePatchType, patch, metav1.PatchOptions{}) + require.NoError(t, err, "bind profile to helm-repo pods") + + waitDeploy := func(name string) { + t.Helper() + require.Eventually(t, func() bool { + d, e := k8sClient.KubernetesClient.AppsV1().Deployments(ns.Name).Get(context.TODO(), name, metav1.GetOptions{}) + return e == nil && d.Status.ReadyReplicas > 0 && d.Status.UpdatedReplicas == d.Status.ReadyReplicas + }, 3*time.Minute, 5*time.Second, "%s must become ready", name) + } + waitDeploy("helm-repo") + waitDeploy("source-controller") + + countR0012 := func() int { + alerts, _ := testutils.GetAlerts(ns.Name) + n := 0 + for _, a := range alerts { + if a.Labels["rule_id"] == "R0012" && a.Labels["container_name"] == "nginx" { + n++ + } + } + return n + } + + helmRepoGVR := schema.GroupVersionResource{Group: "source.toolkit.fluxcd.io", Version: "v1", Resource: "helmrepositories"} + repoClient := k8sClient.DynamicClient.Resource(helmRepoGVR).Namespace(ns.Name) + repo := &unstructured.Unstructured{Object: map[string]interface{}{ + "apiVersion": "source.toolkit.fluxcd.io/v1", + "kind": "HelmRepository", + "metadata": map[string]interface{}{"name": "primary", "namespace": ns.Name}, + "spec": map[string]interface{}{ + "interval": "30s", + "url": fmt.Sprintf("http://helm-primary.%s.svc.cluster.local./", ns.Name), + }, + }} + _, err = repoClient.Create(context.TODO(), repo, metav1.CreateOptions{}) + require.NoError(t, err, "create HelmRepository") + + time.Sleep(40 * time.Second) + + t.Run("listed_client_no_r0012", func(t *testing.T) { + // Two reconcile intervals of real source-controller fetches into nginx. + time.Sleep(90 * time.Second) + assert.Equal(t, 0, countR0012(), + "ingress from the serviceRef-listed client must not fire R0012") + }) + + t.Run("unlisted_client_fires_r0012", func(t *testing.T) { + before := countR0012() + require.NoError(t, testutils.ApplyMultiDocYAML(ns.Name, path.Join(utils.CurrentDir(), "resources/serviceref-k6.yaml")), + "deploy unlisted k6 client") + require.Eventually(t, func() bool { + return countR0012() > before + }, 4*time.Minute, 15*time.Second, + "ingress from a client no serviceRef names MUST fire R0012") + }) +} diff --git a/tests/resources/containerprofile-user-defined-network.yaml b/tests/resources/containerprofile-user-defined-network.yaml index f2f6edda1c..94e7928940 100644 --- a/tests/resources/containerprofile-user-defined-network.yaml +++ b/tests/resources/containerprofile-user-defined-network.yaml @@ -58,3 +58,36 @@ spec: - name: TCP-80 protocol: TCP port: 80 + # R0011 excludes only loopback (maximally noisy by design): allow the pod's + # own DNS egress to cluster DNS or every nslookup/curl resolution mints R0011. + - identifier: cluster-dns + type: internal + ipAddresses: + - 10.96.0.0/12 + ports: + - name: UDP-53 + protocol: UDP + port: 53 + - name: TCP-53 + protocol: TCP + port: 53 + - identifier: wildcard-zero-port + type: external + ipAddress: 9.9.9.9 + ports: + - name: TCP-any + protocol: TCP + port: 0 + - identifier: wildcard-empty-ports + type: external + ipAddress: 208.67.222.222 + - identifier: cluster-dns + type: internal + ipAddress: 10.96.0.10 + - identifier: kube-api + type: internal + ipAddress: 10.96.0.1 + ports: + - name: TCP-443 + protocol: TCP + port: 443 diff --git a/tests/resources/network_fixture_lint_test.go b/tests/resources/network_fixture_lint_test.go index a7ec6cd0e2..4d6260b047 100644 --- a/tests/resources/network_fixture_lint_test.go +++ b/tests/resources/network_fixture_lint_test.go @@ -74,6 +74,11 @@ type netEndpoint struct { // declared target for R-NN-12. PodSelector json.RawMessage `json:"podSelector"` NamespaceSelector json.RawMessage `json:"namespaceSelector"` + // Service/entity targets resolved to concrete IPs at projection time. + // Presence of any counts as a declared target for R-NN-12. + ServiceRefName string `json:"serviceRefName"` + ServiceSelector json.RawMessage `json:"serviceSelector"` + Entity string `json:"entity"` } // hasSelector reports whether a raw selector field was set to a real object @@ -114,7 +119,7 @@ func (v NetViolation) String() string { // R-NN-02 — at least one endpoint (egress or ingress) declared // R-NN-10 — endpoint identifier non-empty // R-NN-11 — endpoint type in {internal, external} (or unset) -// R-NN-12 — endpoint declares at least one target (dnsNames/ipAddresses/dns/ipAddress) +// R-NN-12 — endpoint declares at least one target (dnsNames/ipAddresses/dns/ipAddress/selector/serviceRef/entity) // R-NN-13 — dnsNames wildcard tokens are whole-label; no recursive "**", no ascii "..." // R-NN-14 — an entry MUST NOT set both singular ipAddress and plural ipAddresses // R-NN-15 — ipAddresses entries are a literal IP, a CIDR, or the "*" sentinel @@ -176,8 +181,9 @@ func lintEndpoint(dir string, e netEndpoint, add func(rule, msg string)) { add("R-NN-11", where(fmt.Sprintf("type %q is not internal|external", e.Type))) } if len(e.DNSNames) == 0 && len(e.IPAddresses) == 0 && e.DNS == "" && e.IPAddress == "" && - !hasSelector(e.PodSelector) && !hasSelector(e.NamespaceSelector) { - add("R-NN-12", where("endpoint declares no target (dnsNames/ipAddresses/dns/ipAddress/selector)")) + !hasSelector(e.PodSelector) && !hasSelector(e.NamespaceSelector) && + e.ServiceRefName == "" && !hasSelector(e.ServiceSelector) && e.Entity == "" { + add("R-NN-12", where("endpoint declares no target (dnsNames/ipAddresses/dns/ipAddress/selector/serviceRef/entity)")) } if e.IPAddress != "" && len(e.IPAddresses) > 0 { add("R-NN-14", where("sets both singular ipAddress and plural ipAddresses — pick one")) @@ -205,8 +211,9 @@ func lintEndpoint(dir string, e netEndpoint, add func(rule, msg string)) { if p.Protocol != "TCP" && p.Protocol != "UDP" { add("R-NN-20", where(fmt.Sprintf("port %q protocol %q is not TCP|UDP", p.Name, p.Protocol))) } - if p.Port < 1 || p.Port > 65535 { - add("R-NN-20", where(fmt.Sprintf("port %q value %d out of range 1..65535", p.Name, p.Port))) + // Port 0 is the any-port wildcard (matches R0011/R0012 port semantics). + if p.Port != 0 && (p.Port < 1 || p.Port > 65535) { + add("R-NN-20", where(fmt.Sprintf("port %q value %d out of range 1..65535 (0 = any)", p.Name, p.Port))) } } } diff --git a/tests/resources/rules_chart_drift_test.go b/tests/resources/rules_chart_drift_test.go new file mode 100644 index 0000000000..6629dad0b3 --- /dev/null +++ b/tests/resources/rules_chart_drift_test.go @@ -0,0 +1,30 @@ +package resources + +import ( + "os" + "strings" + "testing" +) + +// The standalone rules chart ships a copy of the test chart's rules; a drift +// between the two would deploy different detection semantics than CI validates. +func TestRulesChartMatchesTestChart(t *testing.T) { + pairs := [][2]string{ + {"../chart/templates/node-agent/default-rules.yaml", "../../charts/kubescape-rules/templates/rules.yaml"}, + {"../chart/templates/node-agent/default-rule-binding.yaml", "../../charts/kubescape-rules/templates/binding.yaml"}, + } + for _, p := range pairs { + a, err := os.ReadFile(p[0]) + if err != nil { + t.Fatal(err) + } + b, err := os.ReadFile(p[1]) + if err != nil { + t.Fatal(err) + } + got := strings.ReplaceAll(string(b), "{{ .Values.ksNamespace }}", "kubescape") + if got != string(a) { + t.Errorf("%s drifted from %s — regenerate the chart copy", p[1], p[0]) + } + } +} diff --git a/tests/resources/serviceref-k6.yaml b/tests/resources/serviceref-k6.yaml new file mode 100644 index 0000000000..7d703b33b1 --- /dev/null +++ b/tests/resources/serviceref-k6.yaml @@ -0,0 +1,43 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: k6-script +data: + load.js: | + import http from 'k6/http'; + import { sleep } from 'k6'; + export const options = { vus: 2, duration: '30m' }; + export default function () { + http.get('http://helm-primary/index.yaml', { timeout: '5s' }); + sleep(1); + } +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: k6-load +spec: + replicas: 1 + selector: + matchLabels: + app: k6-load + template: + metadata: + labels: + app: k6-load + spec: + containers: + - name: k6 + image: grafana/k6:0.49.0 + args: ["run", "/scripts/load.js", "--quiet", "--no-usage-report"] + volumeMounts: + - name: scripts + mountPath: /scripts + resources: + limits: + cpu: 200m + memory: 192Mi + volumes: + - name: scripts + configMap: + name: k6-script diff --git a/tests/resources/serviceref-suite/00-flux-source-crds.yaml b/tests/resources/serviceref-suite/00-flux-source-crds.yaml new file mode 100644 index 0000000000..bfc14c9c8d --- /dev/null +++ b/tests/resources/serviceref-suite/00-flux-source-crds.yaml @@ -0,0 +1,132 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: helmrepositories.source.toolkit.fluxcd.io +spec: + group: source.toolkit.fluxcd.io + names: + kind: HelmRepository + listKind: HelmRepositoryList + plural: helmrepositories + singular: helmrepository + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + x-kubernetes-preserve-unknown-fields: true +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: helmcharts.source.toolkit.fluxcd.io +spec: + group: source.toolkit.fluxcd.io + names: + kind: HelmChart + listKind: HelmChartList + plural: helmcharts + singular: helmchart + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + x-kubernetes-preserve-unknown-fields: true +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: gitrepositories.source.toolkit.fluxcd.io +spec: + group: source.toolkit.fluxcd.io + names: + kind: GitRepository + listKind: GitRepositoryList + plural: gitrepositories + singular: gitrepository + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + x-kubernetes-preserve-unknown-fields: true +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: buckets.source.toolkit.fluxcd.io +spec: + group: source.toolkit.fluxcd.io + names: + kind: Bucket + listKind: BucketList + plural: buckets + singular: bucket + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + x-kubernetes-preserve-unknown-fields: true + - name: v1beta2 + served: true + storage: false + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + x-kubernetes-preserve-unknown-fields: true +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: ocirepositories.source.toolkit.fluxcd.io +spec: + group: source.toolkit.fluxcd.io + names: + kind: OCIRepository + listKind: OCIRepositoryList + plural: ocirepositories + singular: ocirepository + scope: Namespaced + versions: + - name: v1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + x-kubernetes-preserve-unknown-fields: true + - name: v1beta2 + served: true + storage: false + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + x-kubernetes-preserve-unknown-fields: true diff --git a/tests/resources/serviceref-suite/10-helm-repo.yaml b/tests/resources/serviceref-suite/10-helm-repo.yaml new file mode 100644 index 0000000000..830be5de98 --- /dev/null +++ b/tests/resources/serviceref-suite/10-helm-repo.yaml @@ -0,0 +1,108 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: helm-repo-index +data: + index.yaml: | + apiVersion: v1 + entries: {} + generated: "2020-01-01T00:00:00Z" +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: helm-repo +spec: + replicas: 1 + selector: + matchLabels: + app: helm-repo + template: + metadata: + labels: + app: helm-repo + spec: + containers: + - name: nginx + image: nginx:1.27-alpine + ports: + - containerPort: 80 + volumeMounts: + - name: index + mountPath: /usr/share/nginx/html + resources: + limits: + cpu: 200m + memory: 128Mi + volumes: + - name: index + configMap: + name: helm-repo-index +--- +apiVersion: v1 +kind: Service +metadata: + name: helm-primary + labels: + role: helm-repo +spec: + selector: + app: helm-repo + ports: + - port: 80 + targetPort: 80 +--- +apiVersion: v1 +kind: Service +metadata: + name: helm-mirror + labels: + role: helm-repo +spec: + selector: + app: helm-repo + ports: + - port: 80 + targetPort: 80 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: decoy-repo +spec: + replicas: 1 + selector: + matchLabels: + app: decoy-repo + template: + metadata: + labels: + app: decoy-repo + spec: + containers: + - name: nginx + image: nginx:1.27-alpine + ports: + - containerPort: 80 + volumeMounts: + - name: index + mountPath: /usr/share/nginx/html + resources: + limits: + cpu: 200m + memory: 128Mi + volumes: + - name: index + configMap: + name: helm-repo-index +--- +apiVersion: v1 +kind: Service +metadata: + name: decoy-repo +spec: + selector: + app: decoy-repo + ports: + - port: 80 + targetPort: 80 diff --git a/tests/resources/serviceref-suite/20-source-controller.yaml b/tests/resources/serviceref-suite/20-source-controller.yaml new file mode 100644 index 0000000000..613fcaae50 --- /dev/null +++ b/tests/resources/serviceref-suite/20-source-controller.yaml @@ -0,0 +1,117 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: source-controller +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: source-controller +rules: + - apiGroups: ["source.toolkit.fluxcd.io"] + resources: ["*"] + verbs: ["*"] + - apiGroups: [""] + resources: ["configmaps", "secrets", "events", "serviceaccounts"] + verbs: ["*"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["*"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: source-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: source-controller +subjects: + - kind: ServiceAccount + name: source-controller +--- +apiVersion: v1 +kind: Service +metadata: + name: source-controller +spec: + selector: + app: source-controller + ports: + - name: http + port: 80 + targetPort: 9090 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: source-controller +spec: + replicas: 1 + selector: + matchLabels: + app: source-controller + strategy: + type: Recreate + template: + metadata: + labels: + app: source-controller + kubescape.io/user-defined-profile: serviceref-flux-cp + spec: + serviceAccountName: source-controller + terminationGracePeriodSeconds: 10 + containers: + - name: manager + image: ghcr.io/fluxcd/source-controller:v1.3.0 + imagePullPolicy: IfNotPresent + args: + - --log-level=info + - --log-encoding=json + - --storage-path=/data + - --storage-adv-addr=source-controller.$(RUNTIME_NAMESPACE).svc.cluster.local. + - --watch-all-namespaces=false + - --enable-leader-election=false + env: + - name: RUNTIME_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: TUF_ROOT + value: /tmp/.sigstore + ports: + - containerPort: 9090 + name: http + - containerPort: 9440 + name: healthz + readinessProbe: + httpGet: + path: / + port: http + resources: + limits: + cpu: 500m + memory: 512Mi + requests: + cpu: 50m + memory: 64Mi + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + readOnlyRootFilesystem: true + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + volumeMounts: + - mountPath: /data + name: data + - mountPath: /tmp + name: tmp + securityContext: + fsGroup: 1337 + volumes: + - name: data + emptyDir: {} + - name: tmp + emptyDir: {} diff --git a/tests/scripts/storage-tag.sh b/tests/scripts/storage-tag.sh index 8db14a5ef2..4f508f9273 100755 --- a/tests/scripts/storage-tag.sh +++ b/tests/scripts/storage-tag.sh @@ -1,4 +1,10 @@ #/bin/bash +# go.mod pins the k8sstormcenter storage fork (3844202a); CTs must run its server image. +if go list -m -f '{{with .Replace}}{{.Path}}{{end}}' github.com/kubescape/storage | grep -q k8sstormcenter/storage; then + echo "net-v2-rc1" + exit 0 +fi + curl -s https://raw.githubusercontent.com/kubescape/helm-charts/main/charts/kubescape-operator/values.yaml -o values.yaml DYNAMIC_TAG=$(yq '.storage.image.tag' < values.yaml | tr -d '"') rm -rf values.yaml diff --git a/tests/testutils/k8s.go b/tests/testutils/k8s.go index b2792d04b9..585af679a6 100644 --- a/tests/testutils/k8s.go +++ b/tests/testutils/k8s.go @@ -10,6 +10,7 @@ import ( "math/rand" "os" "path/filepath" + "sort" "strings" "testing" "time" @@ -28,10 +29,15 @@ import ( "github.com/stretchr/testify/assert" v1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/labels" + "k8s.io/client-go/discovery" + "k8s.io/client-go/discovery/cached/memory" "k8s.io/client-go/dynamic" + "k8s.io/client-go/restmapper" "k8s.io/client-go/tools/remotecommand" "k8s.io/kubectl/pkg/scheme" ) @@ -84,6 +90,75 @@ func NewTestWorkload(namespace, resourcePath string) (*TestWorkload, error) { }, nil } +// ApplyMultiDocYAML creates every document in a multi-document YAML file, +// mapping each object's apiVersion/kind to its resource via server discovery so +// custom resources work without a compiled-in table. Namespaced objects land in +// namespace; cluster-scoped ones (CRDs) ignore it. Already-existing objects are +// not an error, so a fixture may be applied by more than one test. +func ApplyMultiDocYAML(namespace, resourcePath string) error { + k8sClient := k8sinterface.NewKubernetesApi() + raw, err := os.ReadFile(resourcePath) + if err != nil { + return err + } + dc, err := discovery.NewDiscoveryClientForConfig(k8sClient.K8SConfig) + if err != nil { + return err + } + mapper := restmapper.NewDeferredDiscoveryRESTMapper(memory.NewMemCacheClient(dc)) + for _, doc := range strings.Split(string(raw), "\n---") { + if strings.TrimSpace(doc) == "" { + continue + } + jsonData, err := yaml.YAMLToJSON([]byte(doc)) + if err != nil { + return fmt.Errorf("%s: %w", resourcePath, err) + } + obj := &unstructured.Unstructured{} + if err := obj.UnmarshalJSON(jsonData); err != nil { + return fmt.Errorf("%s: %w", resourcePath, err) + } + if obj.GetKind() == "" { + continue + } + gvk := obj.GroupVersionKind() + m, err := mapper.RESTMapping(gvk.GroupKind(), gvk.Version) + if err != nil { + return fmt.Errorf("%s %s: %w", resourcePath, gvk.String(), err) + } + var ri dynamic.ResourceInterface = k8sClient.DynamicClient.Resource(m.Resource) + if m.Scope.Name() == meta.RESTScopeNameNamespace { + ri = k8sClient.DynamicClient.Resource(m.Resource).Namespace(namespace) + } + if _, err := ri.Create(context.TODO(), obj, metav1.CreateOptions{}); err != nil && !apierrors.IsAlreadyExists(err) { + return fmt.Errorf("%s %s/%s: %w", resourcePath, obj.GetKind(), obj.GetName(), err) + } + } + return nil +} + +// ApplyMultiDocDir applies every YAML file in dir in lexical order (filenames +// are numbered so CRDs land before the resources that use them). +func ApplyMultiDocDir(namespace, dir string) error { + entries, err := os.ReadDir(dir) + if err != nil { + return err + } + var names []string + for _, e := range entries { + if !e.IsDir() { + names = append(names, e.Name()) + } + } + sort.Strings(names) + for _, n := range names { + if err := ApplyMultiDocYAML(namespace, filepath.Join(dir, n)); err != nil { + return err + } + } + return nil +} + func (w *TestWorkload) ExecIntoPod(command []string, container string) (string, string, error) { pods, err := w.GetPods() if err != nil {