From 492ae11cc801d77e49ec2e9f28216305220e3066 Mon Sep 17 00:00:00 2001 From: Arun Date: Sun, 16 Aug 2026 10:54:57 +0000 Subject: [PATCH] Add dmesg enricher actions for node and pod events Adds node_dmesg_enricher and pod_dmesg_enricher playbook actions that fetch the kernel ring buffer (dmesg) from a node and attach it to the finding as a readable file. The pod variant resolves the node from the pod's spec.nodeName. Closes #549 --- .../actions/event-enrichment.rst | 4 + .../robusta_playbooks/dmesg_enrichments.py | 64 ++++++++++++ tests/test_dmesg_enrichments.py | 97 +++++++++++++++++++ 3 files changed, 165 insertions(+) create mode 100644 playbooks/robusta_playbooks/dmesg_enrichments.py create mode 100644 tests/test_dmesg_enrichments.py diff --git a/docs/playbook-reference/actions/event-enrichment.rst b/docs/playbook-reference/actions/event-enrichment.rst index b1c176487..602395467 100644 --- a/docs/playbook-reference/actions/event-enrichment.rst +++ b/docs/playbook-reference/actions/event-enrichment.rst @@ -12,6 +12,8 @@ These actions can add context to any node-related event, be it from ``on_prometh .. robusta-action:: playbooks.robusta_playbooks.bash_enrichments.node_bash_enricher +.. robusta-action:: playbooks.robusta_playbooks.dmesg_enrichments.node_dmesg_enricher + .. robusta-action:: playbooks.robusta_playbooks.node_enrichments.node_status_enricher .. robusta-action:: playbooks.robusta_playbooks.node_enrichments.node_running_pods_enricher @@ -36,6 +38,8 @@ These actions can add context to any pod-related event, be it from ``on_promethe .. robusta-action:: playbooks.robusta_playbooks.bash_enrichments.pod_bash_enricher :trigger-params: {"alert_name": "ExampleLowDiskAlert"} +.. robusta-action:: playbooks.robusta_playbooks.dmesg_enrichments.pod_dmesg_enricher + .. robusta-action:: playbooks.robusta_playbooks.pod_enrichments.pod_graph_enricher .. robusta-action:: playbooks.robusta_playbooks.pod_enrichments.pod_node_graph_enricher diff --git a/playbooks/robusta_playbooks/dmesg_enrichments.py b/playbooks/robusta_playbooks/dmesg_enrichments.py new file mode 100644 index 000000000..04225fe22 --- /dev/null +++ b/playbooks/robusta_playbooks/dmesg_enrichments.py @@ -0,0 +1,64 @@ +import logging +from typing import List, Optional + +from pydantic import BaseModel +from robusta.api import BaseBlock, FileBlock, MarkdownBlock, NodeEvent, PodEvent, RobustaPod, action + + +class DmesgParams(BaseModel): + """ + :var lines: Number of lines to keep from the end of the dmesg output. If not set, the full output is kept. + + :example lines: 100 + """ + + lines: Optional[int] = None + + +def _build_dmesg_command(params: DmesgParams) -> str: + command = "dmesg" + if params.lines is not None: + command = f"{command} | tail -n {params.lines}" + return command + + +def _dmesg_enrichment_blocks(node_name: str, exec_result: str) -> List[BaseBlock]: + block_list: List[BaseBlock] = [] + block_list.append(MarkdownBlock(f"Dmesg results for node *{node_name}:*")) + block_list.append(FileBlock(f"dmesg-{node_name}.log", exec_result.encode())) + return block_list + + +@action +def node_dmesg_enricher(event: NodeEvent, params: DmesgParams): + """ + Fetch the kernel ring buffer (dmesg) from the target **node**. + Enrich the finding with the dmesg output, readable as a file. + """ + node = event.get_node() + if not node: + logging.error(f"cannot run NodeDmesgEnricher on event with no node: {event}") + return + + exec_result = RobustaPod.exec_in_debugger_pod("node-dmesg-pod", node.metadata.name, _build_dmesg_command(params)) + event.add_enrichment(_dmesg_enrichment_blocks(node.metadata.name, exec_result)) + + +@action +def pod_dmesg_enricher(event: PodEvent, params: DmesgParams): + """ + Fetch the kernel ring buffer (dmesg) from the **node** that the target pod is running on. + Enrich the finding with the dmesg output, readable as a file. + """ + pod = event.get_pod() + if not pod: + logging.error(f"cannot run PodDmesgEnricher on event with no pod: {event}") + return + + node_name = pod.spec.nodeName + if not node_name: + logging.error(f"cannot run PodDmesgEnricher on pod {pod.metadata.name} which is not scheduled on a node") + return + + exec_result = RobustaPod.exec_in_debugger_pod("node-dmesg-pod", node_name, _build_dmesg_command(params)) + event.add_enrichment(_dmesg_enrichment_blocks(node_name, exec_result)) diff --git a/tests/test_dmesg_enrichments.py b/tests/test_dmesg_enrichments.py new file mode 100644 index 000000000..63706c28d --- /dev/null +++ b/tests/test_dmesg_enrichments.py @@ -0,0 +1,97 @@ +from unittest import mock + +import pytest +from hikaru.model.rel_1_26 import Node, ObjectMeta, PodSpec +from robusta.core.reporting import FileBlock, MarkdownBlock +from robusta.integrations.kubernetes.autogenerated.events import NodeEvent, PodEvent +from robusta.integrations.kubernetes.custom_models import RobustaPod + +from playbooks.robusta_playbooks.dmesg_enrichments import ( + DmesgParams, + _build_dmesg_command, + node_dmesg_enricher, + pod_dmesg_enricher, +) + +SINK = "test-sink" + + +def _get_blocks(event): + """Return the enrichment blocks added to the event's first finding.""" + finding = event.sink_findings[SINK][0] + assert len(finding.enrichments) == 1 + return finding.enrichments[0].blocks + + +@pytest.mark.parametrize( + "params,expected_command", + [ + (DmesgParams(), "dmesg"), + (DmesgParams(lines=100), "dmesg | tail -n 100"), + (DmesgParams(lines=1), "dmesg | tail -n 1"), + ], +) +def test_build_dmesg_command(params, expected_command): + assert _build_dmesg_command(params) == expected_command + + +def test_node_dmesg_enricher_adds_file_block(): + node = Node(metadata=ObjectMeta(name="test-node")) + event = NodeEvent(obj=node, named_sinks=[SINK]) + + with mock.patch.object( + RobustaPod, "exec_in_debugger_pod", return_value="[ 0.000000] Linux version 5.15.0" + ) as mock_exec: + node_dmesg_enricher(event, DmesgParams(lines=50)) + + mock_exec.assert_called_once_with("node-dmesg-pod", "test-node", "dmesg | tail -n 50") + blocks = _get_blocks(event) + assert len(blocks) == 2 + assert isinstance(blocks[0], MarkdownBlock) + assert "test-node" in blocks[0].text + assert isinstance(blocks[1], FileBlock) + assert blocks[1].filename == "dmesg-test-node.log" + assert blocks[1].contents == b"[ 0.000000] Linux version 5.15.0" + + +def test_node_dmesg_enricher_no_node(): + event = NodeEvent(obj=None, named_sinks=[SINK]) + with mock.patch.object(RobustaPod, "exec_in_debugger_pod") as mock_exec: + node_dmesg_enricher(event, DmesgParams()) + mock_exec.assert_not_called() + + +def test_pod_dmesg_enricher_uses_pod_node(): + pod = RobustaPod( + metadata=ObjectMeta(name="test-pod", namespace="default"), + spec=PodSpec(containers=[], nodeName="worker-1"), + ) + event = PodEvent(obj=pod, named_sinks=[SINK]) + + with mock.patch.object(RobustaPod, "exec_in_debugger_pod", return_value="dmesg output") as mock_exec: + pod_dmesg_enricher(event, DmesgParams()) + + mock_exec.assert_called_once_with("node-dmesg-pod", "worker-1", "dmesg") + blocks = _get_blocks(event) + assert len(blocks) == 2 + assert isinstance(blocks[1], FileBlock) + assert blocks[1].filename == "dmesg-worker-1.log" + assert blocks[1].contents == b"dmesg output" + + +def test_pod_dmesg_enricher_no_pod(): + event = PodEvent(obj=None, named_sinks=[SINK]) + with mock.patch.object(RobustaPod, "exec_in_debugger_pod") as mock_exec: + pod_dmesg_enricher(event, DmesgParams()) + mock_exec.assert_not_called() + + +def test_pod_dmesg_enricher_unscheduled_pod(): + pod = RobustaPod( + metadata=ObjectMeta(name="pending-pod", namespace="default"), + spec=PodSpec(containers=[]), + ) + event = PodEvent(obj=pod, named_sinks=[SINK]) + with mock.patch.object(RobustaPod, "exec_in_debugger_pod") as mock_exec: + pod_dmesg_enricher(event, DmesgParams()) + mock_exec.assert_not_called()