From c3470637b0b052ad02709f2338626931e43c55c9 Mon Sep 17 00:00:00 2001 From: vasavi-r Date: Tue, 26 May 2026 23:11:18 +0530 Subject: [PATCH 1/3] Updated documentation changes in infrastructure integration paths and added Kafka and Keycloak --- .../sending-data-to-kloudmate.mdx | 6 +- .../docs/docs/infrastructure/index.mdx | 6 +- .../infrastructure/integrations/_meta.json | 4 + .../integrations/kafka-integration.mdx | 116 ++++++++++++++++++ .../integrations/keycloak-integration.mdx | 74 +++++++++++ .../prometheus-integration.mdx | 4 +- .../vcenter-integration.mdx | 4 +- 7 files changed, 207 insertions(+), 7 deletions(-) create mode 100644 src/content/docs/docs/infrastructure/integrations/_meta.json create mode 100644 src/content/docs/docs/infrastructure/integrations/kafka-integration.mdx create mode 100644 src/content/docs/docs/infrastructure/integrations/keycloak-integration.mdx rename src/content/docs/docs/infrastructure/{ => integrations}/prometheus-integration.mdx (96%) rename src/content/docs/docs/infrastructure/{ => integrations}/vcenter-integration.mdx (99%) diff --git a/src/content/docs/docs/getting-started/sending-data-to-kloudmate.mdx b/src/content/docs/docs/getting-started/sending-data-to-kloudmate.mdx index 5dafd65b..4cfd9218 100644 --- a/src/content/docs/docs/getting-started/sending-data-to-kloudmate.mdx +++ b/src/content/docs/docs/getting-started/sending-data-to-kloudmate.mdx @@ -33,7 +33,7 @@ Choose the ingestion path below that best matches your existing architecture, yo @@ -57,7 +57,7 @@ If you are unsure where to begin, use this table to map your immediate goal to t | --- | --- | | Fastest overall onboarding | [KloudMate Agent](../../kloudmate-agent/) | | Reuse existing OTel pipelines | [OpenTelemetry Collector](../../opentelemetry/) | -| Import Prometheus metrics | [Prometheus Integration](../../infrastructure/prometheus-integration/) | +| Import Prometheus metrics | [Prometheus Integration](../../infrastructure/integrations/prometheus-integration/) | | Onboard AWS with minimal setup | [AWS Integration](../../aws-integration/) | | Onboard Azure services | [Azure Integration](../../azure-integration/) | @@ -67,7 +67,7 @@ For teams starting from scratch, we recommend this phased approach: 1. **Infrastructure First:** Start with the [KloudMate Agent](../../kloudmate-agent/) to get baseline telemetry, host metrics, and zero-code visibility. 2. **Application Tracing:** Add [OpenTelemetry](../../opentelemetry/) instrumentation to your applications where you need richer distributed tracing or custom metrics. -3. **Cloud Integrations:** Layer in cloud-specific integrations for [AWS](../../aws-integration/), [Azure](../../azure-integration/), or [Prometheus](../../infrastructure/prometheus-integration/) to centralize your remaining data sources. +3. **Cloud Integrations:** Layer in cloud-specific integrations for [AWS](../../aws-integration/), [Azure](../../azure-integration/), or [Prometheus](../../infrastructure/integrations/prometheus-integration/) to centralize your remaining data sources. *** diff --git a/src/content/docs/docs/infrastructure/index.mdx b/src/content/docs/docs/infrastructure/index.mdx index 91abfbb0..35e9c5ab 100644 --- a/src/content/docs/docs/infrastructure/index.mdx +++ b/src/content/docs/docs/infrastructure/index.mdx @@ -22,8 +22,10 @@ This section covers common infrastructure ingestion paths, whether you are monit - [Hosts & Containers](./hosts/) - [Advanced Host Configuration](./server-metrics-to-kloudmate/) - [Kubernetes Monitoring](./kubernetes/) -- [Prometheus Integration](./prometheus-integration/) -- [vCenter Integration](./vcenter-integration/) +- [Prometheus Integration](./integrations/prometheus-integration/) +- [vCenter Integration](./integrations/vcenter-integration/) +- [Kafka Integration](./integrations/kafka-integration/) +- [Keycloak Integration](./integrations/keycloak-integration/) - [AWS Integration](../../aws-integration/) - [Azure Integration](../../azure-integration/) diff --git a/src/content/docs/docs/infrastructure/integrations/_meta.json b/src/content/docs/docs/infrastructure/integrations/_meta.json new file mode 100644 index 00000000..18901ba9 --- /dev/null +++ b/src/content/docs/docs/infrastructure/integrations/_meta.json @@ -0,0 +1,4 @@ +{ + "label": "Integrations", + "order": 4 +} diff --git a/src/content/docs/docs/infrastructure/integrations/kafka-integration.mdx b/src/content/docs/docs/infrastructure/integrations/kafka-integration.mdx new file mode 100644 index 00000000..a1d58de1 --- /dev/null +++ b/src/content/docs/docs/infrastructure/integrations/kafka-integration.mdx @@ -0,0 +1,116 @@ +--- +title: "Kafka Integration" +description: "Collect Kafka broker, topic, and consumer metrics in KloudMate through the KloudMate Agent." +sidebar: + order: 3 +--- +Use this integration when you want to monitor Kafka brokers, topics, and consumers in KloudMate through the OpenTelemetry `kafkametrics` receiver running in the KloudMate Agent. + +## Prerequisites + +1. Install and run the [KloudMate Agent](../../../kloudmate-agent/). +2. Ensure your Kafka brokers are reachable from the node where the Agent is running. +3. Have your Kafka broker addresses, protocol version, and authentication credentials ready. + +## Step 1: Define the Kafka Metrics Receiver + +Add the following `kafkametrics` receiver to your Agent configuration file (typically `/etc/km-agent/config.yaml`). Replace the broker addresses, credentials, and protocol version to match your environment. + +```yaml +extensions: + health_check: + endpoint: 0.0.0.0:13133 + +receivers: + kafkametrics: + brokers: + - kafka.example.com:9095 + - kafka.example.com:9096 + - kafka.example.com:9097 + protocol_version: 2.8.0 + auth: + plain_text: + username: + password: + scrapers: + - brokers + - topics + - consumers + metrics: + kafka.broker.log_retention_period: + enabled: true + kafka.topic.replication_factor: + enabled: true + kafka.topic.log_retention_period: + enabled: true + kafka.topic.log_retention_size: + enabled: true + kafka.topic.min_insync_replicas: + enabled: true +``` + +The `scrapers` block controls which data is collected. `brokers`, `topics`, and `consumers` are the three available options. The metrics listed under the `metrics` block are not enabled by default and must be explicitly set to `enabled: true` to be collected. + +## Step 2: Add to Pipeline + +Add the `kafkametrics` receiver to the metrics pipeline in your Agent configuration. + +```yaml +service: + extensions: + - health_check + telemetry: + metrics: + level: none + pipelines: + metrics/kafkametrics: + receivers: + - kafkametrics + processors: + - batch + exporters: + - otlphttp +``` + +## Step 3: Restart the Agent + +```bash +systemctl restart kmagent +systemctl status kmagent +``` + +## Step 4: Validate in KloudMate + +After the Agent restarts: + +- Open [Explore](../../explore/) in KloudMate. +- Query one of the scraped Kafka metrics, such as `kafka.brokers` or `kafka.topic.partitions`. +- Build dashboards or alarms once the data is visible. + +## Standard Kafka Dashboards + +KloudMate provides prebuilt Kafka dashboards through [dashboard templates](https://templates.kloudmate.com/kafka/index.html). These dashboards help you visualize broker availability, topic partitions, consumer lag, offsets, and replication health. + +To import and start using these templates, follow the steps described in [Import from Templates](../../visualize-data/dashboards/create-a-dashboard/). + +## Default Kafka Metrics + +Kafka Monitoring automatically collects commonly used Kafka metrics when enabled. + +Example Metrics + +| **Metric Name** | **Description** | +| --- | --- | +| kafka.brokers | Number of brokers in the cluster. | +| kafka.topic.partitions | Number of partitions in a topic. | +| kafka.partition.current_offset | Current offset of a topic partition. | +| kafka.partition.oldest_offset | Oldest available offset of a topic partition. | +| kafka.partition.replicas | Number of replicas configured for a topic partition. | +| kafka.partition.replicas_in_sync | Number of in-sync replicas for a topic partition. | +| kafka.consumer_group.members | Number of members in a consumer group. | +| kafka.consumer_group.lag | Current approximate lag of a consumer group for a topic partition. | +| kafka.consumer_group.lag_sum | Current approximate sum of consumer group lag across all topic partitions. | +| kafka.consumer_group.offset | Current consumer group offset for a topic partition. | +| kafka.consumer_group.offset_sum | Sum of consumer group offsets across topic partitions. | + +For the complete metrics list, refer to the [metrics reference](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/receiver/kafkametricsreceiver/documentation.md). diff --git a/src/content/docs/docs/infrastructure/integrations/keycloak-integration.mdx b/src/content/docs/docs/infrastructure/integrations/keycloak-integration.mdx new file mode 100644 index 00000000..b71461f4 --- /dev/null +++ b/src/content/docs/docs/infrastructure/integrations/keycloak-integration.mdx @@ -0,0 +1,74 @@ +--- +title: "Keycloak Integration" +description: "Collect Keycloak metrics in KloudMate through the KloudMate Agent." +sidebar: + order: 4 +--- +Use this integration when you want to scrape Keycloak metrics from its Prometheus-compatible metrics endpoint and send them to KloudMate through the KloudMate Agent. + +## Prerequisites + +1. Install and run the [KloudMate Agent](../../../kloudmate-agent/). +2. Ensure your Keycloak service exposes the `/auth/metrics` endpoint. If this endpoint is not available, no metrics will be scraped. +3. Keycloak must be reachable from the node where the KloudMate Agent is running. + +## Step 1: Define the Prometheus Receiver + +Add the following Prometheus receiver to your Agent configuration file (typically `/etc/km-agent/config.yaml`). Replace `` and `` with your actual Kubernetes service name and namespace. + +```yaml +extensions: + health_check: + endpoint: 0.0.0.0:13133 + +receivers: + prometheus: + config: + scrape_configs: + - job_name: keycloak + metrics_path: /auth/metrics + scheme: http + static_configs: + - targets: + - ..svc.cluster.local:8080 +``` + +:::info +Validate indentation carefully when copying `scrape_configs` into the Agent configuration. +::: + +## Step 2: Add to Pipeline + +Add the Prometheus receiver to the metrics pipeline in your Agent configuration. + +```yaml +service: + extensions: + - health_check + telemetry: + metrics: + level: none + pipelines: + metrics/keycloak: + receivers: + - prometheus + processors: + - batch + exporters: + - otlphttp +``` + +## Step 3: Restart the Agent + +```bash +systemctl restart kmagent +systemctl status kmagent +``` + +## Step 4: Validate in KloudMate + +After the Agent restarts: + +- Open [Explore](../../explore/) in KloudMate. +- Query one of the scraped Keycloak metrics. +- Build dashboards or alarms once data is visible. diff --git a/src/content/docs/docs/infrastructure/prometheus-integration.mdx b/src/content/docs/docs/infrastructure/integrations/prometheus-integration.mdx similarity index 96% rename from src/content/docs/docs/infrastructure/prometheus-integration.mdx rename to src/content/docs/docs/infrastructure/integrations/prometheus-integration.mdx index e03240ef..43ebaa04 100644 --- a/src/content/docs/docs/infrastructure/prometheus-integration.mdx +++ b/src/content/docs/docs/infrastructure/integrations/prometheus-integration.mdx @@ -1,6 +1,8 @@ --- title: "Prometheus Integration" description: "Ingest Prometheus metrics into KloudMate through the KloudMate Agent." +sidebar: + order: 1 --- Use this integration when your workloads already expose Prometheus metrics and you want KloudMate to ingest them without changing application code. @@ -8,7 +10,7 @@ KloudMate does not scrape Prometheus directly. Instead, you configure the KloudM ## Prerequisites -1. Install and run the [KloudMate Agent](../../kloudmate-agent/). +1. Install and run the [KloudMate Agent](../../../kloudmate-agent/). 2. Ensure you can access the Prometheus configuration or know the scrape targets you want the Agent to use. ## Step 1: Define the Prometheus Receiver diff --git a/src/content/docs/docs/infrastructure/vcenter-integration.mdx b/src/content/docs/docs/infrastructure/integrations/vcenter-integration.mdx similarity index 99% rename from src/content/docs/docs/infrastructure/vcenter-integration.mdx rename to src/content/docs/docs/infrastructure/integrations/vcenter-integration.mdx index 23b3437d..ad18f82e 100644 --- a/src/content/docs/docs/infrastructure/vcenter-integration.mdx +++ b/src/content/docs/docs/infrastructure/integrations/vcenter-integration.mdx @@ -1,6 +1,8 @@ --- title: "vCenter Integration" description: "Collect VMware vCenter metrics in KloudMate through the OpenTelemetry vCenter receiver." +sidebar: + order: 2 --- Use this guide to collect VMware vCenter metrics in KloudMate with the OpenTelemetry `vcenter` receiver running via the KloudMate Agent. @@ -9,7 +11,7 @@ Use this guide to collect VMware vCenter metrics in KloudMate with the OpenTelem 1. Run vCenter 7.0 or 8.x. 2. Create a read-only vSphere user with access to the vCenter server, clusters, and resources you want to monitor. 3. Identify the vCenter or ESXi endpoint with SDK access enabled. -4. Install the [KloudMate Agent](../../kloudmate-agent/) on a host that can reach vCenter. +4. Install the [KloudMate Agent](../../../kloudmate-agent/) on a host that can reach vCenter. ## Step 1: Configure the Receiver From 9b3ee4142afa3b6c0332502ba387007e8c126dc8 Mon Sep 17 00:00:00 2001 From: vasavi-r Date: Wed, 27 May 2026 11:45:58 +0530 Subject: [PATCH 2/3] `Update documentation links to reflect new path structure` --- .../docs/infrastructure/integrations/kafka-integration.mdx | 4 ++-- .../infrastructure/integrations/keycloak-integration.mdx | 2 +- .../infrastructure/integrations/prometheus-integration.mdx | 6 +++--- .../infrastructure/integrations/vcenter-integration.mdx | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/content/docs/docs/infrastructure/integrations/kafka-integration.mdx b/src/content/docs/docs/infrastructure/integrations/kafka-integration.mdx index a1d58de1..0a91ab3c 100644 --- a/src/content/docs/docs/infrastructure/integrations/kafka-integration.mdx +++ b/src/content/docs/docs/infrastructure/integrations/kafka-integration.mdx @@ -83,7 +83,7 @@ systemctl status kmagent After the Agent restarts: -- Open [Explore](../../explore/) in KloudMate. +- Open [Explore](../../../visualize-data/explore/) in KloudMate. - Query one of the scraped Kafka metrics, such as `kafka.brokers` or `kafka.topic.partitions`. - Build dashboards or alarms once the data is visible. @@ -91,7 +91,7 @@ After the Agent restarts: KloudMate provides prebuilt Kafka dashboards through [dashboard templates](https://templates.kloudmate.com/kafka/index.html). These dashboards help you visualize broker availability, topic partitions, consumer lag, offsets, and replication health. -To import and start using these templates, follow the steps described in [Import from Templates](../../visualize-data/dashboards/create-a-dashboard/). +To import and start using these templates, follow the steps described in [Import from Templates](../../../visualize-data/dashboards/create-a-dashboard/). ## Default Kafka Metrics diff --git a/src/content/docs/docs/infrastructure/integrations/keycloak-integration.mdx b/src/content/docs/docs/infrastructure/integrations/keycloak-integration.mdx index b71461f4..8ca2ec51 100644 --- a/src/content/docs/docs/infrastructure/integrations/keycloak-integration.mdx +++ b/src/content/docs/docs/infrastructure/integrations/keycloak-integration.mdx @@ -69,6 +69,6 @@ systemctl status kmagent After the Agent restarts: -- Open [Explore](../../explore/) in KloudMate. +- Open [Explore](../../../visualize-data/explore/) in KloudMate. - Query one of the scraped Keycloak metrics. - Build dashboards or alarms once data is visible. diff --git a/src/content/docs/docs/infrastructure/integrations/prometheus-integration.mdx b/src/content/docs/docs/infrastructure/integrations/prometheus-integration.mdx index 43ebaa04..2e1f77ee 100644 --- a/src/content/docs/docs/infrastructure/integrations/prometheus-integration.mdx +++ b/src/content/docs/docs/infrastructure/integrations/prometheus-integration.mdx @@ -67,11 +67,11 @@ systemctl status kmagent After the Agent restarts: -- open [Explore](../../visualize-data/explore/) +- open [Explore](../../../visualize-data/explore/) - query one of the scraped Prometheus metrics - build dashboards or alarms once the data is visible ## Related Paths -- [KloudMate Agent](../../kloudmate-agent/) -- [Explore](../../visualize-data/explore/) +- [KloudMate Agent](../../../kloudmate-agent/) +- [Explore](../../../visualize-data/explore/) diff --git a/src/content/docs/docs/infrastructure/integrations/vcenter-integration.mdx b/src/content/docs/docs/infrastructure/integrations/vcenter-integration.mdx index ad18f82e..132c563d 100644 --- a/src/content/docs/docs/infrastructure/integrations/vcenter-integration.mdx +++ b/src/content/docs/docs/infrastructure/integrations/vcenter-integration.mdx @@ -59,7 +59,7 @@ For Windows, restart the `KloudMate Agent` service from `services.msc`. After the Agent restarts: -- open [Explore](../../visualize-data/explore/) +- open [Explore](../../../visualize-data/explore/) - query vCenter metrics such as `vcenter_host_cpu_utilization` or `vcenter_vm_memory_usage` - build dashboards and alerts once the data is visible From 850e32de9ce759285903b3b72efb8854ab438ea0 Mon Sep 17 00:00:00 2001 From: vasavi-r Date: Wed, 27 May 2026 22:31:36 +0530 Subject: [PATCH 3/3] Add Monitor SNMP Devices Draft under Network Monitoring in Guides --- .../docs/guides/network-monitoring/_meta.json | 4 + .../network-monitoring/monitoring-snmp.mdx | 180 ++++++++++++++++++ 2 files changed, 184 insertions(+) create mode 100644 src/content/docs/guides/network-monitoring/_meta.json create mode 100644 src/content/docs/guides/network-monitoring/monitoring-snmp.mdx diff --git a/src/content/docs/guides/network-monitoring/_meta.json b/src/content/docs/guides/network-monitoring/_meta.json new file mode 100644 index 00000000..aac507a8 --- /dev/null +++ b/src/content/docs/guides/network-monitoring/_meta.json @@ -0,0 +1,4 @@ +{ + "label": "Network Monitoring", + "order": 3 +} diff --git a/src/content/docs/guides/network-monitoring/monitoring-snmp.mdx b/src/content/docs/guides/network-monitoring/monitoring-snmp.mdx new file mode 100644 index 00000000..262304ef --- /dev/null +++ b/src/content/docs/guides/network-monitoring/monitoring-snmp.mdx @@ -0,0 +1,180 @@ +--- +title: "Monitor SNMP Devices" +description: "Draft guide for monitoring SNMP devices with the KloudMate Agent." +draft: true +--- + +This guide shows how to monitor SNMP devices with the KloudMate Agent. The sample instrumentation below illustrates SNMP monitoring for a `FORTINET-FORTIGATE` firewall device. + +## Prerequisites + +For monitoring SNMP devices via KloudMate, the following requirements must be fulfilled: + +1. A VM is available where the KloudMate Agent will run. It must have access to the network device being monitored. +2. The VM IP address is whitelisted for fetching SNMP data from the device. Otherwise, no metrics will be fetched. +3. The network device is configured to send SNMP trap data to the VM, if trap collection is required. +4. The required MIBs for the network device are available. In most cases, both the brand MIB and product MIB are required. For example, when monitoring a Fortinet FortiGate firewall, both `FORTINET-CORE-MIB` and `FORTINET-FORTIGATE-MIB` are required. +5. The list of signals to be monitored is identified in advance, such as memory, CPU, VPN status, firewall, disk, uptime, and link status. Based on this list, the complete agent configuration can be prepared. + +## Sample Instrumentation + +The sample instrumentation illustrates the SNMP monitoring of `FORTINET-FORTIGATE`, a firewall device. Based on the MIBs provided, the agent configuration describes the signals that will be monitored along with detailed information for each metric, such as name, description, type, and associated attributes. + +```yaml +receivers: + snmp: + endpoint: udp://:1161 + version: v2c + community: monitoring + collection_interval: 30s + timeout: 10s + resource_attributes: + device.name: + scalar_oid: 1.3.6.1.2.1.1.5.0 + attributes: + interface.name: + oid: 1.3.6.1.2.1.31.1.1.1.1 + interface.direction: + enum: + - in + - out + vpn.tunnel.name: + oid: 1.3.6.1.4.1.12356.101.12.2.2.1.2 + metrics: + fortigate.cpu.usage: + unit: "%" + description: Overall CPU utilization percentage + gauge: + value_type: int + scalar_oids: + - oid: 1.3.6.1.4.1.12356.101.4.1.3.0 + resource_attributes: + - device.name + fortigate.memory.usage: + unit: "%" + description: Memory utilization percentage + gauge: + value_type: int + scalar_oids: + - oid: 1.3.6.1.4.1.12356.101.4.1.4.0 + resource_attributes: + - device.name + fortigate.memory.capacity: + unit: KB + description: Total physical memory installed in KB + gauge: + value_type: int + scalar_oids: + - oid: 1.3.6.1.4.1.12356.101.4.1.5.0 + resource_attributes: + - device.name + fortigate.sessions.count: + unit: "{sessions}" + description: Number of active IPv4 sessions on the device + gauge: + value_type: int + scalar_oids: + - oid: 1.3.6.1.4.1.12356.101.4.1.8.0 + resource_attributes: + - device.name + fortigate.uptime: + unit: cs + description: Device uptime in centiseconds since last re-initialization + gauge: + value_type: int + scalar_oids: + - oid: 1.3.6.1.4.1.12356.101.4.1.20.0 + resource_attributes: + - device.name + interface.status.operational: + unit: "{status}" + description: Interface operational status (1=up 2=down) + gauge: + value_type: int + column_oids: + - oid: 1.3.6.1.2.1.2.2.1.8 + attributes: + - name: interface.name + resource_attributes: + - device.name + network.io.bytes: + unit: By + description: Total bytes transferred per interface (64-bit counters) + sum: + aggregation: cumulative + monotonic: true + value_type: int + column_oids: + - oid: 1.3.6.1.2.1.31.1.1.1.6 + attributes: + - name: interface.name + - name: interface.direction + value: in + resource_attributes: + - device.name + - oid: 1.3.6.1.2.1.31.1.1.1.10 + attributes: + - name: interface.name + - name: interface.direction + value: out + resource_attributes: + - device.name + fortigate.vpn.tunnel.status: + unit: "{status}" + description: IPsec VPN tunnel status (1=down 2=up) + gauge: + value_type: int + column_oids: + - oid: 1.3.6.1.4.1.12356.101.12.2.2.1.20 + attributes: + - name: vpn.tunnel.name + resource_attributes: + - device.name + fortigate.vpn.tunnel.bytes: + unit: By + description: Bytes transferred per IPsec VPN tunnel + sum: + aggregation: cumulative + monotonic: true + value_type: int + column_oids: + - oid: 1.3.6.1.4.1.12356.101.12.2.2.1.18 + attributes: + - name: vpn.tunnel.name + - name: interface.direction + value: in + resource_attributes: + - device.name + - oid: 1.3.6.1.4.1.12356.101.12.2.2.1.19 + attributes: + - name: vpn.tunnel.name + - name: interface.direction + value: out + resource_attributes: + - device.name +``` + +## Step 1: Add the Receiver to the Pipeline + +```yaml +service: + pipelines: + metrics/snmp: + receivers: + - snmp + processors: + - batch + exporters: + - otlphttp +``` + +## Step 2: Restart the Agent + +```bash +systemctl restart kmagent +systemctl status kmagent +``` + +## Step 3: Verify Metrics in KloudMate + +After restarting the agent, query the configured SNMP metrics in [Explore](../../visualize-data/explore/). You should see telemetry for the monitored device.