From e704cdd4626cf52fccabafd7e60b58a91a679957 Mon Sep 17 00:00:00 2001 From: John T Skarbek Date: Fri, 16 Oct 2026 09:12:03 -0400 Subject: [PATCH] fix(alloy): scrape all alloy pod targets on the Alloy HTTP port Pod discovery makes one scrape target for each declared container port. It makes a target on port 80 for containers that declare no port, such as config-reloader. Alloy does not listen on port 80, so the config-reloader target is always down. Set the prometheus.io/port annotation on the Alloy pod to the Alloy HTTP port by default. Operators no longer need to set it in their own values. --- README.md | 1 + coder-observability/values.yaml | 5 +++++ compiled/resources.yaml | 1 + 3 files changed, 7 insertions(+) diff --git a/README.md b/README.md index ec9d13b..60c2db0 100644 --- a/README.md +++ b/README.md @@ -319,6 +319,7 @@ values which are defined [here](https://github.com/grafana/helm-charts/tree/main | alloy.alloy.mounts.dockercontainers | bool | `true` | | | alloy.alloy.mounts.varlog | bool | `true` | | | alloy.commonRelabellings | string | `"rule {\n source_labels = [\"__meta_kubernetes_namespace\"]\n target_label = \"namespace\"\n}\nrule {\n source_labels = [\"__meta_kubernetes_pod_name\"]\n target_label = \"pod\"\n}\n// coalesce the following labels and pick the first value; we'll use this to define the \"job\" label\nrule {\n source_labels = [\"__meta_kubernetes_pod_label_app_kubernetes_io_component\", \"app\", \"__meta_kubernetes_pod_container_name\"]\n separator = \"/\"\n target_label = \"__meta_app\"\n action = \"replace\"\n regex = \"^/*([^/]+?)(?:/.*)?$\" // split by the delimiter if it exists, we only want the first one\n replacement = \"${1}\"\n}\nrule {\n source_labels = [\"__meta_kubernetes_namespace\", \"__meta_kubernetes_pod_label_app_kubernetes_io_name\", \"__meta_app\"]\n separator = \"/\"\n target_label = \"job\"\n}\nrule {\n source_labels = [\"__meta_kubernetes_pod_container_name\"]\n target_label = \"container\"\n}\nrule {\n regex = \"__meta_kubernetes_pod_label_(statefulset_kubernetes_io_pod_name|controller_revision_hash)\"\n action = \"labeldrop\"\n}\nrule {\n regex = \"pod_template_generation\"\n action = \"labeldrop\"\n}\nrule {\n source_labels = [\"__meta_kubernetes_pod_phase\"]\n regex = \"Pending|Succeeded|Failed|Completed\"\n action = \"drop\"\n}\nrule {\n source_labels = [\"__meta_kubernetes_pod_node_name\"]\n action = \"replace\"\n target_label = \"node\"\n}\nrule {\n action = \"labelmap\"\n regex = \"__meta_kubernetes_pod_annotation_prometheus_io_param_(.+)\"\n replacement = \"__param_$1\"\n}"` | | +| alloy.controller.podAnnotations."prometheus.io/port" | string | `"12345"` | | | alloy.controller.podAnnotations."prometheus.io/scrape" | string | `"true"` | | | alloy.controller.type | string | `"daemonset"` | | | alloy.crds.create | bool | `false` | | diff --git a/coder-observability/values.yaml b/coder-observability/values.yaml index 834739a..586d247 100644 --- a/coder-observability/values.yaml +++ b/coder-observability/values.yaml @@ -211,6 +211,11 @@ alloy: type: daemonset podAnnotations: prometheus.io/scrape: "true" + # Pod discovery makes one target per declared container port, and a target + # on port 80 for containers that declare none (such as config-reloader). + # Pin every target of this pod to the Alloy HTTP port. + # Must equal alloy.alloy.listenPort. + prometheus.io/port: "12345" crds: create: false diff --git a/compiled/resources.yaml b/compiled/resources.yaml index c735291..5d5d0d7 100644 --- a/compiled/resources.yaml +++ b/compiled/resources.yaml @@ -12072,6 +12072,7 @@ spec: metadata: annotations: kubectl.kubernetes.io/default-container: alloy + prometheus.io/port: "12345" prometheus.io/scrape: "true" labels: app.kubernetes.io/name: alloy