From b7f4f43e96cdf0771ca19f3b3d27d2810fc7b9a1 Mon Sep 17 00:00:00 2001 From: Maha Benzekri Date: Thu, 20 Aug 2026 09:37:36 +0200 Subject: [PATCH 1/3] Name the critical source error rate alert after its threshold Both rules of the source error rate pair were named IngestionProducerSourceErrorRate3Percent, so the critical one could not be told apart from the warning one. Name it after the threshold it uses, as the zookeeper and publish error rate pairs already do. Issue: BB-605 --- monitoring/ingestion/ingestion-producer-alert.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/monitoring/ingestion/ingestion-producer-alert.yaml b/monitoring/ingestion/ingestion-producer-alert.yaml index 73ec1073b4..41331ae2b4 100644 --- a/monitoring/ingestion/ingestion-producer-alert.yaml +++ b/monitoring/ingestion/ingestion-producer-alert.yaml @@ -84,7 +84,7 @@ groups: description: "Ingestion producer source error rate has increased to 3% in the last 5 minutes" summary: "Ingestion producer source error rate has increased" - - alert: IngestionProducerSourceErrorRate3Percent + - alert: IngestionProducerSourceErrorRate5Percent Expr: | sum(rate(s3_ingestion_source_operations_total{status="error", namespace="${namespace}", job="${job_ingestion_producer}"}[5m])) / From 85484723596d34eff32c3a219eaac8241d95d599 Mon Sep 17 00:00:00 2001 From: Maha Benzekri Date: Thu, 20 Aug 2026 09:37:47 +0200 Subject: [PATCH 2/3] Alert when an ingestion source cannot be set up A source whose setup fails is now retried on every cron tick, so it reports a getRaftId error roughly every five seconds for as long as it cannot be resolved, while its location ingests nothing. Nothing reports it today: the journal size and read offset gauges of that source are never published, so the lag and NoMessagesQueued alerts stay blind, and the error ratio alerts drown the failures in the reads of every healthy source, which are orders of magnitude more frequent. Alert on the absolute rate of getRaftId errors instead, which is only called to set a reader up. Ten minutes of sustained failures are needed to fire, so a source that recovers on its own, typically while its location restarts, does not alert. Issue: BB-605 --- .../ingestion/ingestion-producer-alert.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/monitoring/ingestion/ingestion-producer-alert.yaml b/monitoring/ingestion/ingestion-producer-alert.yaml index 41331ae2b4..602658f329 100644 --- a/monitoring/ingestion/ingestion-producer-alert.yaml +++ b/monitoring/ingestion/ingestion-producer-alert.yaml @@ -98,6 +98,22 @@ groups: description: "Ingestion producer source error rate has increased to 5% in the last 5 minutes" summary: "Ingestion producer source error rate has increased" + # The setup of a reader is retried on every cron tick, so a source that cannot + # be resolved fails continuously. It is a handful of errors against the reads + # of every healthy source, so it only shows up on the absolute rate: the ratio + # alerts above stay far below their threshold when a single source is stuck. + - alert: IngestionProducerSourceSetupFailing + Expr: | + sum(rate(s3_ingestion_source_operations_total{op="getRaftId", status="error", namespace="${namespace}", job="${job_ingestion_producer}"}[5m])) + > 0 + For: "10m" + Labels: + severity: warning + Annotations: + zenko_service: backbeat-ingestion-producer + description: "Ingestion producer has been unable to start reading from a source for the last 10 minutes, so the objects of that location are not ingested. Check the producer logs for the bucket it fails to resolve." + summary: "Ingestion producer cannot start reading from a source" + - alert: IngestionProducerZookeeperErrorRate3Percent Expr: | sum(rate(s3_ingestion_zookeeper_operations_total{status="error", namespace="${namespace}", job="${job_ingestion_producer}"}[5m])) From a682dd2a5f39c975779df802fafe55945abf6a90 Mon Sep 17 00:00:00 2001 From: Maha Benzekri Date: Thu, 20 Aug 2026 09:37:55 +0200 Subject: [PATCH 3/3] Test the ingestion producer alerts The ingestion alerts were the only ones shipped without a rendering and unit test step, so a rule could be published broken or silent. Cover the new setup failure alert: it fires on a source that keeps failing, stays silent on one that recovers, and the ratio alerts are asserted silent on the same series to pin down why it is needed. Issue: BB-605 --- .github/workflows/alerts.yaml | 11 ++++ .../ingestion-producer-alert.test.yaml | 53 +++++++++++++++++++ 2 files changed, 64 insertions(+) create mode 100644 monitoring/ingestion/ingestion-producer-alert.test.yaml diff --git a/.github/workflows/alerts.yaml b/.github/workflows/alerts.yaml index 4ebbb7445c..3e3ba043bc 100644 --- a/.github/workflows/alerts.yaml +++ b/.github/workflows/alerts.yaml @@ -74,6 +74,17 @@ jobs: notification_processor_replicas=2 github_token: ${{ secrets.GIT_ACCESS_TOKEN }} + - name: Render and test ingestion producer + uses: scality/action-prom-render-test@1.0.3 + with: + alert_file_path: monitoring/ingestion/ingestion-producer-alert.yaml + test_file_path: monitoring/ingestion/ingestion-producer-alert.test.yaml + alert_inputs: | + namespace=zenko + job_ingestion_producer=artesca-data-backbeat-ingestion-producer-headless + replicas=1 + github_token: ${{ secrets.GIT_ACCESS_TOKEN }} + - name: Render and test oplog populator uses: scality/action-prom-render-test@1.0.3 with: diff --git a/monitoring/ingestion/ingestion-producer-alert.test.yaml b/monitoring/ingestion/ingestion-producer-alert.test.yaml new file mode 100644 index 0000000000..eab23ba3d6 --- /dev/null +++ b/monitoring/ingestion/ingestion-producer-alert.test.yaml @@ -0,0 +1,53 @@ +evaluation_interval: 1m +rule_files: + - ingestion-producer-alert.rendered.yaml + +tests: + + # A source whose setup keeps failing is retried on every cron tick, so it + # fails at a steady rate while every other source keeps reading normally. + - name: Ingestion Producer Source Setup Failing + interval: 1m + input_series: + - series: s3_ingestion_source_operations_total{namespace="zenko",job="artesca-data-backbeat-ingestion-producer-headless",origin="ingestion",op="getRaftId",status="error"} + values: 0+12x20 + - series: s3_ingestion_source_operations_total{namespace="zenko",job="artesca-data-backbeat-ingestion-producer-headless",origin="ingestion",op="getRaftLog",status="success"} + values: 0+12000x20 + alert_rule_test: + - alertname: IngestionProducerSourceSetupFailing + eval_time: 5m + exp_alerts: [] + - alertname: IngestionProducerSourceSetupFailing + eval_time: 12m + exp_alerts: + - exp_labels: + severity: warning + exp_annotations: + zenko_service: backbeat-ingestion-producer + description: "Ingestion producer has been unable to start reading from a source for the last 10 minutes, so the objects of that location are not ingested. Check the producer logs for the bucket it fails to resolve." + summary: "Ingestion producer cannot start reading from a source" + # The same failures are 0.1% of all the source operations, which is why the + # error ratio alerts cannot report a single source that stopped ingesting. + - alertname: IngestionProducerSourceErrorRate3Percent + eval_time: 12m + exp_alerts: [] + - alertname: IngestionProducerSourceErrorRate5Percent + eval_time: 12m + exp_alerts: [] + + # A source that fails to set up while its location restarts, then recovers, + # must not alert. + - name: Ingestion Producer Source Setup Recovering + interval: 1m + input_series: + - series: s3_ingestion_source_operations_total{namespace="zenko",job="artesca-data-backbeat-ingestion-producer-headless",origin="ingestion",op="getRaftId",status="error"} + values: 0 12 24 24x17 + - series: s3_ingestion_source_operations_total{namespace="zenko",job="artesca-data-backbeat-ingestion-producer-headless",origin="ingestion",op="getRaftId",status="success"} + values: 0 0 0 1+1x17 + alert_rule_test: + - alertname: IngestionProducerSourceSetupFailing + eval_time: 8m + exp_alerts: [] + - alertname: IngestionProducerSourceSetupFailing + eval_time: 15m + exp_alerts: []