Improvement/bb 605 alert for ingestion processing

.github/workflows/alerts.yaml

@@ -74,6 +74,17 @@ jobs:
             notification_processor_replicas=2
           github_token: ${{ secrets.GIT_ACCESS_TOKEN }}
 
+      - name: Render and test ingestion producer
+        uses: scality/action-prom-render-test@1.0.3
+        with:
+          alert_file_path: monitoring/ingestion/ingestion-producer-alert.yaml
+          test_file_path: monitoring/ingestion/ingestion-producer-alert.test.yaml
+          alert_inputs: |
+            namespace=zenko
+            job_ingestion_producer=artesca-data-backbeat-ingestion-producer-headless
+            replicas=1
+          github_token: ${{ secrets.GIT_ACCESS_TOKEN }}
+
       - name: Render and test oplog populator 
         uses: scality/action-prom-render-test@1.0.3
         with:

monitoring/ingestion/ingestion-producer-alert.test.yaml

@@ -0,0 +1,53 @@
+evaluation_interval: 1m
+rule_files:
+  - ingestion-producer-alert.rendered.yaml
+
+tests:
+
+  # A source whose setup keeps failing is retried on every cron tick, so it
+  # fails at a steady rate while every other source keeps reading normally.
+  - name: Ingestion Producer Source Setup Failing
+    interval: 1m
+    input_series:
+      - series: s3_ingestion_source_operations_total{namespace="zenko",job="artesca-data-backbeat-ingestion-producer-headless",origin="ingestion",op="getRaftId",status="error"}
+        values: 0+12x20
+      - series: s3_ingestion_source_operations_total{namespace="zenko",job="artesca-data-backbeat-ingestion-producer-headless",origin="ingestion",op="getRaftLog",status="success"}
+        values: 0+12000x20
+    alert_rule_test:
+      - alertname: IngestionProducerSourceSetupFailing
+        eval_time: 5m
+        exp_alerts: []
+      - alertname: IngestionProducerSourceSetupFailing
+        eval_time: 12m
+        exp_alerts:
+          - exp_labels:
+              severity: warning
+            exp_annotations:
+              zenko_service: backbeat-ingestion-producer
+              description: "Ingestion producer has been unable to start reading from a source for the last 10 minutes, so the objects of that location are not ingested. Check the producer logs for the bucket it fails to resolve."
+              summary: "Ingestion producer cannot start reading from a source"
+      # The same failures are 0.1% of all the source operations, which is why the
+      # error ratio alerts cannot report a single source that stopped ingesting.
+      - alertname: IngestionProducerSourceErrorRate3Percent
+        eval_time: 12m
+        exp_alerts: []
+      - alertname: IngestionProducerSourceErrorRate5Percent
+        eval_time: 12m
+        exp_alerts: []
+
+  # A source that fails to set up while its location restarts, then recovers,
+  # must not alert.
+  - name: Ingestion Producer Source Setup Recovering
+    interval: 1m
+    input_series:
+      - series: s3_ingestion_source_operations_total{namespace="zenko",job="artesca-data-backbeat-ingestion-producer-headless",origin="ingestion",op="getRaftId",status="error"}
+        values: 0 12 24 24x17
+      - series: s3_ingestion_source_operations_total{namespace="zenko",job="artesca-data-backbeat-ingestion-producer-headless",origin="ingestion",op="getRaftId",status="success"}
+        values: 0 0 0 1+1x17
+    alert_rule_test:
+      - alertname: IngestionProducerSourceSetupFailing
+        eval_time: 8m
+        exp_alerts: []
+      - alertname: IngestionProducerSourceSetupFailing
+        eval_time: 15m
+        exp_alerts: []

monitoring/ingestion/ingestion-producer-alert.yaml

@@ -84,7 +84,7 @@ groups:
       description: "Ingestion producer source error rate has increased to 3% in the last 5 minutes"
       summary: "Ingestion producer source error rate has increased"
 
-  - alert: IngestionProducerSourceErrorRate3Percent
+  - alert: IngestionProducerSourceErrorRate5Percent
     Expr: |
       sum(rate(s3_ingestion_source_operations_total{status="error", namespace="${namespace}", job="${job_ingestion_producer}"}[5m]))
         /
@@ -98,6 +98,22 @@ groups:
       description: "Ingestion producer source error rate has increased to 5% in the last 5 minutes"
       summary: "Ingestion producer source error rate has increased"
 
+  # The setup of a reader is retried on every cron tick, so a source that cannot
+  # be resolved fails continuously. It is a handful of errors against the reads
+  # of every healthy source, so it only shows up on the absolute rate: the ratio
+  # alerts above stay far below their threshold when a single source is stuck.
+  - alert: IngestionProducerSourceSetupFailing
+    Expr: |
+      sum(rate(s3_ingestion_source_operations_total{op="getRaftId", status="error", namespace="${namespace}", job="${job_ingestion_producer}"}[5m]))
+      > 0
+    For:   "10m"
+    Labels:
+      severity: warning
+    Annotations:
+      zenko_service: backbeat-ingestion-producer
+      description: "Ingestion producer has been unable to start reading from a source for the last 10 minutes, so the objects of that location are not ingested. Check the producer logs for the bucket it fails to resolve."
+      summary: "Ingestion producer cannot start reading from a source"
+
   - alert: IngestionProducerZookeeperErrorRate3Percent
     Expr: |
       sum(rate(s3_ingestion_zookeeper_operations_total{status="error", namespace="${namespace}", job="${job_ingestion_producer}"}[5m]))