diff --git a/.github/workflows/alerts.yaml b/.github/workflows/alerts.yaml index 4ebbb7445c..3e3ba043bc 100644 --- a/.github/workflows/alerts.yaml +++ b/.github/workflows/alerts.yaml @@ -74,6 +74,17 @@ jobs: notification_processor_replicas=2 github_token: ${{ secrets.GIT_ACCESS_TOKEN }} + - name: Render and test ingestion producer + uses: scality/action-prom-render-test@1.0.3 + with: + alert_file_path: monitoring/ingestion/ingestion-producer-alert.yaml + test_file_path: monitoring/ingestion/ingestion-producer-alert.test.yaml + alert_inputs: | + namespace=zenko + job_ingestion_producer=artesca-data-backbeat-ingestion-producer-headless + replicas=1 + github_token: ${{ secrets.GIT_ACCESS_TOKEN }} + - name: Render and test oplog populator uses: scality/action-prom-render-test@1.0.3 with: diff --git a/monitoring/ingestion/ingestion-producer-alert.test.yaml b/monitoring/ingestion/ingestion-producer-alert.test.yaml new file mode 100644 index 0000000000..eab23ba3d6 --- /dev/null +++ b/monitoring/ingestion/ingestion-producer-alert.test.yaml @@ -0,0 +1,53 @@ +evaluation_interval: 1m +rule_files: + - ingestion-producer-alert.rendered.yaml + +tests: + + # A source whose setup keeps failing is retried on every cron tick, so it + # fails at a steady rate while every other source keeps reading normally. + - name: Ingestion Producer Source Setup Failing + interval: 1m + input_series: + - series: s3_ingestion_source_operations_total{namespace="zenko",job="artesca-data-backbeat-ingestion-producer-headless",origin="ingestion",op="getRaftId",status="error"} + values: 0+12x20 + - series: s3_ingestion_source_operations_total{namespace="zenko",job="artesca-data-backbeat-ingestion-producer-headless",origin="ingestion",op="getRaftLog",status="success"} + values: 0+12000x20 + alert_rule_test: + - alertname: IngestionProducerSourceSetupFailing + eval_time: 5m + exp_alerts: [] + - alertname: IngestionProducerSourceSetupFailing + eval_time: 12m + exp_alerts: + - exp_labels: + severity: warning + exp_annotations: + zenko_service: backbeat-ingestion-producer + description: "Ingestion producer has been unable to start reading from a source for the last 10 minutes, so the objects of that location are not ingested. Check the producer logs for the bucket it fails to resolve." + summary: "Ingestion producer cannot start reading from a source" + # The same failures are 0.1% of all the source operations, which is why the + # error ratio alerts cannot report a single source that stopped ingesting. + - alertname: IngestionProducerSourceErrorRate3Percent + eval_time: 12m + exp_alerts: [] + - alertname: IngestionProducerSourceErrorRate5Percent + eval_time: 12m + exp_alerts: [] + + # A source that fails to set up while its location restarts, then recovers, + # must not alert. + - name: Ingestion Producer Source Setup Recovering + interval: 1m + input_series: + - series: s3_ingestion_source_operations_total{namespace="zenko",job="artesca-data-backbeat-ingestion-producer-headless",origin="ingestion",op="getRaftId",status="error"} + values: 0 12 24 24x17 + - series: s3_ingestion_source_operations_total{namespace="zenko",job="artesca-data-backbeat-ingestion-producer-headless",origin="ingestion",op="getRaftId",status="success"} + values: 0 0 0 1+1x17 + alert_rule_test: + - alertname: IngestionProducerSourceSetupFailing + eval_time: 8m + exp_alerts: [] + - alertname: IngestionProducerSourceSetupFailing + eval_time: 15m + exp_alerts: [] diff --git a/monitoring/ingestion/ingestion-producer-alert.yaml b/monitoring/ingestion/ingestion-producer-alert.yaml index 73ec1073b4..602658f329 100644 --- a/monitoring/ingestion/ingestion-producer-alert.yaml +++ b/monitoring/ingestion/ingestion-producer-alert.yaml @@ -84,7 +84,7 @@ groups: description: "Ingestion producer source error rate has increased to 3% in the last 5 minutes" summary: "Ingestion producer source error rate has increased" - - alert: IngestionProducerSourceErrorRate3Percent + - alert: IngestionProducerSourceErrorRate5Percent Expr: | sum(rate(s3_ingestion_source_operations_total{status="error", namespace="${namespace}", job="${job_ingestion_producer}"}[5m])) / @@ -98,6 +98,22 @@ groups: description: "Ingestion producer source error rate has increased to 5% in the last 5 minutes" summary: "Ingestion producer source error rate has increased" + # The setup of a reader is retried on every cron tick, so a source that cannot + # be resolved fails continuously. It is a handful of errors against the reads + # of every healthy source, so it only shows up on the absolute rate: the ratio + # alerts above stay far below their threshold when a single source is stuck. + - alert: IngestionProducerSourceSetupFailing + Expr: | + sum(rate(s3_ingestion_source_operations_total{op="getRaftId", status="error", namespace="${namespace}", job="${job_ingestion_producer}"}[5m])) + > 0 + For: "10m" + Labels: + severity: warning + Annotations: + zenko_service: backbeat-ingestion-producer + description: "Ingestion producer has been unable to start reading from a source for the last 10 minutes, so the objects of that location are not ingested. Check the producer logs for the bucket it fails to resolve." + summary: "Ingestion producer cannot start reading from a source" + - alert: IngestionProducerZookeeperErrorRate3Percent Expr: | sum(rate(s3_ingestion_zookeeper_operations_total{status="error", namespace="${namespace}", job="${job_ingestion_producer}"}[5m]))