From 17cf824f1d23c44f6b4a36a1ed11af405fdfa0bc Mon Sep 17 00:00:00 2001 From: Piero Rospigliosi Date: Sat, 8 Aug 2026 20:50:40 +0200 Subject: [PATCH 1/3] feat: add operator observability metrics and Grafana dashboard --- README.md | 1 + charts/github-platform-operator/README.md | 6 + config/manager/kustomization.yaml | 6 + dashboards/grafana/README.md | 40 + .../grafana/github-platform-operator.json | 1640 +++++++++++++++++ docs/operations.md | 40 + go.mod | 3 +- .../github/auth_rate_limit_contract_test.go | 4 +- internal/github/metrics.go | 223 +++ internal/github/metrics_contract_test.go | 158 ++ internal/github/rate_limit.go | 32 +- test/e2e/e2e_test.go | 3 + 12 files changed, 2146 insertions(+), 10 deletions(-) create mode 100644 dashboards/grafana/README.md create mode 100644 dashboards/grafana/github-platform-operator.json create mode 100644 internal/github/metrics.go create mode 100644 internal/github/metrics_contract_test.go diff --git a/README.md b/README.md index 20d0d51..002a524 100644 --- a/README.md +++ b/README.md @@ -176,6 +176,7 @@ policies or upgrading CRD schemas. - [Getting started](docs/getting-started.md) - [Custom resources and examples](docs/resources.md) - [Operations, upgrades and troubleshooting](docs/operations.md) +- [Grafana dashboard](dashboards/grafana/README.md) - [Helm chart](charts/github-platform-operator/README.md) - [Development guide](docs/development.md) - [Release process](docs/releasing.md) diff --git a/charts/github-platform-operator/README.md b/charts/github-platform-operator/README.md index 44bd7a2..677e668 100644 --- a/charts/github-platform-operator/README.md +++ b/charts/github-platform-operator/README.md @@ -170,6 +170,12 @@ helm show values \ oci://ghcr.io/pierinho13/charts/github-platform-operator ``` +The operator exposes standard `controller-runtime` metrics plus GitHub API and +rate-limit metrics. An importable Grafana dashboard is available in the source +repository at `dashboards/grafana/github-platform-operator.json`. See the +[operations guide](https://github.com/pierinho13/github-platform-operator/blob/main/docs/operations.md#metrics-and-grafana) +for scraping and dashboard details. + ## CRDs and upgrades The chart includes generated CRDs under `crds/`. Helm installs them before the diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 5c5f0b8..b0b8c68 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -1,2 +1,8 @@ resources: - manager.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +images: +- name: controller + newName: example.com/github-platform-operator + newTag: v0.0.1 diff --git a/dashboards/grafana/README.md b/dashboards/grafana/README.md new file mode 100644 index 0000000..64b4cf8 --- /dev/null +++ b/dashboards/grafana/README.md @@ -0,0 +1,40 @@ +# Grafana dashboard + +`github-platform-operator.json` is an importable Grafana dashboard for the +operator's Prometheus metrics. + +It combines the custom GitHub API metrics exported by the operator with the +standard metrics already exposed by `controller-runtime`, the Go runtime and +the process collector. + +## Prerequisite + +Prometheus must scrape the operator `/metrics` endpoint. The project already +ships the metrics Service and a ServiceMonitor example under +[`config/prometheus`](../../config/prometheus). + +When secure metrics are enabled, the Prometheus service account must be allowed +to read the authenticated `/metrics` endpoint. See +[`docs/operations.md`](../../docs/operations.md#metrics-and-grafana) for the +operational notes. + +## Import + +In Grafana, use **Dashboards → New → Import**, upload +`github-platform-operator.json`, and select the Prometheus data source. + +The dashboard includes: + +- GitHub API request volume and HTTP status codes +- GitHub API p95 latency +- transport errors +- rate-limit remaining/limit/reset state +- primary and secondary rate-limit events +- shared rate-limit gate blocking time +- controller reconciliation rate, errors and p95 duration +- workqueue depth +- process CPU, resident memory and Go goroutines + +The dashboard intentionally avoids repository, organization, username and token +labels to keep metric cardinality bounded and to avoid exposing sensitive +identifiers through Prometheus labels. diff --git a/dashboards/grafana/github-platform-operator.json b/dashboards/grafana/github-platform-operator.json new file mode 100644 index 0000000..b6ab9d5 --- /dev/null +++ b/dashboards/grafana/github-platform-operator.json @@ -0,0 +1,1640 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "10.0.0" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Operational dashboard for github-platform-operator: GitHub API usage, rate limits, reconciliation health, workqueues and runtime resources.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "id": 1, + "type": "stat", + "title": "GitHub API requests", + "gridPos": { + "x": 0, + "y": 0, + "w": 4, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(increase(github_platform_operator_github_api_requests_total{job=~\"$job\"}[$__range]))", + "legendFormat": "", + "range": false, + "refId": "A", + "instant": true + } + ] + }, + { + "id": 2, + "type": "stat", + "title": "GitHub API 4xx / 5xx", + "gridPos": { + "x": 4, + "y": 0, + "w": 4, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(increase(github_platform_operator_github_api_requests_total{job=~\"$job\",status_code=~\"4..|5..\"}[$__range]))", + "legendFormat": "", + "range": false, + "refId": "A", + "instant": true + } + ] + }, + { + "id": 3, + "type": "stat", + "title": "Rate limit remaining", + "gridPos": { + "x": 8, + "y": 0, + "w": 4, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "max(github_platform_operator_github_rate_limit_remaining{job=~\"$job\",resource=\"$rate_resource\"})", + "legendFormat": "", + "range": false, + "refId": "A", + "instant": true + } + ] + }, + { + "id": 4, + "type": "stat", + "title": "Rate limit used", + "gridPos": { + "x": 12, + "y": 0, + "w": 4, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 70 + }, + { + "color": "red", + "value": 90 + } + ] + }, + "decimals": 1 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "100 * (1 - max(github_platform_operator_github_rate_limit_remaining{job=~\"$job\",resource=\"$rate_resource\"}) / clamp_min(max(github_platform_operator_github_rate_limit_limit{job=~\"$job\",resource=\"$rate_resource\"}), 1))", + "legendFormat": "", + "range": false, + "refId": "A", + "instant": true + } + ] + }, + { + "id": 5, + "type": "stat", + "title": "Rate-limit events", + "gridPos": { + "x": 16, + "y": 0, + "w": 4, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(increase(github_platform_operator_github_rate_limit_events_total{job=~\"$job\"}[$__range]))", + "legendFormat": "", + "range": false, + "refId": "A", + "instant": true + } + ] + }, + { + "id": 6, + "type": "stat", + "title": "Gate blocked for", + "gridPos": { + "x": 20, + "y": 0, + "w": 4, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "clamp_min(max(github_platform_operator_github_rate_limit_blocked_until_timestamp_seconds{job=~\"$job\"}) - time(), 0)", + "legendFormat": "", + "range": false, + "refId": "A", + "instant": true + } + ] + }, + { + "id": 7, + "type": "timeseries", + "title": "GitHub API request rate", + "description": "", + "gridPos": { + "x": 0, + "y": 4, + "w": 12, + "h": 7 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 10, + "gradientMode": "none", + "showPoints": "auto", + "spanNulls": false, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (status_code) (rate(github_platform_operator_github_api_requests_total{job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "HTTP {{status_code}}", + "range": true, + "refId": "A", + "instant": false + } + ] + }, + { + "id": 8, + "type": "timeseries", + "title": "GitHub API p95 latency by method", + "description": "", + "gridPos": { + "x": 12, + "y": 4, + "w": 12, + "h": 7 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 10, + "gradientMode": "none", + "showPoints": "auto", + "spanNulls": false, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by (le, method) (rate(github_platform_operator_github_api_request_duration_seconds_bucket{job=~\"$job\"}[$__rate_interval])))", + "legendFormat": "{{method}}", + "range": true, + "refId": "A", + "instant": false + } + ] + }, + { + "id": 9, + "type": "timeseries", + "title": "GitHub rate limit", + "description": "Last rate-limit values observed from GitHub response headers.", + "gridPos": { + "x": 0, + "y": 11, + "w": 12, + "h": 7 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 10, + "gradientMode": "none", + "showPoints": "auto", + "spanNulls": false, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "max by (resource) (github_platform_operator_github_rate_limit_remaining{job=~\"$job\",resource=\"$rate_resource\"})", + "legendFormat": "remaining {{resource}}", + "range": true, + "refId": "A", + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "max by (resource) (github_platform_operator_github_rate_limit_limit{job=~\"$job\",resource=\"$rate_resource\"})", + "legendFormat": "limit {{resource}}", + "range": true, + "refId": "B", + "instant": false + } + ] + }, + { + "id": 10, + "type": "timeseries", + "title": "Rate-limit events by kind", + "description": "", + "gridPos": { + "x": 12, + "y": 11, + "w": 12, + "h": 7 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 10, + "gradientMode": "none", + "showPoints": "auto", + "spanNulls": false, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (kind) (rate(github_platform_operator_github_rate_limit_events_total{job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "{{kind}}", + "range": true, + "refId": "A", + "instant": false + } + ] + }, + { + "id": 11, + "type": "timeseries", + "title": "Reconciliation rate by controller", + "description": "", + "gridPos": { + "x": 0, + "y": 18, + "w": 12, + "h": 7 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 10, + "gradientMode": "none", + "showPoints": "auto", + "spanNulls": false, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (controller) (rate(controller_runtime_reconcile_total{job=~\"$job\",controller=~\"$controller\"}[$__rate_interval]))", + "legendFormat": "{{controller}}", + "range": true, + "refId": "A", + "instant": false + } + ] + }, + { + "id": 12, + "type": "timeseries", + "title": "Reconciliation errors by controller", + "description": "", + "gridPos": { + "x": 12, + "y": 18, + "w": 12, + "h": 7 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 10, + "gradientMode": "none", + "showPoints": "auto", + "spanNulls": false, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (controller) (rate(controller_runtime_reconcile_total{job=~\"$job\",controller=~\"$controller\",result=\"error\"}[$__rate_interval]))", + "legendFormat": "{{controller}}", + "range": true, + "refId": "A", + "instant": false + } + ] + }, + { + "id": 13, + "type": "timeseries", + "title": "Reconciliation p95 duration", + "description": "", + "gridPos": { + "x": 0, + "y": 25, + "w": 12, + "h": 7 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 10, + "gradientMode": "none", + "showPoints": "auto", + "spanNulls": false, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by (le, controller) (rate(controller_runtime_reconcile_time_seconds_bucket{job=~\"$job\",controller=~\"$controller\"}[$__rate_interval])))", + "legendFormat": "{{controller}}", + "range": true, + "refId": "A", + "instant": false + } + ] + }, + { + "id": 14, + "type": "timeseries", + "title": "Workqueue depth", + "description": "", + "gridPos": { + "x": 12, + "y": 25, + "w": 12, + "h": 7 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 10, + "gradientMode": "none", + "showPoints": "auto", + "spanNulls": false, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (name) (workqueue_depth{job=~\"$job\"})", + "legendFormat": "{{name}}", + "range": true, + "refId": "A", + "instant": false + } + ] + }, + { + "id": 15, + "type": "timeseries", + "title": "GitHub transport errors", + "description": "", + "gridPos": { + "x": 0, + "y": 32, + "w": 8, + "h": 6 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 10, + "gradientMode": "none", + "showPoints": "auto", + "spanNulls": false, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (method) (rate(github_platform_operator_github_api_transport_errors_total{job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "{{method}}", + "range": true, + "refId": "A", + "instant": false + } + ] + }, + { + "id": 16, + "type": "timeseries", + "title": "Process CPU", + "description": "", + "gridPos": { + "x": 8, + "y": 32, + "w": 8, + "h": 6 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "cores", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 10, + "gradientMode": "none", + "showPoints": "auto", + "spanNulls": false, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate(process_cpu_seconds_total{job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "CPU", + "range": true, + "refId": "A", + "instant": false + } + ] + }, + { + "id": 17, + "type": "timeseries", + "title": "Resident memory", + "description": "", + "gridPos": { + "x": 16, + "y": 32, + "w": 8, + "h": 6 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 10, + "gradientMode": "none", + "showPoints": "auto", + "spanNulls": false, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(process_resident_memory_bytes{job=~\"$job\"})", + "legendFormat": "RSS", + "range": true, + "refId": "A", + "instant": false + } + ] + }, + { + "id": 18, + "type": "stat", + "title": "Go goroutines", + "gridPos": { + "x": 0, + "y": 38, + "w": 6, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(go_goroutines{job=~\"$job\"})", + "legendFormat": "", + "range": false, + "refId": "A", + "instant": true + } + ] + }, + { + "id": 19, + "type": "stat", + "title": "Reconcile errors", + "gridPos": { + "x": 6, + "y": 38, + "w": 6, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(increase(controller_runtime_reconcile_total{job=~\"$job\",controller=~\"$controller\",result=\"error\"}[$__range]))", + "legendFormat": "", + "range": false, + "refId": "A", + "instant": true + } + ] + }, + { + "id": 20, + "type": "stat", + "title": "Transport errors", + "gridPos": { + "x": 12, + "y": 38, + "w": 6, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(increase(github_platform_operator_github_api_transport_errors_total{job=~\"$job\"}[$__range]))", + "legendFormat": "", + "range": false, + "refId": "A", + "instant": true + } + ] + }, + { + "id": 21, + "type": "stat", + "title": "Rate reset in", + "gridPos": { + "x": 18, + "y": 38, + "w": 6, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "clamp_min(max(github_platform_operator_github_rate_limit_reset_timestamp_seconds{job=~\"$job\",resource=\"$rate_resource\"}) - time(), 0)", + "legendFormat": "", + "range": false, + "refId": "A", + "instant": true + } + ] + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": [ + "github-platform-operator", + "kubernetes", + "operator", + "github" + ], + "templating": { + "list": [ + { + "name": "job", + "label": "Prometheus job", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "query": { + "query": "label_values(github_platform_operator_github_rate_limit_blocked_until_timestamp_seconds, job)", + "refId": "StandardVariableQuery" + }, + "definition": "label_values(github_platform_operator_github_rate_limit_blocked_until_timestamp_seconds, job)", + "includeAll": true, + "allValue": ".*", + "multi": true, + "refresh": 1, + "sort": 1, + "current": { + "selected": false, + "text": "All", + "value": "$__all" + } + }, + { + "name": "controller", + "label": "Controller", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "query": { + "query": "label_values(controller_runtime_reconcile_total{job=~\"$job\"}, controller)", + "refId": "StandardVariableQuery" + }, + "definition": "label_values(controller_runtime_reconcile_total{job=~\"$job\"}, controller)", + "includeAll": true, + "allValue": ".*", + "multi": true, + "refresh": 1, + "sort": 1, + "current": { + "selected": false, + "text": "All", + "value": "$__all" + } + }, + { + "name": "rate_resource", + "label": "Rate-limit resource", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "query": { + "query": "label_values(github_platform_operator_github_rate_limit_remaining{job=~\"$job\"}, resource)", + "refId": "StandardVariableQuery" + }, + "definition": "label_values(github_platform_operator_github_rate_limit_remaining{job=~\"$job\"}, resource)", + "includeAll": false, + "multi": false, + "refresh": 1, + "sort": 1, + "current": { + "selected": true, + "text": "core", + "value": "core" + } + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "GitHub Platform Operator", + "uid": "github-platform-operator", + "version": 1, + "weekStart": "" +} diff --git a/docs/operations.md b/docs/operations.md index d467311..5691db9 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -115,6 +115,46 @@ resources without producing a continuous error storm. A normal authorization or feature-plan `403` is not treated as a rate limit. Inspect the response message in the resource condition or controller logs. +## Metrics and Grafana + +The manager exposes the standard `controller-runtime`, Go runtime and process +metrics together with a small set of GitHub-specific Prometheus metrics: + +| Metric | Purpose | +|---|---| +| `github_platform_operator_github_api_requests_total` | GitHub API responses by HTTP method and status code | +| `github_platform_operator_github_api_request_duration_seconds` | GitHub API HTTP round-trip latency histogram by method | +| `github_platform_operator_github_api_transport_errors_total` | Requests that failed before an HTTP response was received | +| `github_platform_operator_github_rate_limit_limit` | Last observed GitHub rate-limit ceiling by resource | +| `github_platform_operator_github_rate_limit_remaining` | Last observed remaining requests by rate-limit resource | +| `github_platform_operator_github_rate_limit_reset_timestamp_seconds` | Last observed rate-limit reset timestamp | +| `github_platform_operator_github_rate_limit_events_total` | Detected primary, secondary or unknown rate-limit events | +| `github_platform_operator_github_rate_limit_blocked_until_timestamp_seconds` | Timestamp until which the shared reactive rate-limit gate is blocked | + +The metrics are instrumented in the shared GitHub HTTP transport, so the same +measurements cover repository, organization, team, ruleset, Actions and GitHub +App token requests without duplicating instrumentation in every controller. +Rate-limit gauges represent the last response observed by the shared HTTP client; +they are intentionally not labeled by provider, organization, repository or +credential in order to avoid sensitive labels and unbounded cardinality. + +Prometheus must scrape the manager metrics Service. The Kustomize deployment +contains a ServiceMonitor example under `config/prometheus`. With secure metrics, +the scraping service account must also be authorized for the non-resource +`/metrics` URL. + +An importable Grafana dashboard is included at: + +```text +dashboards/grafana/github-platform-operator.json +``` + +The dashboard combines these GitHub-specific metrics with existing metrics such +as `controller_runtime_reconcile_total`, +`controller_runtime_reconcile_time_seconds`, `workqueue_depth`, +`process_cpu_seconds_total`, `process_resident_memory_bytes` and +`go_goroutines`. + ## Secret and variable rotation Update the referenced Kubernetes Secret: diff --git a/go.mod b/go.mod index f084115..30fc206 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.26.0 require ( github.com/onsi/ginkgo/v2 v2.32.0 github.com/onsi/gomega v1.42.1 + github.com/prometheus/client_golang v1.23.2 golang.org/x/crypto v0.53.0 k8s.io/api v0.36.3 k8s.io/apimachinery v0.36.3 @@ -42,12 +43,12 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/prometheus/client_golang v1.23.2 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.67.5 // indirect github.com/prometheus/procfs v0.19.2 // indirect diff --git a/internal/github/auth_rate_limit_contract_test.go b/internal/github/auth_rate_limit_contract_test.go index 7e11a71..486149d 100644 --- a/internal/github/auth_rate_limit_contract_test.go +++ b/internal/github/auth_rate_limit_contract_test.go @@ -400,8 +400,8 @@ func TestPrimaryRateLimitResetContract(t *testing.T) { response := &http.Response{ StatusCode: http.StatusForbidden, Header: http.Header{ - "X-Ratelimit-Remaining": []string{"0"}, - "X-Ratelimit-Reset": []string{strconv.FormatInt(resetAt, 10)}, + rateLimitHeaderRemaining: []string{"0"}, + rateLimitHeaderReset: []string{strconv.FormatInt(resetAt, 10)}, }, Body: io.NopCloser(strings.NewReader(`{"message":"API rate limit exceeded"}`)), } diff --git a/internal/github/metrics.go b/internal/github/metrics.go new file mode 100644 index 0000000..90d581f --- /dev/null +++ b/internal/github/metrics.go @@ -0,0 +1,223 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package github + +import ( + "net/http" + "strconv" + "strings" + "time" + + "github.com/prometheus/client_golang/prometheus" + controllermetrics "sigs.k8s.io/controller-runtime/pkg/metrics" +) + +const ( + metricsNamespace = "github_platform_operator" + metricsSubsystem = "github" + metricsMethodLabel = "method" + metricsResourceLabel = "resource" + rateLimitResourceCore = "core" +) + +type githubMetrics struct { + apiRequests *prometheus.CounterVec + apiRequestDuration *prometheus.HistogramVec + apiTransportErrors *prometheus.CounterVec + rateLimitLimit *prometheus.GaugeVec + rateLimitRemaining *prometheus.GaugeVec + rateLimitReset *prometheus.GaugeVec + rateLimitEvents *prometheus.CounterVec + rateLimitBlockedUntil prometheus.Gauge +} + +func newGitHubMetrics(registerer prometheus.Registerer) *githubMetrics { + m := &githubMetrics{ + apiRequests: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: metricsNamespace, + Subsystem: metricsSubsystem, + Name: "api_requests_total", + Help: "Total GitHub API responses observed by the shared HTTP client.", + }, + []string{metricsMethodLabel, "status_code"}, + ), + apiRequestDuration: prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: metricsNamespace, + Subsystem: metricsSubsystem, + Name: "api_request_duration_seconds", + Help: "GitHub API HTTP round-trip duration in seconds until response headers are received.", + Buckets: prometheus.DefBuckets, + }, + []string{metricsMethodLabel}, + ), + apiTransportErrors: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: metricsNamespace, + Subsystem: metricsSubsystem, + Name: "api_transport_errors_total", + Help: "Total GitHub API requests that failed before an HTTP response was received.", + }, + []string{metricsMethodLabel}, + ), + rateLimitLimit: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: metricsNamespace, + Subsystem: metricsSubsystem, + Name: "rate_limit_limit", + Help: "Last observed GitHub API rate-limit ceiling for a rate-limit resource.", + }, + []string{metricsResourceLabel}, + ), + rateLimitRemaining: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: metricsNamespace, + Subsystem: metricsSubsystem, + Name: "rate_limit_remaining", + Help: "Last observed GitHub API requests remaining for a rate-limit resource.", + }, + []string{metricsResourceLabel}, + ), + rateLimitReset: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: metricsNamespace, + Subsystem: metricsSubsystem, + Name: "rate_limit_reset_timestamp_seconds", + Help: "Last observed GitHub API rate-limit reset time as a Unix timestamp.", + }, + []string{metricsResourceLabel}, + ), + rateLimitEvents: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: metricsNamespace, + Subsystem: metricsSubsystem, + Name: "rate_limit_events_total", + Help: "Total GitHub API rate-limit responses detected by kind.", + }, + []string{"kind"}, + ), + rateLimitBlockedUntil: prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: metricsNamespace, + Subsystem: metricsSubsystem, + Name: "rate_limit_blocked_until_timestamp_seconds", + Help: "Unix timestamp until which the shared GitHub API gate is blocked; zero means no block has been observed.", + }, + ), + } + + registerer.MustRegister( + m.apiRequests, + m.apiRequestDuration, + m.apiTransportErrors, + m.rateLimitLimit, + m.rateLimitRemaining, + m.rateLimitReset, + m.rateLimitEvents, + m.rateLimitBlockedUntil, + ) + + return m +} + +var defaultGitHubMetrics = newGitHubMetrics(controllermetrics.Registry) + +func (m *githubMetrics) observeResponse(request *http.Request, response *http.Response, duration time.Duration) { + if m == nil || response == nil { + return + } + + method := normalizeHTTPMethod(request) + m.apiRequests.WithLabelValues(method, strconv.Itoa(response.StatusCode)).Inc() + m.apiRequestDuration.WithLabelValues(method).Observe(duration.Seconds()) + m.observeRateLimitHeaders(response.Header) +} + +func (m *githubMetrics) observeTransportError(request *http.Request, duration time.Duration) { + if m == nil { + return + } + + method := normalizeHTTPMethod(request) + m.apiTransportErrors.WithLabelValues(method).Inc() + m.apiRequestDuration.WithLabelValues(method).Observe(duration.Seconds()) +} + +func (m *githubMetrics) observeRateLimitHeaders(header http.Header) { + if m == nil || header == nil { + return + } + + resource := strings.TrimSpace(header.Get(rateLimitHeaderResource)) + if resource == "" { + resource = rateLimitResourceCore + } + + setGaugeFromHeader(m.rateLimitLimit.WithLabelValues(resource), header.Get(rateLimitHeaderLimit)) + setGaugeFromHeader(m.rateLimitRemaining.WithLabelValues(resource), header.Get(rateLimitHeaderRemaining)) + setGaugeFromHeader(m.rateLimitReset.WithLabelValues(resource), header.Get(rateLimitHeaderReset)) +} + +func (m *githubMetrics) observeRateLimit( + response *http.Response, + rateErr *RateLimitError, + blockedUntil time.Time, +) { + if m == nil || rateErr == nil { + return + } + + m.rateLimitEvents.WithLabelValues(rateLimitKind(response, rateErr)).Inc() + if !blockedUntil.IsZero() { + m.rateLimitBlockedUntil.Set(float64(blockedUntil.Unix())) + } +} + +func setGaugeFromHeader(gauge prometheus.Gauge, value string) { + parsed, err := strconv.ParseFloat(strings.TrimSpace(value), 64) + if err != nil { + return + } + gauge.Set(parsed) +} + +func normalizeHTTPMethod(request *http.Request) string { + if request == nil || strings.TrimSpace(request.Method) == "" { + return "UNKNOWN" + } + return strings.ToUpper(request.Method) +} + +func rateLimitKind(response *http.Response, rateErr *RateLimitError) string { + if response != nil && strings.TrimSpace(response.Header.Get(rateLimitHeaderRemaining)) == "0" { + return "primary" + } + + body := "" + if rateErr != nil { + body = strings.ToLower(rateErr.Body) + } + if response != nil && (response.StatusCode == http.StatusTooManyRequests || + strings.TrimSpace(response.Header.Get("Retry-After")) != "") { + return "secondary" + } + if strings.Contains(body, "secondary rate limit") || strings.Contains(body, "abuse detection") { + return "secondary" + } + return "unknown" +} diff --git a/internal/github/metrics_contract_test.go b/internal/github/metrics_contract_test.go new file mode 100644 index 0000000..4f4d9a7 --- /dev/null +++ b/internal/github/metrics_contract_test.go @@ -0,0 +1,158 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package github + +import ( + "errors" + "io" + "net/http" + "strconv" + "strings" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" +) + +func TestRateLimitTransportMetricsContract(t *testing.T) { + t.Parallel() + + registry := prometheus.NewRegistry() + metricSet := newGitHubMetrics(registry) + reset := time.Now().Add(10 * time.Minute).Unix() + transport := &RateLimitTransport{ + Base: roundTripFunc(func(request *http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{ + rateLimitHeaderLimit: []string{"5000"}, + rateLimitHeaderRemaining: []string{"4242"}, + rateLimitHeaderReset: []string{strconv.FormatInt(reset, 10)}, + rateLimitHeaderResource: []string{rateLimitResourceCore}, + }, + Body: io.NopCloser(strings.NewReader(`{"ok":true}`)), + Request: request, + }, nil + }), + gate: &rateLimitGate{}, + metrics: metricSet, + } + + request, err := http.NewRequest(http.MethodGet, "https://api.github.test/repos/example/repo", nil) + if err != nil { + t.Fatalf("create request: %v", err) + } + response, err := transport.RoundTrip(request) + if err != nil { + t.Fatalf("round trip: %v", err) + } + closeResponseBody(response.Body) + + if got := testutil.ToFloat64(metricSet.apiRequests.WithLabelValues(http.MethodGet, "200")); got != 1 { + t.Fatalf("api request counter = %v, want 1", got) + } + if got := testutil.ToFloat64(metricSet.rateLimitLimit.WithLabelValues(rateLimitResourceCore)); got != 5000 { + t.Fatalf("rate-limit limit = %v, want 5000", got) + } + if got := testutil.ToFloat64(metricSet.rateLimitRemaining.WithLabelValues(rateLimitResourceCore)); got != 4242 { + t.Fatalf("rate-limit remaining = %v, want 4242", got) + } + if got := testutil.ToFloat64(metricSet.rateLimitReset.WithLabelValues(rateLimitResourceCore)); got != float64(reset) { + t.Fatalf("rate-limit reset = %v, want %d", got, reset) + } + if count := testutil.CollectAndCount(metricSet.apiRequestDuration); count != 1 { + t.Fatalf("request duration metric families = %d, want 1", count) + } +} + +func TestRateLimitTransportRecordsTransportErrorsContract(t *testing.T) { + t.Parallel() + + registry := prometheus.NewRegistry() + metricSet := newGitHubMetrics(registry) + transport := &RateLimitTransport{ + Base: roundTripFunc(func(_ *http.Request) (*http.Response, error) { + return nil, errors.New("network unavailable") + }), + gate: &rateLimitGate{}, + metrics: metricSet, + } + + request, err := http.NewRequest(http.MethodPost, "https://api.github.test/app/installations/1/access_tokens", nil) + if err != nil { + t.Fatalf("create request: %v", err) + } + if _, err := transport.RoundTrip(request); err == nil { + t.Fatal("expected transport error") + } + + if got := testutil.ToFloat64(metricSet.apiTransportErrors.WithLabelValues(http.MethodPost)); got != 1 { + t.Fatalf("transport error counter = %v, want 1", got) + } +} + +func TestRateLimitTransportRecordsRateLimitEventsContract(t *testing.T) { + t.Parallel() + + registry := prometheus.NewRegistry() + metricSet := newGitHubMetrics(registry) + reset := time.Now().Add(2 * time.Minute).Unix() + transport := &RateLimitTransport{ + Base: roundTripFunc(func(request *http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: http.StatusForbidden, + Header: http.Header{ + rateLimitHeaderLimit: []string{"5000"}, + rateLimitHeaderRemaining: []string{"0"}, + rateLimitHeaderReset: []string{strconv.FormatInt(reset, 10)}, + rateLimitHeaderResource: []string{rateLimitResourceCore}, + }, + Body: io.NopCloser(strings.NewReader(`{"message":"API rate limit exceeded"}`)), + Request: request, + }, nil + }), + gate: &rateLimitGate{}, + metrics: metricSet, + } + + request, err := http.NewRequest(http.MethodGet, "https://api.github.test/orgs/example/teams", nil) + if err != nil { + t.Fatalf("create request: %v", err) + } + _, err = transport.RoundTrip(request) + var rateErr *RateLimitError + if !errors.As(err, &rateErr) { + t.Fatalf("expected RateLimitError, got %v", err) + } + + if got := testutil.ToFloat64(metricSet.rateLimitEvents.WithLabelValues("primary")); got != 1 { + t.Fatalf("primary rate-limit events = %v, want 1", got) + } + if got := testutil.ToFloat64(metricSet.rateLimitRemaining.WithLabelValues(rateLimitResourceCore)); got != 0 { + t.Fatalf("rate-limit remaining = %v, want 0", got) + } + if got := testutil.ToFloat64(metricSet.rateLimitBlockedUntil); got < float64(reset) { + t.Fatalf("blocked-until timestamp = %v, want at least %d", got, reset) + } +} + +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (f roundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) { + return f(request) +} diff --git a/internal/github/rate_limit.go b/internal/github/rate_limit.go index 16ae5cf..617ec20 100644 --- a/internal/github/rate_limit.go +++ b/internal/github/rate_limit.go @@ -28,7 +28,13 @@ import ( "time" ) -const fallbackSecondaryRateLimitDelay = time.Minute +const ( + fallbackSecondaryRateLimitDelay = time.Minute + rateLimitHeaderLimit = "X-Ratelimit-Limit" + rateLimitHeaderRemaining = "X-Ratelimit-Remaining" + rateLimitHeaderReset = "X-Ratelimit-Reset" + rateLimitHeaderResource = "X-Ratelimit-Resource" +) // RateLimitError indicates that GitHub asked the operator to stop making // requests temporarily. It can represent both primary and secondary limits. @@ -85,7 +91,7 @@ func (g *rateLimitGate) currentError(now time.Time) *RateLimitError { } } -func (g *rateLimitGate) block(until time.Time, statusCode int, body string) { +func (g *rateLimitGate) block(until time.Time, statusCode int, body string) time.Time { g.mu.Lock() defer g.mu.Unlock() @@ -94,6 +100,7 @@ func (g *rateLimitGate) block(until time.Time, statusCode int, body string) { g.statusCode = statusCode g.body = body } + return g.blockedUntil } // RateLimitTransport adds a shared, reactive GitHub rate-limit gate. @@ -107,6 +114,7 @@ type RateLimitTransport struct { gateOnce sync.Once gate *rateLimitGate + metrics *githubMetrics } // NewRateLimitedHTTPClient creates an HTTP client whose rate-limit state is @@ -115,8 +123,9 @@ func NewRateLimitedHTTPClient() *http.Client { return &http.Client{ Timeout: defaultRequestTimeout, Transport: &RateLimitTransport{ - Base: http.DefaultTransport, - gate: &rateLimitGate{}, + Base: http.DefaultTransport, + gate: &rateLimitGate{}, + metrics: defaultGitHubMetrics, }, } } @@ -142,10 +151,18 @@ func (t *RateLimitTransport) RoundTrip(request *http.Request) (*http.Response, e base = http.DefaultTransport } + requestStartedAt := time.Now() response, err := base.RoundTrip(request) + duration := time.Since(requestStartedAt) + metrics := t.metrics + if metrics == nil { + metrics = defaultGitHubMetrics + } if err != nil { + metrics.observeTransportError(request, duration) return nil, err } + metrics.observeResponse(request, response, duration) rateErr, err := rateLimitErrorFromResponse(response) if err != nil { @@ -156,7 +173,8 @@ func (t *RateLimitTransport) RoundTrip(request *http.Request) (*http.Response, e return response, nil } - gate.block(rateErr.RetryAt, rateErr.StatusCode, rateErr.Body) + blockedUntil := gate.block(rateErr.RetryAt, rateErr.StatusCode, rateErr.Body) + metrics.observeRateLimit(response, rateErr, blockedUntil) closeResponseBody(response.Body) return nil, rateErr } @@ -178,7 +196,7 @@ func rateLimitErrorFromResponse(response *http.Response) (*RateLimitError, error bodyText := strings.TrimSpace(string(body)) lowerBody := strings.ToLower(bodyText) - remaining := strings.TrimSpace(response.Header.Get("X-RateLimit-Remaining")) + remaining := strings.TrimSpace(response.Header.Get(rateLimitHeaderRemaining)) retryAfter := strings.TrimSpace(response.Header.Get("Retry-After")) isRateLimited := response.StatusCode == http.StatusTooManyRequests || @@ -202,7 +220,7 @@ func rateLimitErrorFromResponse(response *http.Response) (*RateLimitError, error } } else if remaining == "0" { if reset, parseErr := strconv.ParseInt( - strings.TrimSpace(response.Header.Get("X-RateLimit-Reset")), + strings.TrimSpace(response.Header.Get(rateLimitHeaderReset)), 10, 64, ); parseErr == nil && reset > 0 { diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 0aa4a1c..edee5da 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -254,6 +254,9 @@ var _ = Describe("Manager", Ordered, func() { Expect(metricsOutput).To(ContainSubstring( "controller_runtime_reconcile_total", )) + Expect(metricsOutput).To(ContainSubstring( + "github_platform_operator_github_rate_limit_blocked_until_timestamp_seconds", + )) }) // +kubebuilder:scaffold:e2e-webhooks-checks From 8f0b816e9b0a67d8602a19e046017ea8e8cea3df Mon Sep 17 00:00:00 2001 From: Piero Rospigliosi Date: Sat, 8 Aug 2026 21:12:33 +0200 Subject: [PATCH 2/3] chore: prepare Helm chart for v0.6.0 --- charts/github-platform-operator/Chart.yaml | 26 ++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/charts/github-platform-operator/Chart.yaml b/charts/github-platform-operator/Chart.yaml index abdb288..6d89292 100644 --- a/charts/github-platform-operator/Chart.yaml +++ b/charts/github-platform-operator/Chart.yaml @@ -2,29 +2,47 @@ apiVersion: v2 name: github-platform-operator description: Kubernetes operator for managing GitHub repositories, access and Actions configuration type: application -version: 0.1.0 -appVersion: "v0.1.0" + +version: 0.6.0 +appVersion: "v0.6.0" + home: https://github.com/pierinho13/github-platform-operator + sources: - https://github.com/pierinho13/github-platform-operator + maintainers: - name: pierinho13 email: pierinho13@gmail.com url: https://github.com/pierinho13 + keywords: - github - github-actions - kubernetes - operator - platform-engineering + annotations: artifacthub.io/category: integration-delivery artifacthub.io/license: Apache-2.0 artifacthub.io/operator: "true" artifacthub.io/operatorCapabilities: Full Lifecycle + + artifacthub.io/changes: | + - kind: added + description: Added Prometheus metrics for GitHub API requests, latency, transport errors and rate-limit state. + - kind: added + description: Added GitHub API rate-limit observability including remaining requests, reset time and rate-limit events. + - kind: added + description: Added a ready-to-import Grafana dashboard for operator and GitHub API observability. + - kind: changed + description: Extended metrics endpoint tests to cover operator-specific Prometheus metrics. + artifacthub.io/signKey: | fingerprint: DBBC4A49D2BB68565A9BCC383CA8825330932B17 url: https://raw.githubusercontent.com/pierinho13/github-platform-operator/main/helm-signing-key.asc + artifacthub.io/links: | - name: Source url: https://github.com/pierinho13/github-platform-operator @@ -32,5 +50,5 @@ annotations: url: https://github.com/pierinho13/github-platform-operator#documentation - name: Getting Started url: https://github.com/pierinho13/github-platform-operator/blob/main/docs/getting-started.md - - name: support - url: https://github.com/pierinho13/github-platform-operator/issues + - name: Support + url: https://github.com/pierinho13/github-platform-operator/issues \ No newline at end of file From 32e29ef5ecfa25d4a2fb793dfce7ceecaf893229 Mon Sep 17 00:00:00 2001 From: Piero Rospigliosi Date: Sat, 8 Aug 2026 21:49:27 +0200 Subject: [PATCH 3/3] adding chart auto chagelog --- .github/workflows/ci.yaml | 4 +- .github/workflows/release.yaml | 4 +- .pre-commit-config.yaml | 2 +- charts/github-platform-operator/Chart.yaml | 16 +- .../prepare-artifacthub-chart.cpython-314.pyc | Bin 0 -> 9589 bytes hack/package-release-helm-chart.sh | 75 +++++++ hack/prepare-artifacthub-chart.py | 208 ++++++++++++++++++ 7 files changed, 293 insertions(+), 16 deletions(-) create mode 100644 hack/__pycache__/prepare-artifacthub-chart.cpython-314.pyc create mode 100755 hack/package-release-helm-chart.sh create mode 100755 hack/prepare-artifacthub-chart.py diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7b3aa29..20ecfe0 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -149,7 +149,7 @@ jobs: run: | set -euo pipefail - ./hack/package-helm-chart.sh \ + ./hack/package-release-helm-chart.sh \ 0.0.0-snapshot \ v0.0.0-snapshot @@ -212,7 +212,7 @@ jobs: } ' "${rendered}"; then echo "The packaged Helm chart does not contain the complete GitHubRepositoryRuleset RBAC." >&2 - echo "Check config/rbac/role.yaml and hack/package-helm-chart.sh." >&2 + echo "Check config/rbac/role.yaml and hack/package-release-helm-chart.sh." >&2 exit 1 fi diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 6514afe..9c02fc4 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -237,7 +237,7 @@ jobs: run: | set -euo pipefail - ./hack/package-helm-chart.sh \ + ./hack/package-release-helm-chart.sh \ "${CHART_VERSION}" \ "${APP_VERSION}" @@ -316,7 +316,7 @@ jobs: ' "${rendered}"; then echo "The packaged Helm chart does not contain the complete GitHubRepositoryRuleset RBAC." >&2 echo "Publishing has been stopped." >&2 - echo "Check config/rbac/role.yaml and hack/package-helm-chart.sh." >&2 + echo "Check config/rbac/role.yaml and hack/package-release-helm-chart.sh." >&2 exit 1 fi diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index eb9ce4d..0026680 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -106,7 +106,7 @@ repos: rm -rf dist/helm - ./hack/package-helm-chart.sh \ + ./hack/package-release-helm-chart.sh \ "${chart_version}" \ "${app_version}" diff --git a/charts/github-platform-operator/Chart.yaml b/charts/github-platform-operator/Chart.yaml index 6d89292..f52e6c9 100644 --- a/charts/github-platform-operator/Chart.yaml +++ b/charts/github-platform-operator/Chart.yaml @@ -3,8 +3,9 @@ name: github-platform-operator description: Kubernetes operator for managing GitHub repositories, access and Actions configuration type: application -version: 0.6.0 -appVersion: "v0.6.0" +# Release versions are injected into the packaged chart by the release pipeline. +version: 0.0.0 +appVersion: "v0.0.0" home: https://github.com/pierinho13/github-platform-operator @@ -29,15 +30,8 @@ annotations: artifacthub.io/operator: "true" artifacthub.io/operatorCapabilities: Full Lifecycle - artifacthub.io/changes: | - - kind: added - description: Added Prometheus metrics for GitHub API requests, latency, transport errors and rate-limit state. - - kind: added - description: Added GitHub API rate-limit observability including remaining requests, reset time and rate-limit events. - - kind: added - description: Added a ready-to-import Grafana dashboard for operator and GitHub API observability. - - kind: changed - description: Extended metrics endpoint tests to cover operator-specific Prometheus metrics. + # artifacthub.io/changes is generated only in the packaged release chart from + # Conventional Commits since the previous Git tag. artifacthub.io/signKey: | fingerprint: DBBC4A49D2BB68565A9BCC383CA8825330932B17 diff --git a/hack/__pycache__/prepare-artifacthub-chart.cpython-314.pyc b/hack/__pycache__/prepare-artifacthub-chart.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..8365b672d716f585428935c4800eeea0cd903514 GIT binary patch literal 9589 zcmb7KYj9Inem_^blCE^GWZ9B!`~YNQ46-d793I9ousjSHV@F&&VgcKVu&!-^EGhS1 z0ai&@yR$nnv>7{1GjVn^joWRfPSa`Jc0Z8m&Lp2YOg~A;LE=Iu?rb`p^phEuVVY0< zpL2C(VFb&bvCld8ydVF^@Bcq)wOK6)!ax7w+U5V)gwXHshZaZ;Le%4S3ZW$8k&MO= zPxVo!bh2(M&g=U0vYw>0Oq0|g8%S!DjU+Y6CP?*t%qg>M-ik&IUZ*zvdfI?^dJoKj zgUs><-pHGHhBxyTo`rvo=Q<2B_rku`LSHLyIP@@f>ehjp@zuYfNH{*udh zC*&(h-p*G+zMACA`Av|oA^8fv7V>o@@8IhpznSEnqm}%Yt{)>5v>{ZFMyndosIvj> zQJ0OhHB3>wZX0rMP4s(3Ar=q?mnejVfFyXLk?@S`ln|bFO$%}$7?1<5Yaw|m8kb$u z(O_s|CKQ=;$y0($ipIrp!F6mZAj<7Cf$4C&n^M?7BodVaawr;+2H~6b2IQ&4>9L)M zy*=_wOgMD$i04AUlNfDv?>yW+vh(7Yd$e`L?LLefToQ%A6&U_dgB$9kR$LYoq;WA6 z!vh{_Z*?~*^n?(Q6+<8x6oQI=B6M9bKm>U@CWsS?X?!XWnS@V96ea@Wa#U0pNf?id zA$cY-G`4fsjf=ZI2S!JNE$)%_7k|OxJQsI)4ydbb_rQ8#iQOZu8w;BhYakqsUh_`` zLgBb5oEXOxlBW{HKwgh&b0=hy$cM&sT|bArfwFpZmV#S=d*Jmvou?+cgN9pSzSlu> z#SAwS^^4J{EJAyv&;fB$QuLB6iX8dA6pe-zqbSI6G4dIL(>6{-#p!@7;prQdhN-~# zm5!KtzaF@y5I&Hp_$3dV5!_j9M&bPa{DldAzgP|3w!*)puJ$RiQVoxAG%v5=WbqhD zZqp>W;~;g9p#vhD#u+@b8U9Up)0f~YiAHca30}Od4FQ-6wNpOb82me&=vrZ+T& z&zINo`u#?=PfYLAd-M6?QN~c$H8}o$eEdFYN4`qE4jztv1h1V0=${uXS_w|Q*5WTU zud&wah4Ch<@c3I1zEp47^F4ezjSV@(n>zK{*AxY_+MjR#vRN^?DEi5ed}7#bQ1s*1 zf{I~$N*KSQ(6Vq{R*aGy1l}d7Ac_h~(9qbWP_Bf@g=tm@y3LA3ieHL}(Q!cn0u$pA z;M+cLuVPVYYdjhh6poKa4{ z{;;fO?k6{Yl23kbHQ9{G6Ki(F*s`WgY15{KBP*uO-?E$TJ1QUAY8Uo>dhp|e*}C?0 zU3;c(SGJB%)$z;0+pm4q^<~%B$N%k6s{8bv9jQHh#x|5tqN2q!m@mA;gd=?*ZS zVkIOhu`fq8U3-LtIW16@(7Z*ecYV}()0^(7%;^mydJq?E9DJb_Cf*smVk zrBGwTpHYN3V88h4-Fo$S+90nmAgQi~qH#&Wr`6n?rxhnqP&9m1@N3oN_&T8pIT$Zw zlIZsgvRCJr@*Go?f)@yoW{67mE#*JfO}>Sgaq)`2_5`U84gMEML8Y|3m1X0+FTHgj{ZUN zI0XYpZgL7wQ46vfQjp!= zf<#_#?K64V^)D@_jiE`?7~^9!GMF;&(v&x9y-LYzZRD7Fq#&KWn0Ge> zpII9vX7O1j)HyG|a88rf`enO`u`gILqxIJIsNNNCj#+q%kA=I^#n^M7zov_E=l=Zw zFn<0F{sq=!^kQkVA^13L#I49@@M18qAr!ahzlRn57HA2K&;FomZ&&o;=wyQRcp_2H zv>;9j68=yT$-(W?2@y^oKnzBK3R?v06h^~+BEIW{iDH-(qw$zx0B|OV zO6{>zM+Q&!`eESX{-fu$KEvlh&FLp$cs(FW#TbaighLgOlnqw--UsbNEPSceYsCj4d7Ju`*7VTcoVB;5?QKiv zz8L*%^mBj4esI?CZJF~v=UlC-`{>I1R~DNyRqkw6N4lybQ?+}ReOTtsmU+@;o~0|9 zvhG zoFh5-yo{#dgn;F!$1cc#5PK2U)^%XZhbFQar~}i}4`w2JvA}&bog9TJq*^l1Q&dS+ zJi^i9`8Av@A0S`r3xsAQ6c%`uTp>tEY7E6d*%nJvjR(6b0H^*PmRjd@ZFr`9@D^mM zR)Rp0YrP7xV1meP-5AwYgjioX7s15O&P7o4OQbcgBZel(V$c!K%{8zM&}^yI7VvIi zwP4Pa8lnF@LUpMBFF;~ZUeo%1wcN9-)y@E<*K=#>{|z1kU;FpCbE3xODzb;#L(z^r zpLL5aTu)=3bkiWxfT=B#42>mh1ry04_CgaNAPhZ3H$;j7SX;QR(1e)qHDjh$xw%FZ zrlWvf^5bjf7BII6Mmi#d^eJSLD7UFLc_PPIlfA2!vcKFmFMQClVyOp)c&`VTCe1dc zwjIi{hwrk7b6olS+4oM*_U0<8KiRe*{HkT8qA6Eh|KW{!I_IeVsQdly4|*0mR~*}O z&YFd@AD*7?g$%Aa{Qlv^eOX7#T}Mly>b|o+XRpM(&e-OxH?8x+hv5}wTaK~Lm%nGZ zWmsj)lLPD2tU<^>782Uxh#cKne02IN!VCe!is5P?92Z`V|K5eJ1uL}-^B+VD zHETFoa*$+sh$Oeqle*_IPXW-sMYyrZvMdJBFb@vcN-b)EuXsaf0Q%A>5YQ5}Zq)%7 z0YW020{ShFrt}z1@pLCWU?k|uprWf7xTj(UkPJp20xWbAV|4F z(Tyb$Q^9eCb^8XXVCdgRFag%S8^Zc5TTy?Xrt!xpVnu_5sz)OQB*v>dG5HLDJqq)I zO=|?cV?>PB6DDx$w6wQ0DHc$)K0(A9HKFfzH7ZnNLf7p|Y!$A@Kz|BBmv9}N72t}B zMg*4}jd{YtRUzz3Fm0}hXgm^hfwiF%1yI1z=@?kCiV++|0dN6f%}Y2`Y=#_6Ola)_ zF1-kmVu}T1Sr8)%3yzU+U|g66U8|S~a1bOBjvR^SAd)bFYwB6$WotYJ{xHE06NLj{ z%0Yv7am#;&2#i;oeeUwD%L_+7x%6r1m!T#7uOg|k8KlnsNT^_s-lpxtcF*y2ovO#G>-5H5+2e<{N%~18m{c=APR_w|l?d zbZ5t%npE9T#x|T{hQZ`5#nT0zXd*j$p$f(BDqXRWTi|CAV-#^Oj$X;Q{TNzG+sJHB zQ3VC66QV`;8ctIC`qk76JPh9|4;OPMrdXo#y@ZF??G_d5`cSdi%A;zuZjty3u26fy zISP6-KRVWw6GSVG6!afo@Y;aePTx*pz6KsWhR`+WCH9u)Q=azG7>jGN0r;;ZKl6qW zOmHNW@;rfjAr9yL`4v7pW0>p;b&Vzp5 zs}E6bT?zFpg(f5Y!b~^#l*Kn;7Pnr}wSz|?r1C0;8K84jHZ}K53K5Va;vvNipuU9a zZSM(3frq6-?S-a)1tUnX8}vPFEX7<**S6feHv9Hc@QdhY(bS>y+4c)_<+sW+wJrZg zB*V|@+Ni&=v{8eJ)(uD^zf{+{C)@=taaC$1>Rt#uTsIONpaz%6g$)G|g%14Xaur!h z-L4bh+7Q=5H(>U)-0fY*}GiSJk3ab<02Q`J($Dx|gq{yl<|w`cf_DQk6fx$DDs; zLR+@O;0DXPXMTESzG8)TuCn&|+Ie@1tzU?L!!|9(moB7sbbZ5irRc8b@eJlGB1`JY zPD;zZ4vZo7Du%)GP0@$IlygGvWgfhLf_e!z#~!>bl=TsgX2(gQE(Gcdp&*2B)3`?w z?8vizp(SM2Uv^@^+NE48*w44_PT_U?F|R^& z;CI40O57I)%p;vzTdzjCS^=bd?|6$>V+ffUVX%(@v2AmYt#A)ARdE_S08N* zV3&JS!sJok8He3E{22RoJR}N1@c@jD{et*y(}UkO6K0R+s%i;<484x4t?*_Li)p3! zGyie|y(Sd;RPT}Fpa)Gsc;JP%rG(An(fsPT0a1wxbv3ay6akYm6jWccC1E-MPPK7Y zelpOohPXW8>4KpcAp&n)@bavf^0ne9EJUF!iK-*{cT?A?_$8O-C2)!Ol%OX)0k%iN zUAoakJREjSfmc5&&bZ<+Fk@UB#>0-lkRXAlvtD7~c^6*dB|)(r5hp>9M`SPl6cr9D zU#KNz>RYgdw8Vvy_zqqwr}`P-An=p8d8l{5*UMv9{y_V%dtDd z*@4mYz-VUYSn|w6TjS!Mzd!WXhn9P?N6w~?oXyzyB-noTxzjgKXE|4zb1m&iajp#4 zndJ_qxq}(*aI*Iyw|!B_aBVnOv#@=!W@#Y9b!%VsOLDg3aJu7ghWimNYh2t5-N6Ax zGw+`M>FF$8lcsAHnm%=Z3`h>3@6zx$wYzVhOwlzf^zr{OIUX|Bl)ZWJ#$sgo=(3#M ze=@!QWNP23l&x>YJCyZaNP92bV=jJ&TOGLFl;YmQAoBxM{s3A26Kz_psr$6{m$f|?#IR+mCmYZhc+XHVv6^DHLC z*}fcYn>XXgvc?*>K$F@vJF=F|oxXK?O-C6zAM5p2UDCY9!LKxK$H@;jp19zB_(>1WG=Uxcqy^yT+EEJ8+-zb+UvuB$e1G@@U&d0OqU#^&5nW3*^#3py Bf5!j- literal 0 HcmV?d00001 diff --git a/hack/package-release-helm-chart.sh b/hack/package-release-helm-chart.sh new file mode 100755 index 0000000..697b03c --- /dev/null +++ b/hack/package-release-helm-chart.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -lt 1 || $# -gt 4 ]]; then + cat >&2 <<'EOF' +Usage: + hack/package-release-helm-chart.sh [app-version] [current-ref] [previous-ref] + +Examples: + hack/package-release-helm-chart.sh 0.6.0 v0.6.0 + hack/package-release-helm-chart.sh 0.6.1 v0.6.1 HEAD v0.6.0 + +The version must come from the existing release/version calculation. This +script only prepares release-specific Helm metadata and then delegates the +actual packaging and optional signing to package-helm-chart.sh. +EOF + exit 2 +fi + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +CHART_FILE="${ROOT_DIR}/charts/github-platform-operator/Chart.yaml" + +CHART_VERSION="${1#v}" +APP_VERSION="${2:-v${CHART_VERSION}}" +CURRENT_REF="${3:-HEAD}" +PREVIOUS_REF="${4:-}" + +chart_backup="$(mktemp)" + +restore_chart() { + if [[ -f "${chart_backup}" ]]; then + cp "${chart_backup}" "${CHART_FILE}" + rm -f "${chart_backup}" + fi +} + +trap restore_chart EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +cp "${CHART_FILE}" "${chart_backup}" + +prepare_args=( + --chart "${CHART_FILE}" + --repo-root "${ROOT_DIR}" + --version "${CHART_VERSION}" + --app-version "${APP_VERSION}" + --current-ref "${CURRENT_REF}" +) + +if [[ -n "${PREVIOUS_REF}" ]]; then + prepare_args+=(--previous-ref "${PREVIOUS_REF}") +fi + +python3 "${ROOT_DIR}/hack/prepare-artifacthub-chart.py" "${prepare_args[@]}" + +( + cd "${ROOT_DIR}" + + # Keep the existing packaging implementation as the single source of truth + # for CRD/chart validation and Helm provenance signing. + ./hack/package-helm-chart.sh "${CHART_VERSION}" "${APP_VERSION}" +) + +chart="${ROOT_DIR}/dist/helm/github-platform-operator-${CHART_VERSION}.tgz" + +if [[ ! -f "${chart}" ]]; then + echo "Helm chart was not generated: ${chart}" >&2 + exit 1 +fi + +echo "Release Helm metadata prepared automatically:" +echo " chart version: ${CHART_VERSION}" +echo " app version: ${APP_VERSION}" +echo " chart: ${chart}" diff --git a/hack/prepare-artifacthub-chart.py b/hack/prepare-artifacthub-chart.py new file mode 100755 index 0000000..8c5d56f --- /dev/null +++ b/hack/prepare-artifacthub-chart.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +"""Prepare release-only Helm metadata without modifying the source Chart.yaml.""" + +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import sys +from pathlib import Path + +CONVENTIONAL_COMMIT_RE = re.compile( + r"^(?P[A-Za-z]+)(?:\([^)]+\))?(?P!)?:\s+(?P.+)$" +) + +CHANGE_KIND_BY_COMMIT_TYPE = { + "feat": "added", + "fix": "fixed", + "perf": "changed", + "refactor": "changed", + "security": "security", +} + +SEMVER_RE = re.compile( + r"^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)" + r"(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?$" +) + + +def run_git(repo_root: Path, *args: str, allow_failure: bool = False) -> str: + result = subprocess.run( + ["git", *args], + cwd=repo_root, + check=False, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + if result.returncode != 0: + if allow_failure: + return "" + raise RuntimeError(result.stderr.strip() or f"git {' '.join(args)} failed") + return result.stdout.strip() + + +def resolve_previous_ref(repo_root: Path, current_ref: str) -> str: + # If current_ref is an already-created release tag, current_ref^ finds the + # tag before it. If the release tag does not exist yet, HEAD^ still resolves + # the latest reachable release tag. + previous = run_git( + repo_root, + "describe", + "--tags", + "--abbrev=0", + f"{current_ref}^", + allow_failure=True, + ) + if previous: + return previous + + return run_git( + repo_root, + "describe", + "--tags", + "--abbrev=0", + current_ref, + allow_failure=True, + ) + + +def conventional_changes(repo_root: Path, previous_ref: str, current_ref: str) -> list[tuple[str, str]]: + revision_range = f"{previous_ref}..{current_ref}" if previous_ref else current_ref + output = run_git( + repo_root, + "log", + "--no-merges", + "--format=%s", + revision_range, + allow_failure=False, + ) + + changes: list[tuple[str, str]] = [] + seen: set[tuple[str, str]] = set() + + for subject in output.splitlines(): + match = CONVENTIONAL_COMMIT_RE.match(subject.strip()) + if not match: + continue + + commit_type = match.group("type").lower() + kind = CHANGE_KIND_BY_COMMIT_TYPE.get(commit_type) + if kind is None: + # docs/test/ci/build/chore and unknown commit types are intentionally + # excluded from the user-facing Artifact Hub changelog. + continue + + description = match.group("description").strip() + if match.group("breaking"): + kind = "changed" + description = f"BREAKING: {description}" + + entry = (kind, description) + if entry not in seen: + seen.add(entry) + changes.append(entry) + + return changes + + +def remove_changes_annotation(lines: list[str]) -> list[str]: + result: list[str] = [] + index = 0 + + while index < len(lines): + if lines[index].startswith(" artifacthub.io/changes:"): + index += 1 + while index < len(lines): + line = lines[index] + if line.strip() == "": + index += 1 + break + if line.startswith(" "): + index += 1 + continue + break + continue + + result.append(lines[index]) + index += 1 + + return result + + +def update_scalar(lines: list[str], key: str, value: str) -> list[str]: + pattern = re.compile(rf"^{re.escape(key)}:\s*.*$") + replacement = f'{key}: "{value}"' if key == "appVersion" else f"{key}: {value}" + + matches = 0 + output: list[str] = [] + for line in lines: + if pattern.match(line): + output.append(replacement) + matches += 1 + else: + output.append(line) + + if matches != 1: + raise RuntimeError(f"expected exactly one top-level {key}, found {matches}") + return output + + +def add_changes_annotation(lines: list[str], changes: list[tuple[str, str]]) -> list[str]: + if not changes: + return lines + + insert_at = next( + (i for i, line in enumerate(lines) if line.startswith(" artifacthub.io/signKey:")), + None, + ) + if insert_at is None: + raise RuntimeError("artifacthub.io/signKey annotation not found") + + block = [" artifacthub.io/changes: |"] + for kind, description in changes: + block.append(f" - kind: {kind}") + # JSON string syntax is valid YAML and safely escapes quotes/newlines. + block.append(f" description: {json.dumps(description, ensure_ascii=False)}") + block.append("") + + return lines[:insert_at] + block + lines[insert_at:] + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--chart", required=True, type=Path) + parser.add_argument("--repo-root", required=True, type=Path) + parser.add_argument("--version", required=True) + parser.add_argument("--app-version", required=True) + parser.add_argument("--current-ref", default="HEAD") + parser.add_argument("--previous-ref", default="") + args = parser.parse_args() + + chart_version = args.version.removeprefix("v") + if not SEMVER_RE.fullmatch(chart_version): + parser.error(f"invalid Helm semantic version: {chart_version}") + + previous_ref = args.previous_ref or resolve_previous_ref(args.repo_root, args.current_ref) + changes = conventional_changes(args.repo_root, previous_ref, args.current_ref) + + lines = args.chart.read_text(encoding="utf-8").splitlines() + lines = remove_changes_annotation(lines) + lines = update_scalar(lines, "version", chart_version) + lines = update_scalar(lines, "appVersion", args.app_version) + lines = add_changes_annotation(lines, changes) + + args.chart.write_text("\n".join(lines) + "\n", encoding="utf-8") + + if previous_ref: + print(f"Artifact Hub changes: {previous_ref}..{args.current_ref}", file=sys.stderr) + else: + print(f"Artifact Hub changes: full history up to {args.current_ref}", file=sys.stderr) + print(f"Artifact Hub entries: {len(changes)}", file=sys.stderr) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())