diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7b3aa29..20ecfe0 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -149,7 +149,7 @@ jobs: run: | set -euo pipefail - ./hack/package-helm-chart.sh \ + ./hack/package-release-helm-chart.sh \ 0.0.0-snapshot \ v0.0.0-snapshot @@ -212,7 +212,7 @@ jobs: } ' "${rendered}"; then echo "The packaged Helm chart does not contain the complete GitHubRepositoryRuleset RBAC." >&2 - echo "Check config/rbac/role.yaml and hack/package-helm-chart.sh." >&2 + echo "Check config/rbac/role.yaml and hack/package-release-helm-chart.sh." >&2 exit 1 fi diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 6514afe..9c02fc4 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -237,7 +237,7 @@ jobs: run: | set -euo pipefail - ./hack/package-helm-chart.sh \ + ./hack/package-release-helm-chart.sh \ "${CHART_VERSION}" \ "${APP_VERSION}" @@ -316,7 +316,7 @@ jobs: ' "${rendered}"; then echo "The packaged Helm chart does not contain the complete GitHubRepositoryRuleset RBAC." >&2 echo "Publishing has been stopped." >&2 - echo "Check config/rbac/role.yaml and hack/package-helm-chart.sh." >&2 + echo "Check config/rbac/role.yaml and hack/package-release-helm-chart.sh." >&2 exit 1 fi diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index eb9ce4d..0026680 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -106,7 +106,7 @@ repos: rm -rf dist/helm - ./hack/package-helm-chart.sh \ + ./hack/package-release-helm-chart.sh \ "${chart_version}" \ "${app_version}" diff --git a/README.md b/README.md index 20d0d51..002a524 100644 --- a/README.md +++ b/README.md @@ -176,6 +176,7 @@ policies or upgrading CRD schemas. - [Getting started](docs/getting-started.md) - [Custom resources and examples](docs/resources.md) - [Operations, upgrades and troubleshooting](docs/operations.md) +- [Grafana dashboard](dashboards/grafana/README.md) - [Helm chart](charts/github-platform-operator/README.md) - [Development guide](docs/development.md) - [Release process](docs/releasing.md) diff --git a/charts/github-platform-operator/Chart.yaml b/charts/github-platform-operator/Chart.yaml index abdb288..f52e6c9 100644 --- a/charts/github-platform-operator/Chart.yaml +++ b/charts/github-platform-operator/Chart.yaml @@ -2,29 +2,41 @@ apiVersion: v2 name: github-platform-operator description: Kubernetes operator for managing GitHub repositories, access and Actions configuration type: application -version: 0.1.0 -appVersion: "v0.1.0" + +# Release versions are injected into the packaged chart by the release pipeline. +version: 0.0.0 +appVersion: "v0.0.0" + home: https://github.com/pierinho13/github-platform-operator + sources: - https://github.com/pierinho13/github-platform-operator + maintainers: - name: pierinho13 email: pierinho13@gmail.com url: https://github.com/pierinho13 + keywords: - github - github-actions - kubernetes - operator - platform-engineering + annotations: artifacthub.io/category: integration-delivery artifacthub.io/license: Apache-2.0 artifacthub.io/operator: "true" artifacthub.io/operatorCapabilities: Full Lifecycle + + # artifacthub.io/changes is generated only in the packaged release chart from + # Conventional Commits since the previous Git tag. + artifacthub.io/signKey: | fingerprint: DBBC4A49D2BB68565A9BCC383CA8825330932B17 url: https://raw.githubusercontent.com/pierinho13/github-platform-operator/main/helm-signing-key.asc + artifacthub.io/links: | - name: Source url: https://github.com/pierinho13/github-platform-operator @@ -32,5 +44,5 @@ annotations: url: https://github.com/pierinho13/github-platform-operator#documentation - name: Getting Started url: https://github.com/pierinho13/github-platform-operator/blob/main/docs/getting-started.md - - name: support - url: https://github.com/pierinho13/github-platform-operator/issues + - name: Support + url: https://github.com/pierinho13/github-platform-operator/issues \ No newline at end of file diff --git a/charts/github-platform-operator/README.md b/charts/github-platform-operator/README.md index 44bd7a2..677e668 100644 --- a/charts/github-platform-operator/README.md +++ b/charts/github-platform-operator/README.md @@ -170,6 +170,12 @@ helm show values \ oci://ghcr.io/pierinho13/charts/github-platform-operator ``` +The operator exposes standard `controller-runtime` metrics plus GitHub API and +rate-limit metrics. An importable Grafana dashboard is available in the source +repository at `dashboards/grafana/github-platform-operator.json`. See the +[operations guide](https://github.com/pierinho13/github-platform-operator/blob/main/docs/operations.md#metrics-and-grafana) +for scraping and dashboard details. + ## CRDs and upgrades The chart includes generated CRDs under `crds/`. Helm installs them before the diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 5c5f0b8..b0b8c68 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -1,2 +1,8 @@ resources: - manager.yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +images: +- name: controller + newName: example.com/github-platform-operator + newTag: v0.0.1 diff --git a/dashboards/grafana/README.md b/dashboards/grafana/README.md new file mode 100644 index 0000000..64b4cf8 --- /dev/null +++ b/dashboards/grafana/README.md @@ -0,0 +1,40 @@ +# Grafana dashboard + +`github-platform-operator.json` is an importable Grafana dashboard for the +operator's Prometheus metrics. + +It combines the custom GitHub API metrics exported by the operator with the +standard metrics already exposed by `controller-runtime`, the Go runtime and +the process collector. + +## Prerequisite + +Prometheus must scrape the operator `/metrics` endpoint. The project already +ships the metrics Service and a ServiceMonitor example under +[`config/prometheus`](../../config/prometheus). + +When secure metrics are enabled, the Prometheus service account must be allowed +to read the authenticated `/metrics` endpoint. See +[`docs/operations.md`](../../docs/operations.md#metrics-and-grafana) for the +operational notes. + +## Import + +In Grafana, use **Dashboards → New → Import**, upload +`github-platform-operator.json`, and select the Prometheus data source. + +The dashboard includes: + +- GitHub API request volume and HTTP status codes +- GitHub API p95 latency +- transport errors +- rate-limit remaining/limit/reset state +- primary and secondary rate-limit events +- shared rate-limit gate blocking time +- controller reconciliation rate, errors and p95 duration +- workqueue depth +- process CPU, resident memory and Go goroutines + +The dashboard intentionally avoids repository, organization, username and token +labels to keep metric cardinality bounded and to avoid exposing sensitive +identifiers through Prometheus labels. diff --git a/dashboards/grafana/github-platform-operator.json b/dashboards/grafana/github-platform-operator.json new file mode 100644 index 0000000..b6ab9d5 --- /dev/null +++ b/dashboards/grafana/github-platform-operator.json @@ -0,0 +1,1640 @@ +{ + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "10.0.0" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Operational dashboard for github-platform-operator: GitHub API usage, rate limits, reconciliation health, workqueues and runtime resources.", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "id": 1, + "type": "stat", + "title": "GitHub API requests", + "gridPos": { + "x": 0, + "y": 0, + "w": 4, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(increase(github_platform_operator_github_api_requests_total{job=~\"$job\"}[$__range]))", + "legendFormat": "", + "range": false, + "refId": "A", + "instant": true + } + ] + }, + { + "id": 2, + "type": "stat", + "title": "GitHub API 4xx / 5xx", + "gridPos": { + "x": 4, + "y": 0, + "w": 4, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(increase(github_platform_operator_github_api_requests_total{job=~\"$job\",status_code=~\"4..|5..\"}[$__range]))", + "legendFormat": "", + "range": false, + "refId": "A", + "instant": true + } + ] + }, + { + "id": 3, + "type": "stat", + "title": "Rate limit remaining", + "gridPos": { + "x": 8, + "y": 0, + "w": 4, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "max(github_platform_operator_github_rate_limit_remaining{job=~\"$job\",resource=\"$rate_resource\"})", + "legendFormat": "", + "range": false, + "refId": "A", + "instant": true + } + ] + }, + { + "id": 4, + "type": "stat", + "title": "Rate limit used", + "gridPos": { + "x": 12, + "y": 0, + "w": 4, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "percent", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 70 + }, + { + "color": "red", + "value": 90 + } + ] + }, + "decimals": 1 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "100 * (1 - max(github_platform_operator_github_rate_limit_remaining{job=~\"$job\",resource=\"$rate_resource\"}) / clamp_min(max(github_platform_operator_github_rate_limit_limit{job=~\"$job\",resource=\"$rate_resource\"}), 1))", + "legendFormat": "", + "range": false, + "refId": "A", + "instant": true + } + ] + }, + { + "id": 5, + "type": "stat", + "title": "Rate-limit events", + "gridPos": { + "x": 16, + "y": 0, + "w": 4, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(increase(github_platform_operator_github_rate_limit_events_total{job=~\"$job\"}[$__range]))", + "legendFormat": "", + "range": false, + "refId": "A", + "instant": true + } + ] + }, + { + "id": 6, + "type": "stat", + "title": "Gate blocked for", + "gridPos": { + "x": 20, + "y": 0, + "w": 4, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "clamp_min(max(github_platform_operator_github_rate_limit_blocked_until_timestamp_seconds{job=~\"$job\"}) - time(), 0)", + "legendFormat": "", + "range": false, + "refId": "A", + "instant": true + } + ] + }, + { + "id": 7, + "type": "timeseries", + "title": "GitHub API request rate", + "description": "", + "gridPos": { + "x": 0, + "y": 4, + "w": 12, + "h": 7 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "reqps", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 10, + "gradientMode": "none", + "showPoints": "auto", + "spanNulls": false, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (status_code) (rate(github_platform_operator_github_api_requests_total{job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "HTTP {{status_code}}", + "range": true, + "refId": "A", + "instant": false + } + ] + }, + { + "id": 8, + "type": "timeseries", + "title": "GitHub API p95 latency by method", + "description": "", + "gridPos": { + "x": 12, + "y": 4, + "w": 12, + "h": 7 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 10, + "gradientMode": "none", + "showPoints": "auto", + "spanNulls": false, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by (le, method) (rate(github_platform_operator_github_api_request_duration_seconds_bucket{job=~\"$job\"}[$__rate_interval])))", + "legendFormat": "{{method}}", + "range": true, + "refId": "A", + "instant": false + } + ] + }, + { + "id": 9, + "type": "timeseries", + "title": "GitHub rate limit", + "description": "Last rate-limit values observed from GitHub response headers.", + "gridPos": { + "x": 0, + "y": 11, + "w": 12, + "h": 7 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 10, + "gradientMode": "none", + "showPoints": "auto", + "spanNulls": false, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "max by (resource) (github_platform_operator_github_rate_limit_remaining{job=~\"$job\",resource=\"$rate_resource\"})", + "legendFormat": "remaining {{resource}}", + "range": true, + "refId": "A", + "instant": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "max by (resource) (github_platform_operator_github_rate_limit_limit{job=~\"$job\",resource=\"$rate_resource\"})", + "legendFormat": "limit {{resource}}", + "range": true, + "refId": "B", + "instant": false + } + ] + }, + { + "id": 10, + "type": "timeseries", + "title": "Rate-limit events by kind", + "description": "", + "gridPos": { + "x": 12, + "y": 11, + "w": 12, + "h": 7 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 10, + "gradientMode": "none", + "showPoints": "auto", + "spanNulls": false, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (kind) (rate(github_platform_operator_github_rate_limit_events_total{job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "{{kind}}", + "range": true, + "refId": "A", + "instant": false + } + ] + }, + { + "id": 11, + "type": "timeseries", + "title": "Reconciliation rate by controller", + "description": "", + "gridPos": { + "x": 0, + "y": 18, + "w": 12, + "h": 7 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 10, + "gradientMode": "none", + "showPoints": "auto", + "spanNulls": false, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (controller) (rate(controller_runtime_reconcile_total{job=~\"$job\",controller=~\"$controller\"}[$__rate_interval]))", + "legendFormat": "{{controller}}", + "range": true, + "refId": "A", + "instant": false + } + ] + }, + { + "id": 12, + "type": "timeseries", + "title": "Reconciliation errors by controller", + "description": "", + "gridPos": { + "x": 12, + "y": 18, + "w": 12, + "h": 7 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 10, + "gradientMode": "none", + "showPoints": "auto", + "spanNulls": false, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (controller) (rate(controller_runtime_reconcile_total{job=~\"$job\",controller=~\"$controller\",result=\"error\"}[$__rate_interval]))", + "legendFormat": "{{controller}}", + "range": true, + "refId": "A", + "instant": false + } + ] + }, + { + "id": 13, + "type": "timeseries", + "title": "Reconciliation p95 duration", + "description": "", + "gridPos": { + "x": 0, + "y": 25, + "w": 12, + "h": 7 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 10, + "gradientMode": "none", + "showPoints": "auto", + "spanNulls": false, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, sum by (le, controller) (rate(controller_runtime_reconcile_time_seconds_bucket{job=~\"$job\",controller=~\"$controller\"}[$__rate_interval])))", + "legendFormat": "{{controller}}", + "range": true, + "refId": "A", + "instant": false + } + ] + }, + { + "id": 14, + "type": "timeseries", + "title": "Workqueue depth", + "description": "", + "gridPos": { + "x": 12, + "y": 25, + "w": 12, + "h": 7 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 10, + "gradientMode": "none", + "showPoints": "auto", + "spanNulls": false, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (name) (workqueue_depth{job=~\"$job\"})", + "legendFormat": "{{name}}", + "range": true, + "refId": "A", + "instant": false + } + ] + }, + { + "id": 15, + "type": "timeseries", + "title": "GitHub transport errors", + "description": "", + "gridPos": { + "x": 0, + "y": 32, + "w": 8, + "h": 6 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 10, + "gradientMode": "none", + "showPoints": "auto", + "spanNulls": false, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum by (method) (rate(github_platform_operator_github_api_transport_errors_total{job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "{{method}}", + "range": true, + "refId": "A", + "instant": false + } + ] + }, + { + "id": 16, + "type": "timeseries", + "title": "Process CPU", + "description": "", + "gridPos": { + "x": 8, + "y": 32, + "w": 8, + "h": 6 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "cores", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 10, + "gradientMode": "none", + "showPoints": "auto", + "spanNulls": false, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(rate(process_cpu_seconds_total{job=~\"$job\"}[$__rate_interval]))", + "legendFormat": "CPU", + "range": true, + "refId": "A", + "instant": false + } + ] + }, + { + "id": 17, + "type": "timeseries", + "title": "Resident memory", + "description": "", + "gridPos": { + "x": 16, + "y": 32, + "w": 8, + "h": 6 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 1, + "fillOpacity": 10, + "gradientMode": "none", + "showPoints": "auto", + "spanNulls": false, + "axisPlacement": "auto", + "axisLabel": "", + "axisColorMode": "text", + "scaleDistribution": { + "type": "linear" + }, + "hideFrom": { + "tooltip": false, + "viz": false, + "legend": false + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom", + "calcs": [] + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(process_resident_memory_bytes{job=~\"$job\"})", + "legendFormat": "RSS", + "range": true, + "refId": "A", + "instant": false + } + ] + }, + { + "id": 18, + "type": "stat", + "title": "Go goroutines", + "gridPos": { + "x": 0, + "y": 38, + "w": 6, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(go_goroutines{job=~\"$job\"})", + "legendFormat": "", + "range": false, + "refId": "A", + "instant": true + } + ] + }, + { + "id": 19, + "type": "stat", + "title": "Reconcile errors", + "gridPos": { + "x": 6, + "y": 38, + "w": 6, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(increase(controller_runtime_reconcile_total{job=~\"$job\",controller=~\"$controller\",result=\"error\"}[$__range]))", + "legendFormat": "", + "range": false, + "refId": "A", + "instant": true + } + ] + }, + { + "id": 20, + "type": "stat", + "title": "Transport errors", + "gridPos": { + "x": 12, + "y": 38, + "w": 6, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 1 + } + ] + }, + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(increase(github_platform_operator_github_api_transport_errors_total{job=~\"$job\"}[$__range]))", + "legendFormat": "", + "range": false, + "refId": "A", + "instant": true + } + ] + }, + { + "id": 21, + "type": "stat", + "title": "Rate reset in", + "gridPos": { + "x": 18, + "y": 38, + "w": 6, + "h": 4 + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "decimals": 0 + }, + "overrides": [] + }, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto", + "wideLayout": true + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "clamp_min(max(github_platform_operator_github_rate_limit_reset_timestamp_seconds{job=~\"$job\",resource=\"$rate_resource\"}) - time(), 0)", + "legendFormat": "", + "range": false, + "refId": "A", + "instant": true + } + ] + } + ], + "refresh": "30s", + "schemaVersion": 39, + "tags": [ + "github-platform-operator", + "kubernetes", + "operator", + "github" + ], + "templating": { + "list": [ + { + "name": "job", + "label": "Prometheus job", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "query": { + "query": "label_values(github_platform_operator_github_rate_limit_blocked_until_timestamp_seconds, job)", + "refId": "StandardVariableQuery" + }, + "definition": "label_values(github_platform_operator_github_rate_limit_blocked_until_timestamp_seconds, job)", + "includeAll": true, + "allValue": ".*", + "multi": true, + "refresh": 1, + "sort": 1, + "current": { + "selected": false, + "text": "All", + "value": "$__all" + } + }, + { + "name": "controller", + "label": "Controller", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "query": { + "query": "label_values(controller_runtime_reconcile_total{job=~\"$job\"}, controller)", + "refId": "StandardVariableQuery" + }, + "definition": "label_values(controller_runtime_reconcile_total{job=~\"$job\"}, controller)", + "includeAll": true, + "allValue": ".*", + "multi": true, + "refresh": 1, + "sort": 1, + "current": { + "selected": false, + "text": "All", + "value": "$__all" + } + }, + { + "name": "rate_resource", + "label": "Rate-limit resource", + "type": "query", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "query": { + "query": "label_values(github_platform_operator_github_rate_limit_remaining{job=~\"$job\"}, resource)", + "refId": "StandardVariableQuery" + }, + "definition": "label_values(github_platform_operator_github_rate_limit_remaining{job=~\"$job\"}, resource)", + "includeAll": false, + "multi": false, + "refresh": 1, + "sort": 1, + "current": { + "selected": true, + "text": "core", + "value": "core" + } + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "GitHub Platform Operator", + "uid": "github-platform-operator", + "version": 1, + "weekStart": "" +} diff --git a/docs/operations.md b/docs/operations.md index d467311..5691db9 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -115,6 +115,46 @@ resources without producing a continuous error storm. A normal authorization or feature-plan `403` is not treated as a rate limit. Inspect the response message in the resource condition or controller logs. +## Metrics and Grafana + +The manager exposes the standard `controller-runtime`, Go runtime and process +metrics together with a small set of GitHub-specific Prometheus metrics: + +| Metric | Purpose | +|---|---| +| `github_platform_operator_github_api_requests_total` | GitHub API responses by HTTP method and status code | +| `github_platform_operator_github_api_request_duration_seconds` | GitHub API HTTP round-trip latency histogram by method | +| `github_platform_operator_github_api_transport_errors_total` | Requests that failed before an HTTP response was received | +| `github_platform_operator_github_rate_limit_limit` | Last observed GitHub rate-limit ceiling by resource | +| `github_platform_operator_github_rate_limit_remaining` | Last observed remaining requests by rate-limit resource | +| `github_platform_operator_github_rate_limit_reset_timestamp_seconds` | Last observed rate-limit reset timestamp | +| `github_platform_operator_github_rate_limit_events_total` | Detected primary, secondary or unknown rate-limit events | +| `github_platform_operator_github_rate_limit_blocked_until_timestamp_seconds` | Timestamp until which the shared reactive rate-limit gate is blocked | + +The metrics are instrumented in the shared GitHub HTTP transport, so the same +measurements cover repository, organization, team, ruleset, Actions and GitHub +App token requests without duplicating instrumentation in every controller. +Rate-limit gauges represent the last response observed by the shared HTTP client; +they are intentionally not labeled by provider, organization, repository or +credential in order to avoid sensitive labels and unbounded cardinality. + +Prometheus must scrape the manager metrics Service. The Kustomize deployment +contains a ServiceMonitor example under `config/prometheus`. With secure metrics, +the scraping service account must also be authorized for the non-resource +`/metrics` URL. + +An importable Grafana dashboard is included at: + +```text +dashboards/grafana/github-platform-operator.json +``` + +The dashboard combines these GitHub-specific metrics with existing metrics such +as `controller_runtime_reconcile_total`, +`controller_runtime_reconcile_time_seconds`, `workqueue_depth`, +`process_cpu_seconds_total`, `process_resident_memory_bytes` and +`go_goroutines`. + ## Secret and variable rotation Update the referenced Kubernetes Secret: diff --git a/go.mod b/go.mod index f084115..30fc206 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.26.0 require ( github.com/onsi/ginkgo/v2 v2.32.0 github.com/onsi/gomega v1.42.1 + github.com/prometheus/client_golang v1.23.2 golang.org/x/crypto v0.53.0 k8s.io/api v0.36.3 k8s.io/apimachinery v0.36.3 @@ -42,12 +43,12 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect - github.com/prometheus/client_golang v1.23.2 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.67.5 // indirect github.com/prometheus/procfs v0.19.2 // indirect diff --git a/hack/__pycache__/prepare-artifacthub-chart.cpython-314.pyc b/hack/__pycache__/prepare-artifacthub-chart.cpython-314.pyc new file mode 100644 index 0000000..8365b67 Binary files /dev/null and b/hack/__pycache__/prepare-artifacthub-chart.cpython-314.pyc differ diff --git a/hack/package-release-helm-chart.sh b/hack/package-release-helm-chart.sh new file mode 100755 index 0000000..697b03c --- /dev/null +++ b/hack/package-release-helm-chart.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +set -euo pipefail + +if [[ $# -lt 1 || $# -gt 4 ]]; then + cat >&2 <<'EOF' +Usage: + hack/package-release-helm-chart.sh [app-version] [current-ref] [previous-ref] + +Examples: + hack/package-release-helm-chart.sh 0.6.0 v0.6.0 + hack/package-release-helm-chart.sh 0.6.1 v0.6.1 HEAD v0.6.0 + +The version must come from the existing release/version calculation. This +script only prepares release-specific Helm metadata and then delegates the +actual packaging and optional signing to package-helm-chart.sh. +EOF + exit 2 +fi + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +CHART_FILE="${ROOT_DIR}/charts/github-platform-operator/Chart.yaml" + +CHART_VERSION="${1#v}" +APP_VERSION="${2:-v${CHART_VERSION}}" +CURRENT_REF="${3:-HEAD}" +PREVIOUS_REF="${4:-}" + +chart_backup="$(mktemp)" + +restore_chart() { + if [[ -f "${chart_backup}" ]]; then + cp "${chart_backup}" "${CHART_FILE}" + rm -f "${chart_backup}" + fi +} + +trap restore_chart EXIT +trap 'exit 130' INT +trap 'exit 143' TERM + +cp "${CHART_FILE}" "${chart_backup}" + +prepare_args=( + --chart "${CHART_FILE}" + --repo-root "${ROOT_DIR}" + --version "${CHART_VERSION}" + --app-version "${APP_VERSION}" + --current-ref "${CURRENT_REF}" +) + +if [[ -n "${PREVIOUS_REF}" ]]; then + prepare_args+=(--previous-ref "${PREVIOUS_REF}") +fi + +python3 "${ROOT_DIR}/hack/prepare-artifacthub-chart.py" "${prepare_args[@]}" + +( + cd "${ROOT_DIR}" + + # Keep the existing packaging implementation as the single source of truth + # for CRD/chart validation and Helm provenance signing. + ./hack/package-helm-chart.sh "${CHART_VERSION}" "${APP_VERSION}" +) + +chart="${ROOT_DIR}/dist/helm/github-platform-operator-${CHART_VERSION}.tgz" + +if [[ ! -f "${chart}" ]]; then + echo "Helm chart was not generated: ${chart}" >&2 + exit 1 +fi + +echo "Release Helm metadata prepared automatically:" +echo " chart version: ${CHART_VERSION}" +echo " app version: ${APP_VERSION}" +echo " chart: ${chart}" diff --git a/hack/prepare-artifacthub-chart.py b/hack/prepare-artifacthub-chart.py new file mode 100755 index 0000000..8c5d56f --- /dev/null +++ b/hack/prepare-artifacthub-chart.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +"""Prepare release-only Helm metadata without modifying the source Chart.yaml.""" + +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import sys +from pathlib import Path + +CONVENTIONAL_COMMIT_RE = re.compile( + r"^(?P[A-Za-z]+)(?:\([^)]+\))?(?P!)?:\s+(?P.+)$" +) + +CHANGE_KIND_BY_COMMIT_TYPE = { + "feat": "added", + "fix": "fixed", + "perf": "changed", + "refactor": "changed", + "security": "security", +} + +SEMVER_RE = re.compile( + r"^(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)" + r"(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?$" +) + + +def run_git(repo_root: Path, *args: str, allow_failure: bool = False) -> str: + result = subprocess.run( + ["git", *args], + cwd=repo_root, + check=False, + text=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) + if result.returncode != 0: + if allow_failure: + return "" + raise RuntimeError(result.stderr.strip() or f"git {' '.join(args)} failed") + return result.stdout.strip() + + +def resolve_previous_ref(repo_root: Path, current_ref: str) -> str: + # If current_ref is an already-created release tag, current_ref^ finds the + # tag before it. If the release tag does not exist yet, HEAD^ still resolves + # the latest reachable release tag. + previous = run_git( + repo_root, + "describe", + "--tags", + "--abbrev=0", + f"{current_ref}^", + allow_failure=True, + ) + if previous: + return previous + + return run_git( + repo_root, + "describe", + "--tags", + "--abbrev=0", + current_ref, + allow_failure=True, + ) + + +def conventional_changes(repo_root: Path, previous_ref: str, current_ref: str) -> list[tuple[str, str]]: + revision_range = f"{previous_ref}..{current_ref}" if previous_ref else current_ref + output = run_git( + repo_root, + "log", + "--no-merges", + "--format=%s", + revision_range, + allow_failure=False, + ) + + changes: list[tuple[str, str]] = [] + seen: set[tuple[str, str]] = set() + + for subject in output.splitlines(): + match = CONVENTIONAL_COMMIT_RE.match(subject.strip()) + if not match: + continue + + commit_type = match.group("type").lower() + kind = CHANGE_KIND_BY_COMMIT_TYPE.get(commit_type) + if kind is None: + # docs/test/ci/build/chore and unknown commit types are intentionally + # excluded from the user-facing Artifact Hub changelog. + continue + + description = match.group("description").strip() + if match.group("breaking"): + kind = "changed" + description = f"BREAKING: {description}" + + entry = (kind, description) + if entry not in seen: + seen.add(entry) + changes.append(entry) + + return changes + + +def remove_changes_annotation(lines: list[str]) -> list[str]: + result: list[str] = [] + index = 0 + + while index < len(lines): + if lines[index].startswith(" artifacthub.io/changes:"): + index += 1 + while index < len(lines): + line = lines[index] + if line.strip() == "": + index += 1 + break + if line.startswith(" "): + index += 1 + continue + break + continue + + result.append(lines[index]) + index += 1 + + return result + + +def update_scalar(lines: list[str], key: str, value: str) -> list[str]: + pattern = re.compile(rf"^{re.escape(key)}:\s*.*$") + replacement = f'{key}: "{value}"' if key == "appVersion" else f"{key}: {value}" + + matches = 0 + output: list[str] = [] + for line in lines: + if pattern.match(line): + output.append(replacement) + matches += 1 + else: + output.append(line) + + if matches != 1: + raise RuntimeError(f"expected exactly one top-level {key}, found {matches}") + return output + + +def add_changes_annotation(lines: list[str], changes: list[tuple[str, str]]) -> list[str]: + if not changes: + return lines + + insert_at = next( + (i for i, line in enumerate(lines) if line.startswith(" artifacthub.io/signKey:")), + None, + ) + if insert_at is None: + raise RuntimeError("artifacthub.io/signKey annotation not found") + + block = [" artifacthub.io/changes: |"] + for kind, description in changes: + block.append(f" - kind: {kind}") + # JSON string syntax is valid YAML and safely escapes quotes/newlines. + block.append(f" description: {json.dumps(description, ensure_ascii=False)}") + block.append("") + + return lines[:insert_at] + block + lines[insert_at:] + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--chart", required=True, type=Path) + parser.add_argument("--repo-root", required=True, type=Path) + parser.add_argument("--version", required=True) + parser.add_argument("--app-version", required=True) + parser.add_argument("--current-ref", default="HEAD") + parser.add_argument("--previous-ref", default="") + args = parser.parse_args() + + chart_version = args.version.removeprefix("v") + if not SEMVER_RE.fullmatch(chart_version): + parser.error(f"invalid Helm semantic version: {chart_version}") + + previous_ref = args.previous_ref or resolve_previous_ref(args.repo_root, args.current_ref) + changes = conventional_changes(args.repo_root, previous_ref, args.current_ref) + + lines = args.chart.read_text(encoding="utf-8").splitlines() + lines = remove_changes_annotation(lines) + lines = update_scalar(lines, "version", chart_version) + lines = update_scalar(lines, "appVersion", args.app_version) + lines = add_changes_annotation(lines, changes) + + args.chart.write_text("\n".join(lines) + "\n", encoding="utf-8") + + if previous_ref: + print(f"Artifact Hub changes: {previous_ref}..{args.current_ref}", file=sys.stderr) + else: + print(f"Artifact Hub changes: full history up to {args.current_ref}", file=sys.stderr) + print(f"Artifact Hub entries: {len(changes)}", file=sys.stderr) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/internal/github/auth_rate_limit_contract_test.go b/internal/github/auth_rate_limit_contract_test.go index 7e11a71..486149d 100644 --- a/internal/github/auth_rate_limit_contract_test.go +++ b/internal/github/auth_rate_limit_contract_test.go @@ -400,8 +400,8 @@ func TestPrimaryRateLimitResetContract(t *testing.T) { response := &http.Response{ StatusCode: http.StatusForbidden, Header: http.Header{ - "X-Ratelimit-Remaining": []string{"0"}, - "X-Ratelimit-Reset": []string{strconv.FormatInt(resetAt, 10)}, + rateLimitHeaderRemaining: []string{"0"}, + rateLimitHeaderReset: []string{strconv.FormatInt(resetAt, 10)}, }, Body: io.NopCloser(strings.NewReader(`{"message":"API rate limit exceeded"}`)), } diff --git a/internal/github/metrics.go b/internal/github/metrics.go new file mode 100644 index 0000000..90d581f --- /dev/null +++ b/internal/github/metrics.go @@ -0,0 +1,223 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package github + +import ( + "net/http" + "strconv" + "strings" + "time" + + "github.com/prometheus/client_golang/prometheus" + controllermetrics "sigs.k8s.io/controller-runtime/pkg/metrics" +) + +const ( + metricsNamespace = "github_platform_operator" + metricsSubsystem = "github" + metricsMethodLabel = "method" + metricsResourceLabel = "resource" + rateLimitResourceCore = "core" +) + +type githubMetrics struct { + apiRequests *prometheus.CounterVec + apiRequestDuration *prometheus.HistogramVec + apiTransportErrors *prometheus.CounterVec + rateLimitLimit *prometheus.GaugeVec + rateLimitRemaining *prometheus.GaugeVec + rateLimitReset *prometheus.GaugeVec + rateLimitEvents *prometheus.CounterVec + rateLimitBlockedUntil prometheus.Gauge +} + +func newGitHubMetrics(registerer prometheus.Registerer) *githubMetrics { + m := &githubMetrics{ + apiRequests: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: metricsNamespace, + Subsystem: metricsSubsystem, + Name: "api_requests_total", + Help: "Total GitHub API responses observed by the shared HTTP client.", + }, + []string{metricsMethodLabel, "status_code"}, + ), + apiRequestDuration: prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: metricsNamespace, + Subsystem: metricsSubsystem, + Name: "api_request_duration_seconds", + Help: "GitHub API HTTP round-trip duration in seconds until response headers are received.", + Buckets: prometheus.DefBuckets, + }, + []string{metricsMethodLabel}, + ), + apiTransportErrors: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: metricsNamespace, + Subsystem: metricsSubsystem, + Name: "api_transport_errors_total", + Help: "Total GitHub API requests that failed before an HTTP response was received.", + }, + []string{metricsMethodLabel}, + ), + rateLimitLimit: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: metricsNamespace, + Subsystem: metricsSubsystem, + Name: "rate_limit_limit", + Help: "Last observed GitHub API rate-limit ceiling for a rate-limit resource.", + }, + []string{metricsResourceLabel}, + ), + rateLimitRemaining: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: metricsNamespace, + Subsystem: metricsSubsystem, + Name: "rate_limit_remaining", + Help: "Last observed GitHub API requests remaining for a rate-limit resource.", + }, + []string{metricsResourceLabel}, + ), + rateLimitReset: prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: metricsNamespace, + Subsystem: metricsSubsystem, + Name: "rate_limit_reset_timestamp_seconds", + Help: "Last observed GitHub API rate-limit reset time as a Unix timestamp.", + }, + []string{metricsResourceLabel}, + ), + rateLimitEvents: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: metricsNamespace, + Subsystem: metricsSubsystem, + Name: "rate_limit_events_total", + Help: "Total GitHub API rate-limit responses detected by kind.", + }, + []string{"kind"}, + ), + rateLimitBlockedUntil: prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: metricsNamespace, + Subsystem: metricsSubsystem, + Name: "rate_limit_blocked_until_timestamp_seconds", + Help: "Unix timestamp until which the shared GitHub API gate is blocked; zero means no block has been observed.", + }, + ), + } + + registerer.MustRegister( + m.apiRequests, + m.apiRequestDuration, + m.apiTransportErrors, + m.rateLimitLimit, + m.rateLimitRemaining, + m.rateLimitReset, + m.rateLimitEvents, + m.rateLimitBlockedUntil, + ) + + return m +} + +var defaultGitHubMetrics = newGitHubMetrics(controllermetrics.Registry) + +func (m *githubMetrics) observeResponse(request *http.Request, response *http.Response, duration time.Duration) { + if m == nil || response == nil { + return + } + + method := normalizeHTTPMethod(request) + m.apiRequests.WithLabelValues(method, strconv.Itoa(response.StatusCode)).Inc() + m.apiRequestDuration.WithLabelValues(method).Observe(duration.Seconds()) + m.observeRateLimitHeaders(response.Header) +} + +func (m *githubMetrics) observeTransportError(request *http.Request, duration time.Duration) { + if m == nil { + return + } + + method := normalizeHTTPMethod(request) + m.apiTransportErrors.WithLabelValues(method).Inc() + m.apiRequestDuration.WithLabelValues(method).Observe(duration.Seconds()) +} + +func (m *githubMetrics) observeRateLimitHeaders(header http.Header) { + if m == nil || header == nil { + return + } + + resource := strings.TrimSpace(header.Get(rateLimitHeaderResource)) + if resource == "" { + resource = rateLimitResourceCore + } + + setGaugeFromHeader(m.rateLimitLimit.WithLabelValues(resource), header.Get(rateLimitHeaderLimit)) + setGaugeFromHeader(m.rateLimitRemaining.WithLabelValues(resource), header.Get(rateLimitHeaderRemaining)) + setGaugeFromHeader(m.rateLimitReset.WithLabelValues(resource), header.Get(rateLimitHeaderReset)) +} + +func (m *githubMetrics) observeRateLimit( + response *http.Response, + rateErr *RateLimitError, + blockedUntil time.Time, +) { + if m == nil || rateErr == nil { + return + } + + m.rateLimitEvents.WithLabelValues(rateLimitKind(response, rateErr)).Inc() + if !blockedUntil.IsZero() { + m.rateLimitBlockedUntil.Set(float64(blockedUntil.Unix())) + } +} + +func setGaugeFromHeader(gauge prometheus.Gauge, value string) { + parsed, err := strconv.ParseFloat(strings.TrimSpace(value), 64) + if err != nil { + return + } + gauge.Set(parsed) +} + +func normalizeHTTPMethod(request *http.Request) string { + if request == nil || strings.TrimSpace(request.Method) == "" { + return "UNKNOWN" + } + return strings.ToUpper(request.Method) +} + +func rateLimitKind(response *http.Response, rateErr *RateLimitError) string { + if response != nil && strings.TrimSpace(response.Header.Get(rateLimitHeaderRemaining)) == "0" { + return "primary" + } + + body := "" + if rateErr != nil { + body = strings.ToLower(rateErr.Body) + } + if response != nil && (response.StatusCode == http.StatusTooManyRequests || + strings.TrimSpace(response.Header.Get("Retry-After")) != "") { + return "secondary" + } + if strings.Contains(body, "secondary rate limit") || strings.Contains(body, "abuse detection") { + return "secondary" + } + return "unknown" +} diff --git a/internal/github/metrics_contract_test.go b/internal/github/metrics_contract_test.go new file mode 100644 index 0000000..4f4d9a7 --- /dev/null +++ b/internal/github/metrics_contract_test.go @@ -0,0 +1,158 @@ +/* +Copyright 2026. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package github + +import ( + "errors" + "io" + "net/http" + "strconv" + "strings" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" +) + +func TestRateLimitTransportMetricsContract(t *testing.T) { + t.Parallel() + + registry := prometheus.NewRegistry() + metricSet := newGitHubMetrics(registry) + reset := time.Now().Add(10 * time.Minute).Unix() + transport := &RateLimitTransport{ + Base: roundTripFunc(func(request *http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{ + rateLimitHeaderLimit: []string{"5000"}, + rateLimitHeaderRemaining: []string{"4242"}, + rateLimitHeaderReset: []string{strconv.FormatInt(reset, 10)}, + rateLimitHeaderResource: []string{rateLimitResourceCore}, + }, + Body: io.NopCloser(strings.NewReader(`{"ok":true}`)), + Request: request, + }, nil + }), + gate: &rateLimitGate{}, + metrics: metricSet, + } + + request, err := http.NewRequest(http.MethodGet, "https://api.github.test/repos/example/repo", nil) + if err != nil { + t.Fatalf("create request: %v", err) + } + response, err := transport.RoundTrip(request) + if err != nil { + t.Fatalf("round trip: %v", err) + } + closeResponseBody(response.Body) + + if got := testutil.ToFloat64(metricSet.apiRequests.WithLabelValues(http.MethodGet, "200")); got != 1 { + t.Fatalf("api request counter = %v, want 1", got) + } + if got := testutil.ToFloat64(metricSet.rateLimitLimit.WithLabelValues(rateLimitResourceCore)); got != 5000 { + t.Fatalf("rate-limit limit = %v, want 5000", got) + } + if got := testutil.ToFloat64(metricSet.rateLimitRemaining.WithLabelValues(rateLimitResourceCore)); got != 4242 { + t.Fatalf("rate-limit remaining = %v, want 4242", got) + } + if got := testutil.ToFloat64(metricSet.rateLimitReset.WithLabelValues(rateLimitResourceCore)); got != float64(reset) { + t.Fatalf("rate-limit reset = %v, want %d", got, reset) + } + if count := testutil.CollectAndCount(metricSet.apiRequestDuration); count != 1 { + t.Fatalf("request duration metric families = %d, want 1", count) + } +} + +func TestRateLimitTransportRecordsTransportErrorsContract(t *testing.T) { + t.Parallel() + + registry := prometheus.NewRegistry() + metricSet := newGitHubMetrics(registry) + transport := &RateLimitTransport{ + Base: roundTripFunc(func(_ *http.Request) (*http.Response, error) { + return nil, errors.New("network unavailable") + }), + gate: &rateLimitGate{}, + metrics: metricSet, + } + + request, err := http.NewRequest(http.MethodPost, "https://api.github.test/app/installations/1/access_tokens", nil) + if err != nil { + t.Fatalf("create request: %v", err) + } + if _, err := transport.RoundTrip(request); err == nil { + t.Fatal("expected transport error") + } + + if got := testutil.ToFloat64(metricSet.apiTransportErrors.WithLabelValues(http.MethodPost)); got != 1 { + t.Fatalf("transport error counter = %v, want 1", got) + } +} + +func TestRateLimitTransportRecordsRateLimitEventsContract(t *testing.T) { + t.Parallel() + + registry := prometheus.NewRegistry() + metricSet := newGitHubMetrics(registry) + reset := time.Now().Add(2 * time.Minute).Unix() + transport := &RateLimitTransport{ + Base: roundTripFunc(func(request *http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: http.StatusForbidden, + Header: http.Header{ + rateLimitHeaderLimit: []string{"5000"}, + rateLimitHeaderRemaining: []string{"0"}, + rateLimitHeaderReset: []string{strconv.FormatInt(reset, 10)}, + rateLimitHeaderResource: []string{rateLimitResourceCore}, + }, + Body: io.NopCloser(strings.NewReader(`{"message":"API rate limit exceeded"}`)), + Request: request, + }, nil + }), + gate: &rateLimitGate{}, + metrics: metricSet, + } + + request, err := http.NewRequest(http.MethodGet, "https://api.github.test/orgs/example/teams", nil) + if err != nil { + t.Fatalf("create request: %v", err) + } + _, err = transport.RoundTrip(request) + var rateErr *RateLimitError + if !errors.As(err, &rateErr) { + t.Fatalf("expected RateLimitError, got %v", err) + } + + if got := testutil.ToFloat64(metricSet.rateLimitEvents.WithLabelValues("primary")); got != 1 { + t.Fatalf("primary rate-limit events = %v, want 1", got) + } + if got := testutil.ToFloat64(metricSet.rateLimitRemaining.WithLabelValues(rateLimitResourceCore)); got != 0 { + t.Fatalf("rate-limit remaining = %v, want 0", got) + } + if got := testutil.ToFloat64(metricSet.rateLimitBlockedUntil); got < float64(reset) { + t.Fatalf("blocked-until timestamp = %v, want at least %d", got, reset) + } +} + +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (f roundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) { + return f(request) +} diff --git a/internal/github/rate_limit.go b/internal/github/rate_limit.go index 16ae5cf..617ec20 100644 --- a/internal/github/rate_limit.go +++ b/internal/github/rate_limit.go @@ -28,7 +28,13 @@ import ( "time" ) -const fallbackSecondaryRateLimitDelay = time.Minute +const ( + fallbackSecondaryRateLimitDelay = time.Minute + rateLimitHeaderLimit = "X-Ratelimit-Limit" + rateLimitHeaderRemaining = "X-Ratelimit-Remaining" + rateLimitHeaderReset = "X-Ratelimit-Reset" + rateLimitHeaderResource = "X-Ratelimit-Resource" +) // RateLimitError indicates that GitHub asked the operator to stop making // requests temporarily. It can represent both primary and secondary limits. @@ -85,7 +91,7 @@ func (g *rateLimitGate) currentError(now time.Time) *RateLimitError { } } -func (g *rateLimitGate) block(until time.Time, statusCode int, body string) { +func (g *rateLimitGate) block(until time.Time, statusCode int, body string) time.Time { g.mu.Lock() defer g.mu.Unlock() @@ -94,6 +100,7 @@ func (g *rateLimitGate) block(until time.Time, statusCode int, body string) { g.statusCode = statusCode g.body = body } + return g.blockedUntil } // RateLimitTransport adds a shared, reactive GitHub rate-limit gate. @@ -107,6 +114,7 @@ type RateLimitTransport struct { gateOnce sync.Once gate *rateLimitGate + metrics *githubMetrics } // NewRateLimitedHTTPClient creates an HTTP client whose rate-limit state is @@ -115,8 +123,9 @@ func NewRateLimitedHTTPClient() *http.Client { return &http.Client{ Timeout: defaultRequestTimeout, Transport: &RateLimitTransport{ - Base: http.DefaultTransport, - gate: &rateLimitGate{}, + Base: http.DefaultTransport, + gate: &rateLimitGate{}, + metrics: defaultGitHubMetrics, }, } } @@ -142,10 +151,18 @@ func (t *RateLimitTransport) RoundTrip(request *http.Request) (*http.Response, e base = http.DefaultTransport } + requestStartedAt := time.Now() response, err := base.RoundTrip(request) + duration := time.Since(requestStartedAt) + metrics := t.metrics + if metrics == nil { + metrics = defaultGitHubMetrics + } if err != nil { + metrics.observeTransportError(request, duration) return nil, err } + metrics.observeResponse(request, response, duration) rateErr, err := rateLimitErrorFromResponse(response) if err != nil { @@ -156,7 +173,8 @@ func (t *RateLimitTransport) RoundTrip(request *http.Request) (*http.Response, e return response, nil } - gate.block(rateErr.RetryAt, rateErr.StatusCode, rateErr.Body) + blockedUntil := gate.block(rateErr.RetryAt, rateErr.StatusCode, rateErr.Body) + metrics.observeRateLimit(response, rateErr, blockedUntil) closeResponseBody(response.Body) return nil, rateErr } @@ -178,7 +196,7 @@ func rateLimitErrorFromResponse(response *http.Response) (*RateLimitError, error bodyText := strings.TrimSpace(string(body)) lowerBody := strings.ToLower(bodyText) - remaining := strings.TrimSpace(response.Header.Get("X-RateLimit-Remaining")) + remaining := strings.TrimSpace(response.Header.Get(rateLimitHeaderRemaining)) retryAfter := strings.TrimSpace(response.Header.Get("Retry-After")) isRateLimited := response.StatusCode == http.StatusTooManyRequests || @@ -202,7 +220,7 @@ func rateLimitErrorFromResponse(response *http.Response) (*RateLimitError, error } } else if remaining == "0" { if reset, parseErr := strconv.ParseInt( - strings.TrimSpace(response.Header.Get("X-RateLimit-Reset")), + strings.TrimSpace(response.Header.Get(rateLimitHeaderReset)), 10, 64, ); parseErr == nil && reset > 0 { diff --git a/test/e2e/e2e_test.go b/test/e2e/e2e_test.go index 0aa4a1c..edee5da 100644 --- a/test/e2e/e2e_test.go +++ b/test/e2e/e2e_test.go @@ -254,6 +254,9 @@ var _ = Describe("Manager", Ordered, func() { Expect(metricsOutput).To(ContainSubstring( "controller_runtime_reconcile_total", )) + Expect(metricsOutput).To(ContainSubstring( + "github_platform_operator_github_rate_limit_blocked_until_timestamp_seconds", + )) }) // +kubebuilder:scaffold:e2e-webhooks-checks