diff --git a/.github/scripts/changelog_generator.py b/.github/scripts/changelog_generator.py index 48f5308..bd1e91f 100644 --- a/.github/scripts/changelog_generator.py +++ b/.github/scripts/changelog_generator.py @@ -221,10 +221,22 @@ def generate_changelog(self, output_file: str = "CHANGELOG.md") -> str: changelog_text = "\n".join(changelog) + # Collapse runs of blank lines. Sections are joined with "\n" but each + # already ends in a newline, so the naive join yields doubled blanks, + # which markdownlint rejects (MD012/no-multiple-blanks). Because this + # file is committed with a skip-ci marker, the hook never runs here to + # catch it -- it surfaces instead on every unrelated PR. + changelog_text = re.sub(r"\n{3,}", "\n\n", changelog_text) + # Save to file try: with open(output_file, "w", encoding="utf-8") as f: - f.write(changelog_text) + # Trailing newline is required. Without it end-of-file-fixer + # rewrites CHANGELOG.md on every subsequent PR, and because + # this file is committed with [skip ci] the hook never runs + # here to correct it. That tripped pre-commit-ci.yml's + # auto-fix step on PRs that do not touch CHANGELOG.md at all. + f.write(changelog_text + "\n") print(f"✅ Changelog generated: {output_file}") except Exception as e: print(f"❌ Error saving changelog: {e}") diff --git a/.github/workflows/intake-canary.yml b/.github/workflows/intake-canary.yml new file mode 100644 index 0000000..cb12301 --- /dev/null +++ b/.github/workflows/intake-canary.yml @@ -0,0 +1,261 @@ +name: Intake Canary + +# --------------------------------------------------------------------------- +# WHY THIS EXISTS +# +# GitHub -> DRC intake has died silently TWICE: +# * 2026-07-23 Event Router "Normalise Issue Payload" threw on every event +# (undetected 7 days) +# * 2026-08-22 Event Router "Validate HMAC Signature" rejected 100% of +# traffic - GITHUB_WEBHOOK_SECRET unset (issue #259) +# +# Both times, EVERY existing signal reported GREEN during a total outage: +# +# 1. GitHub's webhook delivery log -> 200 OK +# The n8n Webhook node answers `onReceived` BEFORE the workflow runs, so +# GitHub records a success and the workflow errors asynchronously after. +# Verified 2026-08-23: delivery at 06:12:29 = 200 OK; the n8n execution it +# spawned (6970) = status "error". The delivery log cannot see this class +# of failure. Do not build alerting on it. +# +# 2. n8n-health-check.yml's DRC probe -> 200 OK +# The "Route Health Ping" IF node (added 2026-07-06 to stop false +# pipeline-down alarms) short-circuits pings to a Pong 200 BEFORE the auth +# gate and before any real work. It proves n8n is serving HTTP. It proves +# nothing about the pipeline. The 07-06 fix for false POSITIVES is what +# manufactured this false NEGATIVE. +# +# 3. n8n host root -> 200 OK (proves only that n8n Cloud is up) +# +# The ONLY signal that distinguishes a healthy pipeline from a dead one is the +# n8n *executions* API. That is what this workflow polls. +# +# KNOWN GAP (deliberate, v1): this detects executions with status "error". A +# success-status LOGIC failure - events routed down a wrong branch, DRC never +# triggered - still slips through. A true end-to-end probe needs a synthetic +# issue, which costs LLM spend per run and may be eaten by the Event Router's +# bot-sender filter. Out of scope until the error-class detection is proven. +# --------------------------------------------------------------------------- + +on: + schedule: + # Every 30 minutes. NOTE: scheduled runs only fire from the DEFAULT branch. + - cron: '*/30 * * * *' + workflow_dispatch: + +permissions: + contents: read + +env: + N8N_HOST: https://gadgetlab.app.n8n.cloud + # Event Router - the component that has broken twice. Webhook-driven, so a + # long silence is itself a symptom (hook deleted / workflow deactivated). + EVENT_ROUTER_ID: oijizIGJtRzBG94Z + # DRC Agent Loop - error check only. A silence check here would be VACUOUS: + # the 6-hourly health-check Pong pings always create executions, so DRC can + # never look silent even when it is doing no real work. + DRC_ID: Wlfhgk4sUfXJcU4D + # Guaranteed nightly traffic: the autopilot daily-summary issue lands ~02:26 + # UTC and produces ~6 Event Router executions. 26h therefore spans at least + # one guaranteed burst; anything shorter false-fires on quiet weekends. + SILENCE_THRESHOLD_HOURS: '26' + # Slack de-duplication. The job goes RED every tick regardless - GitHub + # Actions is the continuous signal. Slack is only for NEW breakage, because + # an alarm that fires 48x/day during a known-open incident is the exact + # input that produced the 2026-07-06 "fix" (move the probe earlier so it + # stops complaining) that created this blind spot in the first place. + # A failing execution newer than this window counts as fresh. + ALERT_WINDOW_MINUTES: '90' + +jobs: + canary: + name: GitHub -> DRC intake canary + runs-on: ubuntu-latest + + steps: + - name: Check n8n pipeline execution health + id: probe + env: + N8N_API_KEY: ${{ secrets.N8N_API_KEY }} + run: | + set -uo pipefail + + # Coarse, stateless reminder gate for conditions that persist until a + # human acts (broken key, prolonged silence). Must be computed BEFORE + # any branch that can exit, or `set -u` kills the branch that needs it. + # hour % 6 == 0 and minute < 30 => ~4 Slack pages/day, at most one per + # 30-min tick pair. + # 10# forces base-10: `08`/`09` are invalid octal and would error out. + if [ $(( 10#$(date -u +%H) % 6 )) -eq 0 ] && [ $(( 10#$(date -u +%M) )) -lt 30 ]; then + REMIND=true + else + REMIND=false + fi + + # --- Fail loudly on a missing secret ----------------------------- + # Without this the curl below sends an empty header, n8n returns 401, + # and `jq '.data | length'` on the error body yields 0 - which reads + # as "zero errors, all healthy". That is the exact silent-pass this + # canary exists to prevent. Verified against the live API 2026-08-23. + if [ -z "${N8N_API_KEY:-}" ]; then + { + echo "status=canary_broken" + echo "detail=N8N_API_KEY secret is not set on this repository." + echo "slack=${REMIND}" + } >> "$GITHUB_OUTPUT" + echo "::error::N8N_API_KEY is not set - the canary cannot see the pipeline." + exit 1 + fi + + fetch() { + # $1 = workflow id, $2 = output file. Echoes the HTTP status. + curl -s -o "$2" -w '%{http_code}' --max-time 30 \ + -H "X-N8N-API-KEY: ${N8N_API_KEY}" \ + "${N8N_HOST}/api/v1/executions?workflowId=$1&limit=20" + } + + ROUTER_BODY=/tmp/router.json + DRC_BODY=/tmp/drc.json + + ROUTER_HTTP=$(fetch "$EVENT_ROUTER_ID" "$ROUTER_BODY") || ROUTER_HTTP="curl_failed" + DRC_HTTP=$(fetch "$DRC_ID" "$DRC_BODY") || DRC_HTTP="curl_failed" + + # --- Assert transport BEFORE reading the body -------------------- + # A 401/403/5xx body still parses as JSON and still yields 0 errors. + # Never fold a transport failure into the healthy path. + if [ "$ROUTER_HTTP" != "200" ] || [ "$DRC_HTTP" != "200" ]; then + { + echo "status=canary_broken" + echo "detail=n8n executions API unreachable (Event Router HTTP ${ROUTER_HTTP}, DRC HTTP ${DRC_HTTP}). Key expired, rotated, or n8n Cloud down." + echo "slack=${REMIND}" + } >> "$GITHUB_OUTPUT" + echo "::error::n8n API returned ${ROUTER_HTTP}/${DRC_HTTP} - cannot assess pipeline health." + exit 1 + fi + + # Shape guard: a 200 that is not the expected envelope is also a + # canary fault, not a healthy pipeline. + if ! jq -e '.data | type == "array"' "$ROUTER_BODY" >/dev/null 2>&1; then + { + echo "status=canary_broken" + echo "detail=n8n returned HTTP 200 but the response had no .data array. API shape changed." + echo "slack=${REMIND}" + } >> "$GITHUB_OUTPUT" + echo "::error::Unexpected response shape from the executions API." + exit 1 + fi + + # --- Consecutive-error streak, newest first ---------------------- + # Chosen over "any error in the last N hours" so the signal SELF-CLEARS: + # the first successful execution after a fix turns the canary green + # immediately, instead of staying red for the length of the window. + # (The raw v1 API has no startedAfter filter - verified 2026-08-23 - + # so all windowing is done here in jq.) + streak() { + # Count LEADING "error" statuses only. Deliberately not + # `index("success")`: that would count a "running" or "waiting" + # execution at position 0 as a failure and fire a false alarm. + jq '[.data[].status] | (map(. == "error") | index(false) // length)' "$1" + } + + ROUTER_STREAK=$(streak "$ROUTER_BODY") + DRC_STREAK=$(streak "$DRC_BODY") + + LATEST_AT=$(jq -r '.data[0].startedAt // empty' "$ROUTER_BODY") + TOTAL=$(jq '.data | length' "$ROUTER_BODY") + + echo "Event Router: ${TOTAL} recent executions, ${ROUTER_STREAK} consecutive errors (newest first)" + echo "DRC Agent Loop: ${DRC_STREAK} consecutive errors (newest first)" + echo "Event Router latest execution: ${LATEST_AT:-}" + + # --- Silence check (Event Router only) --------------------------- + if [ -z "$LATEST_AT" ]; then + { + echo "status=pipeline_down" + echo "detail=Event Router has NO executions on record. The GitHub webhook may be deleted or the workflow deactivated." + echo "slack=${REMIND}" + } >> "$GITHUB_OUTPUT" + echo "::error::Event Router has no executions at all." + exit 1 + fi + + AGE_H=$(( ( $(date -u +%s) - $(date -u -d "$LATEST_AT" +%s) ) / 3600 )) + echo "Event Router last execution age: ${AGE_H}h (threshold ${SILENCE_THRESHOLD_HOURS}h)" + + if [ "$AGE_H" -gt "$SILENCE_THRESHOLD_HOURS" ]; then + { + echo "status=pipeline_down" + echo "detail=Event Router has been SILENT for ${AGE_H}h (threshold ${SILENCE_THRESHOLD_HOURS}h). No GitHub events are arriving - webhook deleted, or workflow deactivated." + echo "slack=${REMIND}" + } >> "$GITHUB_OUTPUT" + echo "::error::Event Router silent for ${AGE_H}h." + exit 1 + fi + + # --- Error check ------------------------------------------------- + if [ "$ROUTER_STREAK" -gt 0 ] || [ "$DRC_STREAK" -gt 0 ]; then + ERR_ID=$(jq -r '.data[0].id // "?"' "$ROUTER_BODY") + + # Slack only on FRESH breakage. A stale streak means a known-open + # incident that nobody has fixed yet - the red job already says so, + # and re-paging every 30 min just trains people to mute the channel. + # + # LIMITATION: freshness is measured from the EVENT ROUTER's newest + # execution. If the router is healthy and only DRC is failing, this + # gates on a router timestamp that has nothing to do with DRC, so a + # DRC-only failure may be suppressed from Slack. The job still exits + # 1 either way, so the Actions signal is unaffected - but do not rely + # on Slack alone for DRC-only faults. Fix in v2: track freshness per + # workflow. + AGE_M=$(( ( $(date -u +%s) - $(date -u -d "$LATEST_AT" +%s) ) / 60 )) + if [ "$AGE_M" -le "$ALERT_WINDOW_MINUTES" ]; then + echo "slack=true" >> "$GITHUB_OUTPUT" + else + echo "slack=false" >> "$GITHUB_OUTPUT" + echo "Known-open incident (newest failure ${AGE_M}m old) - job fails, Slack suppressed." + fi + + { + echo "status=pipeline_down" + echo "detail=Event Router: ${ROUTER_STREAK} consecutive failed executions (latest id ${ERR_ID}, ${AGE_M}m old). DRC: ${DRC_STREAK}. GitHub events are being dropped - inbound automation is DEAD." + } >> "$GITHUB_OUTPUT" + echo "::error::Pipeline erroring - Event Router streak ${ROUTER_STREAK}, DRC streak ${DRC_STREAK}." + exit 1 + fi + + echo "status=healthy" >> "$GITHUB_OUTPUT" + echo "Pipeline healthy." + + - name: Alert Slack + # Two distinct alert classes, because they need DIFFERENT hands: + # canary_broken -> a developer fixes the canary / rotates the key + # pipeline_down -> Gadget fixes credentials in the n8n UI + # Folding them together costs a wasted escalation every time. + if: failure() && steps.probe.outputs.slack == 'true' + env: + SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }} + STATUS: ${{ steps.probe.outputs.status }} + DETAIL: ${{ steps.probe.outputs.detail }} + RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} + run: | + set -uo pipefail + if [ -z "${SLACK_WEBHOOK_URL:-}" ]; then + echo "::warning::SLACK_WEBHOOK_URL not set - job status is the only signal." + exit 0 + fi + + if [ "$STATUS" = "canary_broken" ]; then + HEADLINE=":wrench: *INTAKE CANARY BROKEN* - the canary cannot see the pipeline." + ACTION="Needs a developer: the n8n API key is missing, expired, or rotated. Pipeline state is UNKNOWN, not healthy." + else + HEADLINE=":rotating_light: *GITHUB -> DRC INTAKE IS DOWN* - events are being dropped." + ACTION="Needs Gadget in the n8n UI. Note: GitHub's webhook delivery log will still show 200 OK - that is expected and does NOT mean this is a false alarm." + fi + + jq -n \ + --arg text "${HEADLINE}"$'\n\n'"${DETAIL}"$'\n\n'"${ACTION}"$'\n\n'"<${RUN_URL}|View run>" \ + '{text: $text}' \ + | curl -s -X POST -H 'Content-Type: application/json' \ + --max-time 20 -d @- "$SLACK_WEBHOOK_URL" \ + && echo "Slack alert sent." \ + || echo "::warning::Slack post failed - job status is the backstop." diff --git a/.github/workflows/n8n-health-check.yml b/.github/workflows/n8n-health-check.yml index 9b4f919..c4b50cf 100644 --- a/.github/workflows/n8n-health-check.yml +++ b/.github/workflows/n8n-health-check.yml @@ -1,5 +1,17 @@ name: n8n Pipeline Health Check +# SCOPE LIMIT - READ BEFORE TRUSTING THIS WORKFLOW'S GREEN. +# This checks REACHABILITY ONLY: that n8n Cloud is serving HTTP. +# It CANNOT detect a broken pipeline, and has already failed to twice +# (2026-07-23 Normalise Issue Payload, 2026-08-22 HMAC secret unset). +# * The DRC probe below hits the "Route Health Ping" IF node, which returns +# Pong 200 BEFORE the auth gate and before any real work. A Pong proves +# n8n is up. It proves NOTHING about whether GitHub events are flowing. +# * Verified 2026-08-23: this workflow reported green while 20/20 consecutive +# Event Router executions were erroring and 100% of events were dropped. +# The authoritative pipeline-health signal is `intake-canary.yml`, which polls +# the n8n executions API. If the two disagree, the canary is right. + on: schedule: # Every 6 hours @@ -86,7 +98,7 @@ jobs: \"fields\": [ {\"type\": \"mrkdwn\", \"text\": \"*DRC Webhook:* HTTP ${DRC_STATUS}\"}, {\"type\": \"mrkdwn\", \"text\": \"*n8n Host:* HTTP ${HOST_STATUS}\"}, - {\"type\": \"mrkdwn\", \"text\": \"*Impact:* GitHub events are not being routed through the DRC loop\"}, + {\"type\": \"mrkdwn\", \"text\": \"*Impact:* n8n Cloud is unreachable. NOTE: this check only measures reachability - see intake-canary.yml for actual pipeline health\"}, {\"type\": \"mrkdwn\", \"text\": \"*Action:* Check n8n.cloud status and verify workflows are active\"} ] }, @@ -116,12 +128,16 @@ jobs: DRC_STATUS=${{ steps.drc_ping.outputs.http_status }} HOST_STATUS=${{ steps.router_ping.outputs.http_status }} - echo "## n8n Pipeline Health" >> "$GITHUB_STEP_SUMMARY" + echo "## n8n Reachability (NOT pipeline health)" >> "$GITHUB_STEP_SUMMARY" + echo "" >> "$GITHUB_STEP_SUMMARY" + echo "> This workflow only proves n8n Cloud is serving HTTP. The DRC probe" >> "$GITHUB_STEP_SUMMARY" + echo "> hits a Pong short-circuit and cannot see a broken pipeline." >> "$GITHUB_STEP_SUMMARY" + echo "> Authoritative signal: \`intake-canary.yml\`." >> "$GITHUB_STEP_SUMMARY" echo "" >> "$GITHUB_STEP_SUMMARY" echo "| Endpoint | HTTP Status | Status |" >> "$GITHUB_STEP_SUMMARY" echo "|----------|-------------|--------|" >> "$GITHUB_STEP_SUMMARY" if [ "$DRC_ALIVE" = "true" ]; then - echo "| DRC Webhook | \`${DRC_STATUS}\` | ✅ Reachable |" >> "$GITHUB_STEP_SUMMARY" + echo "| DRC Webhook | \`${DRC_STATUS}\` | ✅ Pong received (reachability only) |" >> "$GITHUB_STEP_SUMMARY" else echo "| DRC Webhook | \`${DRC_STATUS}\` | 🔴 Unreachable |" >> "$GITHUB_STEP_SUMMARY" fi diff --git a/CHANGELOG.md b/CHANGELOG.md index f2ba715..aa337d1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,10 +5,8 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - ## [Unreleased] - 2026-08-21 - ### ✨ Features - **core**: global incident freeze kill-switch (#241) (3be209c) @@ -385,7 +383,6 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [v1.0.0] - 2026-03-20 - ### ✨ Features - dispatcher, dry-run gate, flaky detection, rollback manifest (#82) (fc6c145) @@ -437,4 +434,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### 💎 Code Style -- apply black formatting to core/, agents/, autopilot/ (#76) (1a74c28) \ No newline at end of file +- apply black formatting to core/, agents/, autopilot/ (#76) (1a74c28)