docs(lab8): golden signals monitoring and alerting

.github/pull_request_template.md

@@ -0,0 +1,13 @@
+## Goal
+<!-- What does this PR accomplish? 1 sentence. -->
+
+## Changes
+-
+
+## Testing
+<!-- How did you verify it? -->
+
+## Checklist
+- [ ] Title is a clear sentence (≤ 70 chars)
+- [ ] Commits are signed (`git log --show-signature`)
+- [ ] `submissions/labN.md` updated

.gitignore

@@ -61,3 +61,5 @@ Thumbs.db
 #   *.sbom.cdx.json, zap-*.html/json, trivy-*.txt   (Lab 9 scan evidence)
 #   flake.nix, flake.lock      (Lab 11)
 #   wasm/main.go, spin.toml, go.sum   (Lab 12)
+data/
+app/data/

app/Dockerfile

@@ -0,0 +1,24 @@
+# syntax=docker/dockerfile:1
+
+FROM golang:1.24.6-bookworm AS builder
+WORKDIR /src
+COPY go.mod go.su[m] ./
+RUN go mod download
+COPY . .
+RUN CGO_ENABLED=0 go build \
+      -trimpath \
+      -ldflags='-s -w' \
+      -o /out/quicknotes .
+RUN mkdir -p /data-empty
+
+FROM busybox:1.37-uclibc AS busybox
+
+FROM gcr.io/distroless/static:nonroot
+WORKDIR /app
+COPY --from=builder /out/quicknotes /app/quicknotes
+COPY --from=builder /src/seed.json /app/seed.json
+COPY --from=busybox /bin/wget /bin/wget
+COPY --from=builder --chown=65532:65532 /data-empty /data
+USER nonroot:nonroot
+EXPOSE 8080
+ENTRYPOINT ["/app/quicknotes"]

app/handlers.go

@@ -50,6 +50,9 @@ func (sw *statusWriter) WriteHeader(code int) {
 func (s *Server) wrap(h http.HandlerFunc) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		sw := &statusWriter{ResponseWriter: w, code: 200}
+		w.Header().Set("X-Content-Type-Options", "nosniff")
+		w.Header().Set("Cross-Origin-Resource-Policy", "same-origin")
+		w.Header().Set("Cache-Control", "no-store")
 		h(sw, r)
 		s.requestsTotal.Add(1)
 		if c, ok := s.requestsByCode[sw.code]; ok {

app/handlers_test.go

@@ -130,4 +130,3 @@ func TestMetrics_ExposesPrometheusFormat(t *testing.T) {
 		}
 	}
 }
-

compose.yaml

@@ -0,0 +1,56 @@
+services:
+  quicknotes:
+    build: ./app
+    image: quicknotes:lab6
+    ports:
+      - "8080:8080"
+    environment:
+      ADDR: ":8080"
+      DATA_PATH: /data/notes.json
+      SEED_PATH: /app/seed.json
+    healthcheck:
+      test: ["CMD", "/bin/wget", "-q", "-O", "-", "http://127.0.0.1:8080/health"]
+      interval: 10s
+      timeout: 3s
+      retries: 3
+      start_period: 5s
+    cap_drop:
+      - ALL
+    read_only: true
+    tmpfs:
+      - /tmp
+    security_opt:
+      - no-new-privileges:true
+    volumes:
+      - quicknotes-data:/data
+    restart: unless-stopped
+
+  prometheus:
+    image: prom/prometheus:v3.7.3
+    volumes:
+      - ./monitoring/prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro
+      - ./monitoring/prometheus/alerts.yml:/etc/prometheus/alerts.yml:ro
+    ports:
+      - "9090:9090"
+    depends_on:
+      quicknotes:
+        condition: service_healthy
+    restart: unless-stopped
+
+  grafana:
+    image: grafana/grafana:13.1.2
+    environment:
+      GF_SECURITY_ADMIN_USER: admin
+      GF_SECURITY_ADMIN_PASSWORD: Qn-Lab6-Gf-9x2v
+    volumes:
+      - ./monitoring/grafana/provisioning:/etc/grafana/provisioning:ro
+      - ./monitoring/grafana/dashboards:/var/lib/grafana/dashboards:ro
+    ports:
+      - "3000:3000"
+    depends_on:
+      - prometheus
+    restart: unless-stopped
+
+
+volumes:
+  quicknotes-data:

docs/runbook/high-error-rate.md

@@ -0,0 +1,61 @@
+# Runbook — HighErrorRate
+
+**Alert:** `HighErrorRate`
+**Severity:** `page`
+**Expression:** 4xx+5xx ratio > 5% sustained for 5 minutes
+
+---
+
+## What this alert means
+
+More than one in twenty requests to QuickNotes is failing, and has been for at
+least five minutes — users are being turned away right now.
+
+---
+
+## Triage steps
+
+1. **Confirm the service is up at all.**
+A non-200 or a container not in `healthy` state means this is an outage, not
+   an error-rate problem — skip to Mitigation 1.
+
+2. **Find out which status code dominates.** Open the Errors panel on the
+   *QuickNotes — Golden Signals* dashboard, or query Prometheus directly:
+4xx means clients are sending bad requests — a broken caller, a bad deploy on
+   their side, or someone probing. 5xx means QuickNotes itself is failing.
+
+3. **Check whether a deploy correlates with the onset.** Compare the time the
+   ratio started climbing against the container start time:
+If the error onset lines up with a restart, treat the deploy as the cause
+   until proven otherwise.
+
+4. **Check disk and the data volume.** The store writes to `/data`; a full or
+   read-only volume produces 5xx on every write:
+---
+
+## Mitigations
+
+1. **Roll back to the previous image.** Fastest fix when the onset correlates
+   with a deploy. Stop the current container first — starting a second instance
+   on the same port fails with `bind: address already in use`:
+Verify with `curl -s http://localhost:8080/health` before declaring recovery.
+
+2. **Restart the container.** Clears wedged state such as an exhausted
+   connection pool or a stuck file handle. Data survives — it lives in the named
+   volume, not the container:
+3. **If the errors are 4xx from a single caller**, the application is behaving
+   correctly and the fix is upstream. Rate-limit or block that client at the
+   proxy rather than changing QuickNotes.
+
+---
+
+## Post-incident
+
+Write a blameless postmortem covering: timeline (first bad request → alert fired
+→ mitigation → recovery), user impact in requests and minutes, root cause, and
+the specific change that prevents recurrence. Follow the structure used in
+`submissions/lab4.md` §2.4 — summary, timeline, what went wrong, why it was hard
+to see, what prevents it.
+
+Then check this alert itself: if it fired and no user was actually affected,
+that is a tuning bug, and it belongs in the postmortem's action items.

evidence/lab8/prometheus-state.txt

@@ -0,0 +1,21 @@
+$ curl -s http://localhost:9090/api/v1/targets | jq ...
+{
+  "job": "quicknotes",
+  "health": "up",
+  "scrapeUrl": "http://quicknotes:8080/metrics"
+}
+
+$ curl -s http://localhost:9090/api/v1/rules | jq ...
+{
+  "name": "HighErrorRate",
+  "state": "firing",
+  "duration": 300,
+  "labels": {
+    "severity": "page"
+  },
+  "activeAt": "2026-08-06T17:28:12.464142167Z",
+  "value": "4.7951176983435045e-01"
+}
+
+$ error ratio at firing time
+0.47951176983435045

monitoring/grafana/dashboards/golden-signals.json

@@ -0,0 +1,105 @@
+{
+  "uid": "quicknotes-golden",
+  "title": "QuickNotes — Golden Signals",
+  "tags": ["quicknotes", "sre"],
+  "timezone": "browser",
+  "schemaVersion": 39,
+  "version": 1,
+  "refresh": "10s",
+  "time": { "from": "now-30m", "to": "now" },
+  "panels": [
+    {
+      "id": 1,
+      "type": "timeseries",
+      "title": "Latency (proxy: scrape duration)",
+      "description": "QuickNotes exposes no request-duration histogram, so true per-request latency is unavailable. This panel shows Prometheus scrape duration for the target as the closest available proxy for how long the app takes to answer an HTTP request. See submissions/lab8.md for why this substitution was necessary.",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 0 },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "s",
+          "custom": { "lineWidth": 2, "fillOpacity": 10 }
+        },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "scrape_duration_seconds{job=\"quicknotes\"}",
+          "legendFormat": "scrape duration"
+        }
+      ]
+    },
+    {
+      "id": 2,
+      "type": "timeseries",
+      "title": "Traffic (requests/sec)",
+      "description": "rate() of the total HTTP request counter over a 5m window.",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 0 },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "reqps",
+          "custom": { "lineWidth": 2, "fillOpacity": 10 }
+        },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "rate(quicknotes_http_requests_total[5m])",
+          "legendFormat": "requests/sec"
+        }
+      ]
+    },
+    {
+      "id": 3,
+      "type": "timeseries",
+      "title": "Errors (4xx + 5xx ratio)",
+      "description": "Ratio of 4xx and 5xx responses to all responses. This is the expression the alert rule fires on.",
+      "gridPos": { "h": 8, "w": 12, "x": 0, "y": 8 },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "percentunit",
+          "min": 0,
+          "max": 1,
+          "custom": { "lineWidth": 2, "fillOpacity": 10 },
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              { "color": "green", "value": null },
+              { "color": "red", "value": 0.05 }
+            ]
+          }
+        },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "sum(rate(quicknotes_http_responses_by_code_total{code=~\"4..|5..\"}[5m])) / clamp_min(sum(rate(quicknotes_http_responses_by_code_total[5m])), 0.001)",
+          "legendFormat": "error ratio"
+        }
+      ]
+    },
+    {
+      "id": 4,
+      "type": "timeseries",
+      "title": "Saturation (notes stored)",
+      "description": "quicknotes_notes_total gauge — the store is in-process and file-backed, so the number of stored notes is the closest available saturation signal.",
+      "gridPos": { "h": 8, "w": 12, "x": 12, "y": 8 },
+      "fieldConfig": {
+        "defaults": {
+          "unit": "short",
+          "custom": { "lineWidth": 2, "fillOpacity": 10 }
+        },
+        "overrides": []
+      },
+      "targets": [
+        {
+          "refId": "A",
+          "expr": "quicknotes_notes_total",
+          "legendFormat": "notes stored"
+        }
+      ]
+    }
+  ]
+}

monitoring/grafana/provisioning/dashboards/dashboard.yml

@@ -0,0 +1,11 @@
+apiVersion: 1
+
+providers:
+  - name: golden-signals
+    orgId: 1
+    folder: ''
+    type: file
+    disableDeletion: false
+    updateIntervalSeconds: 10
+    options:
+      path: /var/lib/grafana/dashboards

monitoring/grafana/provisioning/datasources/datasource.yml

@@ -0,0 +1,8 @@
+apiVersion: 1
+
+datasources:
+  - name: Prometheus
+    type: prometheus
+    access: proxy
+    url: http://prometheus:9090
+    isDefault: true

monitoring/prometheus/alerts.yml

@@ -0,0 +1,16 @@
+groups:
+  - name: quicknotes
+    rules:
+      - alert: HighErrorRate
+        expr: |
+          sum(rate(quicknotes_http_responses_by_code_total{code=~"4..|5.."}[5m]))
+          /
+          clamp_min(sum(rate(quicknotes_http_responses_by_code_total[5m])), 0.001)
+          > 0.05
+        for: 5m
+        labels:
+          severity: page
+        annotations:
+          summary: "QuickNotes error ratio above 5% for 5 minutes"
+          description: "Error ratio is {{ $value | humanizePercentage }} over the last 5 minutes."
+          runbook_url: "https://github.com/HNS2112/DevOps-Intro/blob/feature/lab8/docs/runbook/high-error-rate.md"

monitoring/prometheus/prometheus.yml

@@ -0,0 +1,10 @@
+global:
+  scrape_interval: 15s
+
+rule_files:
+  - /etc/prometheus/alerts.yml
+
+scrape_configs:
+  - job_name: quicknotes
+    static_configs:
+      - targets: ['quicknotes:8080']