From 75ee30b82c346ced692c2575c8b1f7c62d0cdcb0 Mon Sep 17 00:00:00 2001 From: ancplua Date: Mon, 18 May 2026 07:39:23 +0200 Subject: [PATCH 1/2] feat(scripts): semantic config-drift detector + weekly CI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds scripts/drift_check.py which audits a watchlist of shared config files across all framework repos and reports semantic drift (JSON key-order, YAML flow style, .gitignore line order, XML attribute order all collapse to one equivalence class). Watchlist + repo list live in scripts/drift-policy.yaml (16 repos, 33 paths today; extend as repos and files are added). .github/workflows/drift-check.yml runs the detector every Monday 06:00 UTC and opens (or updates) a config-drift labelled issue with the report when drift is found. Verified locally: 8 of 33 paths fully clean. The 21 drifted paths include legitimate per-repo variation (anti-self-bump renovate.json rules, .coderabbit.yaml policies) which still surface for human review. Notable: renovate.json had 16 byte-unique versions but only 5 semantic clusters — 12 of 16 collapse to identical content after JSON normalisation. --- .github/workflows/drift-check.yml | 73 ++++++ README.md | 65 +++++ scripts/drift-policy.yaml | 98 ++++++++ scripts/drift_check.py | 386 ++++++++++++++++++++++++++++++ 4 files changed, 622 insertions(+) create mode 100644 .github/workflows/drift-check.yml create mode 100644 scripts/drift-policy.yaml create mode 100644 scripts/drift_check.py diff --git a/.github/workflows/drift-check.yml b/.github/workflows/drift-check.yml new file mode 100644 index 0000000..669ab11 --- /dev/null +++ b/.github/workflows/drift-check.yml @@ -0,0 +1,73 @@ +name: Config drift check + +on: + schedule: + # Weekly Monday 06:00 UTC. Matches the existing weekly hygiene cadence. + - cron: '0 6 * * 1' + workflow_dispatch: + inputs: + policy: + description: 'Policy file path' + type: string + default: 'scripts/drift-policy.yaml' + +permissions: + contents: read + issues: write + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: '3.x' + + - name: Install PyYAML + run: pip install pyyaml + + - name: Run drift detector + id: drift + env: + GH_TOKEN: ${{ secrets.DRIFT_CHECK_TOKEN || secrets.GITHUB_TOKEN }} + run: | + # The script reads gh CLI which is preinstalled on ubuntu-latest. + # GH_TOKEN above gives it auth. + python scripts/drift_check.py \ + --policy "${{ inputs.policy || 'scripts/drift-policy.yaml' }}" \ + --output drift-report.md \ + --manifest drift-manifest.json || echo "DRIFT_DETECTED=1" >> $GITHUB_ENV + echo "----- report start -----" + cat drift-report.md + echo "----- report end -----" + + - name: Upload report + if: always() + uses: actions/upload-artifact@v4 + with: + name: drift-report + path: | + drift-report.md + drift-manifest.json + retention-days: 90 + + - name: Open issue on drift + if: env.DRIFT_DETECTED == '1' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + # Find existing open drift issue to update rather than spam new ones + existing=$(gh issue list --repo "$GITHUB_REPOSITORY" \ + --label config-drift --state open \ + --json number --jq '.[0].number') + if [ -n "$existing" ]; then + gh issue comment "$existing" --repo "$GITHUB_REPOSITORY" \ + --body-file drift-report.md + else + gh issue create --repo "$GITHUB_REPOSITORY" \ + --title "Config drift detected ($(date -I))" \ + --label config-drift \ + --body-file drift-report.md + fi diff --git a/README.md b/README.md index 352b71b..acbb2c0 100644 --- a/README.md +++ b/README.md @@ -153,3 +153,68 @@ carry every scope above. `renovate-config`, `ancplua-docs`, `dotcov`) is a separate coordinated move; sibling `O-ANcppLua/ANcpLua.OtelConventions.Api` already lives in the org. + + +## Config drift detector + +`scripts/drift_check.py` audits a watchlist of shared configuration files across all listed repositories and reports **semantic** drift — not byte-level. Two files with different whitespace, different JSON key order, or different YAML flow style collapse to the same equivalence class. + +## What "semantic" means per file type + +| File type | Normaliser | Ignores | +|---|---|---| +| `*.json`, `renovate.json`, `package.json`, `.markdownlint.json` | `json` | Whitespace, key order, trailing newlines | +| `*.yaml`, `*.yml`, `.coderabbit.yaml`, `dependabot.yml` | `yaml` | Whitespace, key order, flow vs block style | +| `*.xml`, `*.props`, `*.targets`, `nuget.config`, `Directory.Build.props` | `xml` | Insignificant whitespace, attribute order | +| `.editorconfig`, `.gitmodules`, `.npmrc`, `.globalconfig` | `ini` | Comments, section ordering, key ordering | +| `.gitignore`, `.dockerignore`, `.gitattributes`, `.markdownlintignore` | `lines` | Comments, blank lines, duplicate lines, ordering | +| `LICENSE`, `build.sh`, `build.cmd` | `raw` | Trailing whitespace | + +Override the auto-detected normaliser per entry in `scripts/drift-policy.yaml` with `normalizer: `. + +## Running locally + +```bash +cd +pip install pyyaml +python scripts/drift_check.py \ + --policy scripts/drift-policy.yaml \ + --output drift-report.md \ + --manifest drift-manifest.json +``` + +Exit code `0` if all paths have one semantic cluster, `1` if any drift found, `2` on configuration or auth error. + +Reads GitHub via the `gh` CLI; needs `gh auth status` to be authenticated. + +## CI + +`.github/workflows/drift-check.yml` runs the detector every Monday 06:00 UTC. On drift, it opens (or updates) an issue labelled `config-drift` with the report inline. Artefacts (`drift-report.md`, `drift-manifest.json`) are retained for 90 days. + +To trigger manually: Actions → Config drift check → Run workflow. + +## Adding a repo or a file path + +Edit `scripts/drift-policy.yaml`: + +```yaml +repos: + - ANcpLua/ # add here + +watch: + - path: # add here + # optional: normalizer: json +``` + +The detector auto-picks the right normaliser from the file name; only specify `normalizer:` if you want to override. + +## Interpreting the report + +The report groups each watched path into **semantic clusters**. The largest cluster is marked `**canonical** (majority)`; smaller clusters are `drift #N`. + +- **One cluster** → clean, all repos semantically equivalent for this file. +- **Two or more clusters** → drift. The smaller ones likely need to be aligned with the canonical (or the variation is legitimate and should be allowlisted — see below). + +### Legitimate variation + +Some drift is intentional (e.g. anti-self-bump rules in `renovate.json` referencing each repo's own package name). Currently you read the report and ignore those rows; future revision can add an allowlist mechanism if false-positives become a maintenance burden. diff --git a/scripts/drift-policy.yaml b/scripts/drift-policy.yaml new file mode 100644 index 0000000..1221c68 --- /dev/null +++ b/scripts/drift-policy.yaml @@ -0,0 +1,98 @@ +# drift-policy.yaml — Watchset for the semantic config-drift detector. +# +# Add repos to the `repos:` list and file paths to the `watch:` list. +# Normalisers are auto-detected from filename; override per entry with +# `normalizer: ` if needed. +# +# Available normalisers: +# json — parse + sort-keys + canonical dump (whitespace and key-order +# ignored) +# yaml — parse + sort-keys + canonical dump +# xml — parse + sort attributes + strip insignificant whitespace +# ini — section[key]=value pairs, comments stripped, sorted +# lines — line-based; comments and blank lines stripped, lines deduped +# and sorted (.gitignore, .dockerignore, .gitattributes etc.) +# raw — bytes as UTF-8 text, .strip() + +repos: + # ANcpLua framework + - ANcpLua/ANcpLua.NET.Sdk + - ANcpLua/ANcpLua.Roslyn.Utilities + - ANcpLua/ANcpLua.Analyzers + - ANcpLua/ANcpLua.Agents + - ANcpLua/ANcpLua.OpenTelemetry.SemanticConventions.Analyzers + + # OTel-adjacent + - O-ANcppLua/ANcpLua.OtelConventions.Api + - O-ANcppLua/Nuke.OpenTelemetry.Conventions + - ANcpLua/typespec-otel-semconv + + # Apps and libs + - ANcpLua/ErrorOrX + - ANcpLua/dotcov + - ANcpLua/TourPlanner + - ANcpLua/TourPlanner-Angular + - ANcpLua/Paperless + - ANcpLua/nhmw-digital-collection + + # Docs and meta + - ANcpLua/ancplua-claude-plugins + - ANcpLua/ancplua-docs + + # qyl — to enable later + # - O-ANcppLua/qyl + +watch: + # Renovate + dependency-bot configs + - path: renovate.json + - path: .github/dependabot.yml + + # Editor + linter configs + - path: .editorconfig + - path: .globalconfig + - path: .markdownlint.json + - path: .markdownlintignore + - path: .prettierrc + + # Reviewer/automation configs + - path: .coderabbit.yaml + - path: .codecov.yml + - path: codecov.yml + + # Repo hygiene + - path: .gitattributes + - path: .gitignore + - path: .dockerignore + - path: .gitmodules + + # .NET specific + - path: nuget.config + - path: Directory.Build.props + - path: Directory.Packages.props + - path: Directory.Build.targets + - path: Version.props + - path: global.json + + # Node + - path: .npmrc + - path: package.json + # legitimate to differ — each repo has its own deps; surface for awareness + # only. Drift is expected. + + # Shared workflows + - path: .github/workflows/auto-merge.yml + - path: .github/workflows/coderabbit-autofix.yml + - path: .github/workflows/claude-code-review.yml + - path: .github/workflows/coderabbit.yml + - path: .github/workflows/ci.yml + + # Build host + - path: build.sh + - path: build.cmd + - path: build.ps1 + + # License / agent docs (compared raw — surfaces accidental Mintlify-style + # default attribution leftovers) + - path: LICENSE + - path: CLAUDE.md + - path: AGENTS.md diff --git a/scripts/drift_check.py b/scripts/drift_check.py new file mode 100644 index 0000000..1d6b5b3 --- /dev/null +++ b/scripts/drift_check.py @@ -0,0 +1,386 @@ +#!/usr/bin/env python3 +"""drift_check.py — Semantic config-drift detector across multiple GitHub repos. + +For a watchlist of (file-path) pairs and a list of repos, fetches every version +of each file, normalises each per file type, clusters by *semantic* equivalence, +and reports drift. + +Drift is *semantic* not byte-level: JSON key-order and whitespace, YAML +flow-style, .gitignore line ordering, XML attribute order all collapse to the +same equivalence class. + +Usage: + drift_check.py --policy drift-policy.yaml \ + --output drift-report.md \ + --manifest drift-manifest.json + +Exit codes: + 0 no drift + 1 drift detected + 2 configuration or authentication error + +Auth: uses the `gh` CLI (must be `gh auth login`'d). Hits the contents API for +every (repo, path) pair, so cost is O(repos * paths) requests. +""" + +from __future__ import annotations + +import argparse +import base64 +import collections +import hashlib +import json +import subprocess +import sys +import xml.etree.ElementTree as ET +from pathlib import Path + +try: + import yaml +except ImportError: + print("error: PyYAML required (pip install pyyaml)", file=sys.stderr) + sys.exit(2) + + +# --------------------------------------------------------------------------- +# GitHub access via gh CLI +# --------------------------------------------------------------------------- + +def gh_api(endpoint: str) -> dict | None: + """Call `gh api`, return parsed JSON. None on 404. Exits on other errors.""" + res = subprocess.run( + ["gh", "api", endpoint], + capture_output=True, text=True + ) + if res.returncode != 0: + if "Not Found" in res.stderr or "404" in res.stderr: + return None + print(f"gh api {endpoint} failed:\n{res.stderr}", file=sys.stderr) + sys.exit(2) + try: + return json.loads(res.stdout) + except json.JSONDecodeError as e: + print(f"could not parse response for {endpoint}: {e}", file=sys.stderr) + sys.exit(2) + + +def fetch_file(repo: str, path: str) -> bytes | None: + """Fetch raw file bytes for repo/path. None if file missing.""" + # gh api accepts /-paths directly when passed as endpoint, no URL-encoding + # needed for the slash in `contents/`. + result = gh_api(f"repos/{repo}/contents/{path}") + if result is None: + return None + content = result.get("content") + if not content: + return None + return base64.b64decode(content) + + +# --------------------------------------------------------------------------- +# Normalisers — bytes → canonical-string +# --------------------------------------------------------------------------- + +def norm_raw(data: bytes) -> str: + """Bytes as UTF-8 string, stripped. Last-resort normaliser.""" + return data.decode("utf-8", errors="replace").strip() + + +def norm_lines_sorted(data: bytes) -> str: + """Ignore-style files (.gitignore, .dockerignore, .gitattributes, etc.): + strip blank lines and comments, dedupe, sort, join.""" + text = data.decode("utf-8", errors="replace") + lines = set() + for line in text.splitlines(): + s = line.strip() + if not s or s.startswith("#"): + continue + lines.add(s) + return "\n".join(sorted(lines)) + + +def norm_json(data: bytes) -> str: + """Parse JSON, dump in canonical sorted form.""" + obj = json.loads(data.decode("utf-8")) + return json.dumps(obj, sort_keys=True, indent=2, ensure_ascii=False) + + +def norm_yaml(data: bytes) -> str: + """Parse YAML, dump in canonical sorted form.""" + obj = yaml.safe_load(data.decode("utf-8")) + return yaml.dump(obj, sort_keys=True, default_flow_style=False, allow_unicode=True) + + +def _canonicalize_xml(elt: ET.Element) -> None: + """Strip insignificant whitespace from text/tail; sort attributes.""" + if elt.text and not elt.text.strip(): + elt.text = None + if elt.tail and not elt.tail.strip(): + elt.tail = None + # Sort attributes for deterministic output + if elt.attrib: + items = sorted(elt.attrib.items()) + elt.attrib.clear() + for k, v in items: + elt.attrib[k] = v + for child in elt: + _canonicalize_xml(child) + + +def norm_xml(data: bytes) -> str: + """Parse XML, normalise whitespace and attribute order, dump.""" + root = ET.fromstring(data.decode("utf-8")) + _canonicalize_xml(root) + return ET.tostring(root, encoding="unicode") + + +def norm_ini(data: bytes) -> str: + """INI-like files (.editorconfig, .gitmodules, .npmrc, .globalconfig): + parse sections + key=value pairs, sort, dump.""" + text = data.decode("utf-8", errors="replace") + entries: dict[str, str] = {} + section = "" + for line in text.splitlines(): + s = line.strip() + if not s or s.startswith("#") or s.startswith(";"): + continue + if s.startswith("[") and s.endswith("]"): + section = s[1:-1].strip() + continue + if "=" in s: + k, v = s.split("=", 1) + entries[f"{section}/{k.strip()}"] = v.strip() + return "\n".join(f"{k}={v}" for k, v in sorted(entries.items())) + + +NORMALISERS = { + "raw": norm_raw, + "lines": norm_lines_sorted, + "json": norm_json, + "yaml": norm_yaml, + "xml": norm_xml, + "ini": norm_ini, +} + + +# Default file-name → normaliser mapping. +DEFAULT_NORMALISERS = { + # JSON + "renovate.json": "json", + "package.json": "json", + ".markdownlint.json": "json", + "global.json": "json", + ".prettierrc": "json", + # YAML + ".coderabbit.yaml": "yaml", + ".codecov.yml": "yaml", + "codecov.yml": "yaml", + "dependabot.yml": "yaml", + # Line-based ignore files + ".gitattributes": "lines", + ".gitignore": "lines", + ".dockerignore": "lines", + ".markdownlintignore": "lines", + # INI-like + ".gitmodules": "ini", + ".editorconfig": "ini", + ".globalconfig": "ini", + ".npmrc": "ini", + # XML + "nuget.config": "xml", + "Directory.Build.props": "xml", + "Directory.Build.targets": "xml", + "Directory.Packages.props": "xml", + "Version.props": "xml", + # Raw + "LICENSE": "raw", + "build.sh": "raw", + "build.cmd": "raw", + "build.ps1": "raw", +} + +SUFFIX_NORMALISERS = { + ".json": "json", ".yaml": "yaml", ".yml": "yaml", + ".xml": "xml", ".props": "xml", ".targets": "xml", ".csproj": "xml", +} + + +def get_normaliser(path: str) -> str: + """Resolve normaliser key for a given file path.""" + name = path.rsplit("/", 1)[-1] + if name in DEFAULT_NORMALISERS: + return DEFAULT_NORMALISERS[name] + for suffix, norm in SUFFIX_NORMALISERS.items(): + if name.endswith(suffix): + return norm + return "raw" + + +# --------------------------------------------------------------------------- +# Clustering and reporting +# --------------------------------------------------------------------------- + +def hash_str(s: str) -> str: + """Stable short hash for clustering.""" + return hashlib.sha256(s.encode("utf-8")).hexdigest()[:12] + + +def cluster_versions(repo_versions: dict[str, str]) -> list[dict]: + """Group repos by normalised-content equivalence class. Largest first.""" + clusters: dict[str, list[str]] = collections.defaultdict(list) + for repo, normalised in repo_versions.items(): + clusters[hash_str(normalised)].append(repo) + return [ + {"hash": h, "size": len(repos), "repos": sorted(repos)} + for h, repos in sorted(clusters.items(), key=lambda kv: -len(kv[1])) + ] + + +def check_path(repos: list[str], entry: dict) -> dict: + """Audit one watched path across all repos.""" + path = entry["path"] + normaliser_name = entry.get("normalizer") or get_normaliser(path) + norm_func = NORMALISERS.get(normaliser_name) + if not norm_func: + return { + "path": path, + "normalizer": normaliser_name, + "error": f"unknown normaliser: {normaliser_name}", + } + + repo_versions: dict[str, str] = {} + repo_errors: dict[str, str] = {} + for repo in repos: + data = fetch_file(repo, path) + if data is None: + continue + try: + repo_versions[repo] = norm_func(data) + except Exception as e: + repo_errors[repo] = f"{type(e).__name__}: {e}" + + if not repo_versions: + return { + "path": path, + "normalizer": normaliser_name, + "present_in": 0, + "clusters": [], + } + + clusters = cluster_versions(repo_versions) + return { + "path": path, + "normalizer": normaliser_name, + "present_in": len(repo_versions), + "absent_in": [r for r in repos if r not in repo_versions and r not in repo_errors], + "clusters": clusters, + "errors": repo_errors, + "drift": len(clusters) > 1, + } + + +# --------------------------------------------------------------------------- +# Output renderers +# --------------------------------------------------------------------------- + +def write_markdown(findings: list[dict], path: str) -> int: + """Write markdown report. Return drift count.""" + lines: list[str] = ["# Config Drift Report\n"] + total = len(findings) + drift = [f for f in findings if f.get("drift")] + clean = [f for f in findings if not f.get("drift") and f.get("present_in", 0) > 0] + missing = [f for f in findings if f.get("present_in", 0) == 0] + + lines.append(f"- Paths checked: **{total}**") + lines.append(f"- Drifted: **{len(drift)}**") + lines.append(f"- Clean (semantic 1-cluster): **{len(clean)}**") + lines.append(f"- Not present in any repo: **{len(missing)}**\n") + + if drift: + lines.append("## Drifted paths\n") + for f in drift: + lines.append(f"### `{f['path']}` (normaliser: `{f['normalizer']}`)") + lines.append(f"_{f['present_in']} repos, {len(f['clusters'])} semantic clusters_\n") + for i, c in enumerate(f["clusters"]): + marker = "**canonical** (majority)" if i == 0 else f"drift #{i}" + short_repos = ", ".join(r.split("/")[-1] for r in c["repos"]) + lines.append(f"- {marker} `{c['hash']}` ({c['size']}× ): {short_repos}") + if f.get("errors"): + lines.append("\n_Normalisation errors:_") + for repo, err in f["errors"].items(): + lines.append(f" - `{repo.split('/')[-1]}`: {err}") + lines.append("") + + if clean: + lines.append("## Clean paths\n") + lines.append("| Path | Repos | Hash |") + lines.append("|---|---|---|") + for f in clean: + h = f["clusters"][0]["hash"] if f["clusters"] else "-" + lines.append(f"| `{f['path']}` | {f['present_in']} | `{h}` |") + lines.append("") + + if missing: + lines.append("## Watched but absent everywhere\n") + for f in missing: + lines.append(f"- `{f['path']}`") + lines.append("") + + Path(path).write_text("\n".join(lines) + "\n") + return len(drift) + + +def write_manifest(findings: list[dict], path: str) -> None: + """Machine-readable: which repos have which semantic cluster for which path.""" + Path(path).write_text(json.dumps({ + "version": 1, + "findings": findings, + }, indent=2)) + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + +def main() -> None: + p = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + p.add_argument("--policy", required=True, help="Policy YAML file") + p.add_argument("--output", default="drift-report.md", help="Markdown report path") + p.add_argument("--manifest", default="drift-manifest.json", help="JSON manifest path") + p.add_argument("--quiet", action="store_true") + args = p.parse_args() + + policy_path = Path(args.policy) + if not policy_path.exists(): + print(f"policy file not found: {args.policy}", file=sys.stderr) + sys.exit(2) + policy = yaml.safe_load(policy_path.read_text()) + + repos = policy.get("repos") or [] + watch = policy.get("watch") or [] + if not repos or not watch: + print("policy must define non-empty 'repos' and 'watch' lists", file=sys.stderr) + sys.exit(2) + + findings: list[dict] = [] + for entry in watch: + if isinstance(entry, str): + entry = {"path": entry} + if not args.quiet: + print(f"checking {entry['path']} across {len(repos)} repos...", file=sys.stderr) + findings.append(check_path(repos, entry)) + + drift_count = write_markdown(findings, args.output) + write_manifest(findings, args.manifest) + + if not args.quiet: + print(f"\n{len(findings)} paths checked across {len(repos)} repos", file=sys.stderr) + print(f"{drift_count} paths have drift", file=sys.stderr) + print(f"report → {args.output}", file=sys.stderr) + print(f"manifest → {args.manifest}", file=sys.stderr) + + sys.exit(1 if drift_count > 0 else 0) + + +if __name__ == "__main__": + main() From da3d395067a0c47a48c58a06ced49fd65c32ab48 Mon Sep 17 00:00:00 2001 From: ancplua Date: Mon, 18 May 2026 08:17:49 +0200 Subject: [PATCH 2/2] fix(drift): split lines normaliser into ordered vs set; default ordered MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Git's gitignore and gitattributes semantics give the LAST matching pattern precedence within one level. Sorting + set-dedupe (the previous "lines" normaliser) destroyed that order, so two semantically different files could hash to the same cluster and silently pass. This commit: - Adds norm_lines_set (was norm_lines_sorted) — sort + dedupe, valid only when the consumer treats the file as an unordered set. - Adds norm_lines_ordered — preserves order, collapses only adjacent duplicates. Required for .gitignore and .gitattributes. - Keeps "lines" as an alias that resolves to ordered (fail-safe default for any future entry). - Switches .gitignore, .gitattributes, .dockerignore, .markdownlintignore to lines_ordered in DEFAULT_NORMALISERS. Verified locally: .gitattributes now collapses to 1 cluster across all 5 repos that have it (semantically identical with comment / whitespace noise removed); .gitignore stays at 16/16 (genuine content drift); .dockerignore stays at 2/2 (genuine content drift). Credit: caught by parallel agent reviewing PR #1. --- scripts/drift_check.py | 58 +++++++++++++++++++++++++++++++++++------- 1 file changed, 49 insertions(+), 9 deletions(-) diff --git a/scripts/drift_check.py b/scripts/drift_check.py index 1d6b5b3..1be08ab 100644 --- a/scripts/drift_check.py +++ b/scripts/drift_check.py @@ -86,9 +86,16 @@ def norm_raw(data: bytes) -> str: return data.decode("utf-8", errors="replace").strip() -def norm_lines_sorted(data: bytes) -> str: - """Ignore-style files (.gitignore, .dockerignore, .gitattributes, etc.): - strip blank lines and comments, dedupe, sort, join.""" +def norm_lines_set(data: bytes) -> str: + """Line-based files where order is semantically irrelevant: strip blank + lines and comments, dedupe, sort. Safe for files like ``.npmignore``-style + pattern lists *only when* you have verified the consumer treats them as + an unordered set. + + NOT SAFE for ``.gitignore`` or ``.gitattributes`` — git semantics give the + LAST matching pattern precedence, so reordering can change behaviour. Use + :func:`norm_lines_ordered` for those. + """ text = data.decode("utf-8", errors="replace") lines = set() for line in text.splitlines(): @@ -99,6 +106,34 @@ def norm_lines_sorted(data: bytes) -> str: return "\n".join(sorted(lines)) +def norm_lines_ordered(data: bytes) -> str: + """Line-based files where order matters: strip blank lines and comments, + preserve order, collapse only adjacent duplicates. + + Required for ``.gitignore`` and ``.gitattributes``: + + * gitignore: within one precedence level, the last matching pattern + decides whether a path is ignored. + * gitattributes: when more than one pattern matches a path, the last + matching line overrides earlier ones per attribute. + + Sorting these files would let two semantically different versions hash to + the same cluster and silently pass a drift check. + """ + text = data.decode("utf-8", errors="replace") + out: list[str] = [] + prev: str | None = None + for line in text.splitlines(): + s = line.strip() + if not s or s.startswith("#"): + continue + if s == prev: + continue # collapse adjacent duplicates only + out.append(s) + prev = s + return "\n".join(out) + + def norm_json(data: bytes) -> str: """Parse JSON, dump in canonical sorted form.""" obj = json.loads(data.decode("utf-8")) @@ -155,7 +190,9 @@ def norm_ini(data: bytes) -> str: NORMALISERS = { "raw": norm_raw, - "lines": norm_lines_sorted, + "lines_set": norm_lines_set, # order-irrelevant (use with care) + "lines_ordered": norm_lines_ordered, # order-sensitive (.gitignore et al.) + "lines": norm_lines_ordered, # alias — fail safe: assume order matters "json": norm_json, "yaml": norm_yaml, "xml": norm_xml, @@ -176,11 +213,14 @@ def norm_ini(data: bytes) -> str: ".codecov.yml": "yaml", "codecov.yml": "yaml", "dependabot.yml": "yaml", - # Line-based ignore files - ".gitattributes": "lines", - ".gitignore": "lines", - ".dockerignore": "lines", - ".markdownlintignore": "lines", + # Line-based ignore files. All four use ordered normalisation because + # the consumers (git, docker, markdownlint) all assign per-line precedence + # where a later line can override an earlier one. Sorting would mask real + # behavioural drift behind identical normalised hashes. + ".gitattributes": "lines_ordered", + ".gitignore": "lines_ordered", + ".dockerignore": "lines_ordered", + ".markdownlintignore": "lines_ordered", # INI-like ".gitmodules": "ini", ".editorconfig": "ini",