Land SCF_plus.dta in sources/, and gate sources/ on its recorded hashes (PR B2)

.github/scripts/check_consumed_files.py

@@ -25,17 +25,33 @@
 Files with no manifest yet (Phase 6 backfill pending), and manifests that
 record no hash, are out of scope here — the full validation suite (schema,
 dtypes, invariants) is PLAN Phase 5 and will subsume this check.
+
+Separately, for every file in sources/ (builder inputs, never served, no
+manifest — sources/README.md is their audit trail):
+
+  - the LFS rule must actually capture it
+  - sources/README.md must record its sha256, under a `## <filename>` heading
+  - the committed bytes must hash to that value
+
+Same principle as the manifest gate above, keyed on the README instead: hash
+whenever a hash is recorded. Without it, sources/ would carry no validation of
+any kind while every file in lectures/ is validated as it migrates — and
+SCF_plus.dta is the provenance root for two published datasets, so if its bytes
+drifted both would become unreproducible and nothing would notice.
 """
 from __future__ import annotations
 
 import hashlib
 import pathlib
+import re
+import subprocess
 import sys
 
 import yaml
 
 REPO = pathlib.Path(__file__).resolve().parents[2]
 LECTURES = REPO / "lectures"
+SOURCES = REPO / "sources"
 
 # One builder per published dataset, in builders/ (AGENTS.md, "Builders").
 # `committed` asserts a runnable four-stage builder; `committed-frozen` says the
@@ -45,6 +61,29 @@
 BUILDER_STATUSES = {"committed", "committed-frozen", "unrecovered", "not-applicable"}
 
 
+SHA256_RE = re.compile(r"\b([0-9a-f]{64})\b")
+# A `sources/` entry is a `## <filename>` section. Requiring the heading to look
+# like a filename is what keeps a prose section from being read as one: the
+# README has several, and a sha256 quoted in an example inside any of them would
+# otherwise register as a recorded file and fail the no-such-file check below.
+SOURCE_HEADING_RE = re.compile(r"^[\w.\-]+\.\w+$")
+# An LFS pointer is <200 bytes of text whose second line is `oid sha256:<hex>`.
+# That oid IS the object's sha256, which is what makes this check work under
+# `lfs: false` — the real bytes are never fetched and never need to be.
+#
+# Line endings are tolerated rather than pinned. `.gitattributes` sets `-text` on
+# sources/** so git does no conversion, and CI is ubuntu — but if a pointer ever
+# did pick up a CR or a stray trailing newline, a stricter pattern would fall
+# through to hashing the pointer text and report "committed bytes do not match"
+# for an object that is perfectly correct. Still far too tight for any real
+# data file to match by accident.
+LFS_POINTER_RE = re.compile(
+    rb"\Aversion https://git-lfs\.github\.com/spec/v1\r?\n"
+    rb"oid sha256:([0-9a-f]{64})\r?\n"
+    rb"size (\d+)\s*\Z"
+)
+
+
 def sha256(path: pathlib.Path) -> str:
     h = hashlib.sha256()
     with open(path, "rb") as f:
@@ -60,7 +99,9 @@ def main() -> int:
 
     for manifest_path in manifests:
         try:
-            manifest = yaml.safe_load(manifest_path.read_text())
+            # Explicit encoding: manifests carry em-dashes and non-ASCII source
+            # names, and read_text() otherwise decodes with the platform locale.
+            manifest = yaml.safe_load(manifest_path.read_text(encoding="utf-8"))
         except yaml.YAMLError as exc:
             errors.append(f"{manifest_path.name}: invalid YAML — {exc}")
             continue
@@ -176,6 +217,8 @@ def main() -> int:
                 f"vintages')"
             )
 
+    checked += check_sources(errors)
+
     for e in errors:
         print(f"::error::{e}")
     print(
@@ -185,5 +228,122 @@ def main() -> int:
     return 1 if errors else 0
 
 
+def lfs_tracked(path: pathlib.Path) -> tuple[bool, str | None]:
+    """Whether the LFS rule captures `path`, per .gitattributes.
+
+    Returns (tracked, error). A git failure must NOT come back as `False`: an
+    empty stdout would then read as "not captured by the LFS rule", which is
+    the one catastrophe this check exists to report. A broken environment
+    reporting the disaster it is meant to detect is worse than no check, so it
+    gets its own message.
+    """
+    rel = path.relative_to(REPO).as_posix()
+    try:
+        out = subprocess.run(
+            ["git", "check-attr", "filter", "--", rel],
+            cwd=REPO, capture_output=True, text=True,
+        )
+    except OSError as exc:                       # git absent entirely
+        return False, f"could not run `git check-attr` ({exc})"
+    if out.returncode != 0:
+        detail = out.stderr.strip().splitlines()
+        return False, (
+            f"`git check-attr` exited {out.returncode}"
+            + (f" — {detail[0]}" if detail else "")
+        )
+    return out.stdout.strip().endswith(": lfs"), None
+
+
+def check_sources(errors: list[str]) -> int:
+    """Verify sources/ against the sha256 values in sources/README.md.
+
+    Two assertions per file. The LFS one is not ceremony: SCF_plus.dta sits
+    923,507 B (0.88%) under GitHub's hard blob limit, so a mis-scoped
+    .gitattributes does not error — the push succeeds as plain git and the blob
+    is in history permanently. That failure is silent in the one direction that
+    cannot be undone, so it is worth a check rather than a convention.
+    """
+    if not SOURCES.is_dir():
+        return 0
+
+    readme = SOURCES / "README.md"
+    if not readme.exists():
+        errors.append(
+            "sources/README.md is missing — it is the audit trail for a "
+            "directory whose files carry no manifest (AGENTS.md, 'LFS, and "
+            "sources/ vs lectures/')"
+        )
+        return 0
+
+    # `## <filename>` starts a section; the first 64-hex token inside it is that
+    # file's recorded sha256. Parsed by section rather than by table cell so the
+    # README stays free to change its formatting. Headings that are not
+    # filename-shaped are prose and are skipped — see SOURCE_HEADING_RE.
+    sections = re.split(r"^## +", readme.read_text(encoding="utf-8"), flags=re.M)[1:]
+    recorded: dict[str, str] = {}
+    for sec in sections:
+        name = sec.splitlines()[0].strip().strip("`")
+        if not SOURCE_HEADING_RE.match(name):
+            continue
+        if m := SHA256_RE.search(sec):
+            recorded[name] = m.group(1)
+
+    files = sorted(p for p in SOURCES.iterdir() if p.is_file() and p.name != "README.md")
+    checked = 0
+
+    for path in files:
+        tracked, git_error = lfs_tracked(path)
+        if git_error:
+            errors.append(
+                f"sources/{path.name}: {git_error}. This is a tooling failure, "
+                f"not a finding about the file — the LFS assertion could not be "
+                f"evaluated either way"
+            )
+        elif not tracked:
+            errors.append(
+                f"sources/{path.name}: not captured by the LFS rule — "
+                f"`git check-attr filter` does not say `lfs`. Everything under "
+                f"sources/ must be LFS-tracked; a mis-scoped rule commits the "
+                f"real bytes as plain git and does not error below 100 MiB"
+            )
+
+        want = recorded.get(path.name)
+        if not want:
+            errors.append(
+                f"sources/{path.name}: no sha256 recorded in sources/README.md "
+                f"— add a `## {path.name}` section with its origin, retrieval, "
+                f"licence, sha256 and consuming builder"
+            )
+            continue
+
+        # Under `lfs: false` the working file IS the pointer, and the pointer's
+        # oid is the object's sha256 — so this verifies the real bytes without
+        # fetching ~100 MiB of them. With LFS smudge on locally it is the real
+        # file, and hashing it gives the same answer.
+        blob = path.read_bytes() if path.stat().st_size < 1024 else None
+        if blob is not None and (m := LFS_POINTER_RE.match(blob)):
+            actual, kind = m.group(1).decode(), "LFS pointer oid"
+        else:
+            actual, kind = sha256(path), "committed bytes"
+
+        checked += 1
+        if actual != want:
+            errors.append(
+                f"sources/{path.name}: {kind} {actual} does not match the "
+                f"sha256 recorded in sources/README.md ({want}). This file is "
+                f"a builder input, so a drift here makes its outputs "
+                f"unreproducible — update the README in the same PR if the "
+                f"change is deliberate"
+            )
+
+    for name in sorted(set(recorded) - {p.name for p in files}):
+        errors.append(
+            f"sources/README.md records `{name}`, which is not in sources/ — "
+            f"a stale audit-trail entry is worse than none"
+        )
+
+    return checked
+
+
 if __name__ == "__main__":
     sys.exit(main())

.github/workflows/consumed-file-check.yml

@@ -21,8 +21,16 @@ jobs:
           # goes red — which is exactly what a reader would get from
           # raw.githubusercontent.com. Fetching the real bytes would hash them
           # correctly and pass green while every consumer downloads ~130 bytes
-          # of pointer text. LFS belongs to `sources/**` only (.gitattributes),
-          # and nothing here reads `sources/`.
+          # of pointer text. LFS belongs to `sources/**` only (.gitattributes).
+          #
+          # This job DOES read `sources/` now, and `lfs: false` is right there
+          # too, for the opposite reason: those files are meant to be LFS, so
+          # the checkout hands us pointers — and a pointer's `oid` IS the
+          # object's sha256, which is the value sources/README.md records. The
+          # check verifies ~100 MiB of bytes without fetching one of them, and
+          # the direction still holds: a `sources/` file committed as plain git
+          # by mistake arrives as real bytes, hashes to something the README
+          # does not record, and goes red.
           lfs: false
       - uses: actions/setup-python@v5
         with:

AGENTS.md

@@ -84,15 +84,15 @@ The failure modes are silent in both directions, which is why all three are mach
 LFS exists here for one purpose: **upstream inputs that builders consume and no lecture reads**, which live in `sources/` and are never served.
 
 - `lectures/<file>` — a published dataset. Plain git, sidecar manifest required, its filename is an API.
-- `sources/<file>` — a builder input. Per-path LFS, **no** manifest, not served, recorded instead in `sources/README.md` — the audit trail: origin, retrieval date, licence, upstream identifier (DOI where one exists), `sha256`, and the builder that consumes it.
+- `sources/<file>` — a builder input. Per-path LFS, **no** manifest, not served, recorded instead in `sources/README.md` — the audit trail: origin, retrieval date, licence, upstream identifier (DOI where one exists), `sha256`, and the builder that consumes it. **CI enforces the last of those**: `check_consumed_files.py` requires every file here to be captured by the LFS rule and to hash to a `sha256` recorded under a `## <filename>` heading in that README, and fails on a README entry with no file. It reads the pointer's `oid` rather than the object, so it costs no LFS bandwidth.
 
 Rules that still apply:
 
 - LFS is **per-path**, opt-in, large binaries only. Never a blanket rule like `high_dim_data`'s `*.csv` **and** `*.dta`.
 - Do not LFS-track an **existing** file until you've confirmed no consumer fetches it via `raw.githubusercontent.com` — converting silently turns their download into pointer text.
 - A builder must read its input from `sources/`, never over the network from another QuantEcon repo. That is how a retired repo becomes load-bearing again.
 - **Two** workflows check this repo out, and both now say `lfs: false` — `.github/workflows/audit-dashboard.yml` (the Pages deploy) and `.github/workflows/consumed-file-check.yml` (every pull request). Leave them that way: `lfs: false` is the assertion that nothing published is an LFS object. If a `lectures/` file is ever tracked by mistake, the checker hashes the pointer and goes red, and Pages deploys the same pointer bytes a reader would get from `raw.githubusercontent.com` — whereas `lfs: true` fetches the real bytes, passes green, and publishes a file that works only from Pages. It also keeps `sources/` (a 99 MiB LFS object) off every run; LFS bandwidth is an org-wide quota.
-- `git check-attr filter -- sources/<file>` must print `filter: lfs` **before** you `git add` anything to `sources/`. `SCF_plus.dta` is 103,934,093 B against GitHub's 104,857,600 B hard limit, so a mis-scoped rule does not error — the push succeeds as plain git and the blob is in history permanently.
+- `git check-attr filter -- sources/<file>` must print `filter: lfs` **before** you `git add` anything to `sources/`. `SCF_plus.dta` is 103,934,093 B against GitHub's 104,857,600 B hard limit, so a mis-scoped rule does not error — the push succeeds as plain git and the blob is in history permanently. CI asserts this too, but only after the fact: by the time a PR goes red the blob is already in the branch's history, so run it yourself first.
 
 #### When a published file approaches the 100 MiB blob limit
 

builders/README.md

@@ -40,9 +40,16 @@ re-fetched** — see `AGENTS.md`.
 | `japan_population_by_age.py` | `japan_population_by_age.csv` | committed |
 | `us_adult_heights.py` | `us_adult_heights.csv` | committed |
 | `business_cycle.py` | `business_cycle_data.csv`, `business_cycle_info.md`, `business_cycle_metadata.md` | run by hand, no validate stage yet (PLAN Phase 5); its three outputs are the repo's only unmanifested files |
+| `webscrape_forbes.ipynb` | `forbes-global2000.csv`, `forbes-billionaires.csv` | **committed-frozen** — an undocumented Forbes API, a spoofed user-agent and hardcoded GDPR consent cookies. Defects recorded in the two manifests rather than fixed |
+| `generating_mini.md` | `SCF_plus_mini.csv`, `SCF_plus_mini_no_weights.csv` | **committed-frozen** — its `to_csv` calls are commented out upstream and stay that way. As written it still fetches the `high_dim_data` URL; that URL is historical, and the input is now committed at `sources/SCF_plus.dta`. See `sources/README.md` |
 
-**This listing is the coverage report.** The repo has 13 `constructed` datasets
-and 7 builders; the difference is the Phase 9 recovery backlog, carried as
+Both frozen builders keep their upstream `high_dim_data` filenames rather than
+being renamed to their set stems (`forbes`, `SCF_plus_mini`), which preserves
+the textual link to that repo's history. Permitted by the rule above — what CI
+asserts is that the path exists.
+
+**This listing is the coverage report.** The repo has 17 `constructed` datasets
+and 9 builders; the difference is the Phase 9 recovery backlog, carried as
 `builder_status: unrecovered` in each manifest rather than hidden by
 reclassifying the file as `verbatim`.