Preserve aggregate coverage history with compressed source reports

.github/workflows/ci.yml

@@ -27,6 +27,15 @@ jobs:
       - run: python3 -m unittest discover -s tools -p release_site_test.py
       - run: python3 -m unittest discover -s tools -p release_notes_test.py
       - run: python3 -m unittest discover -s tools -p "*_test.py"
+      - uses: actions/setup-node@v6
+        with:
+          node-version: "22"
+      - name: Test retained coverage in Chromium
+        working-directory: tools/coverage_web
+        run: |
+          npm ci
+          npx playwright install --with-deps chromium
+          npm test
       - name: Verify configured documentation and generated links
         env:
           GH_TOKEN: ${{ github.token }}

.github/workflows/pages.yml

@@ -95,8 +95,10 @@ jobs:
         working-directory: source
         run: |
           set -euo pipefail
-          git fetch origin
-          if git show-ref --verify --quiet refs/remotes/origin/coverage-pages; then
+          retained_ref="$(git ls-remote --heads origin refs/heads/coverage-pages)"
+          if [[ -n "${retained_ref}" ]]; then
+            git fetch --no-tags --depth=1 origin \
+              +refs/heads/coverage-pages:refs/remotes/origin/coverage-pages
             git worktree add ../site -B coverage-pages origin/coverage-pages
           else
             git worktree add --detach ../site
@@ -149,6 +151,7 @@ jobs:
           RELEASE_TAG: ${{ steps.release.outputs.tag }}
         run: |
           set -euo pipefail
+          python3 source/tools/compact_site.py site
           git -C site config user.name 'github-actions[bot]'
           git -C site config user.email '41898282+github-actions[bot]@users.noreply.github.com'
           git -C site add --all
@@ -159,6 +162,7 @@ jobs:
           mkdir -p public
           rsync --archive --exclude='.git' site/ public/
           python3 source/tools/site_artwork.py source/docs/assets public
+          python3 source/tools/compact_site.py public
       - uses: actions/configure-pages@v6
       - uses: actions/upload-pages-artifact@v5
         with:
@@ -191,8 +195,10 @@ jobs:
         working-directory: source
         run: |
           set -euo pipefail
-          git fetch origin
-          if git show-ref --verify --quiet refs/remotes/origin/coverage-pages; then
+          retained_ref="$(git ls-remote --heads origin refs/heads/coverage-pages)"
+          if [[ -n "${retained_ref}" ]]; then
+            git fetch --no-tags --depth=1 origin \
+              +refs/heads/coverage-pages:refs/remotes/origin/coverage-pages
             git worktree add ../site -B coverage-pages origin/coverage-pages
           else
             git worktree add --detach ../site
@@ -300,16 +306,18 @@ jobs:
           gh api --paginate --slurp "repos/${GITHUB_REPOSITORY}/pulls?state=all&per_page=100" > pulls.json
           python3 source/tools/coverage_index.py history site/coverage pulls.json
           python3 source/tools/coverage_index.py regenerate site/coverage
+          python3 source/tools/compact_site.py site
           git -C site config user.name 'github-actions[bot]'
           git -C site config user.email '41898282+github-actions[bot]@users.noreply.github.com'
-          git -C site add coverage
+          git -C site add --all
           if ! git -C site diff --cached --quiet; then
             git -C site commit -m "coverage: publish reports and refresh PR metadata"
             git -C site push origin HEAD:coverage-pages
           fi
           mkdir -p public
           rsync --archive --exclude='.git' site/ public/
           python3 source/tools/site_artwork.py source/docs/assets public
+          python3 source/tools/compact_site.py public
       - uses: actions/configure-pages@v6
       - uses: actions/upload-pages-artifact@v5
         with:

CHANGELOG.md

@@ -9,6 +9,9 @@ versions follow [SemVer](https://semver.org/spec/v2.0.0.html).
 
 ### Changed
 
+- Preserve coverage aggregates indefinitely, compress report history, expire source HTML after
+  seven days, and measure complete publication size against a 250 MB advisory budget.
+
 - Allow serialized coverage backfill from a completed CI run without rerunning tests.
 
 - Publish successful coverage despite unrelated CI failures, and put post-merge before pre-merge

docs/infrastructure.md

@@ -38,16 +38,18 @@ The overview uses the same report table columns as the C++ repositories: Report,
 Completed, Commit, Workflow, Lines, Branches, and Functions. Coderef's Python generator computes
 rates from summed LCOV counters, links each immutable report and its LCOV/metadata downloads,
 and displays completion times in UTC. Zero or unavailable metric totals display `n/a`, including
-branch coverage when Rust instrumentation produces no branch measurements. Existing archives
-are read without modification. The overview shows one result per PR, ordered by merge time,
+branch coverage when Rust instrumentation produces no branch measurements. Aggregate measurements
+and original run metadata are preserved through storage compaction. The overview shows one result
+per PR, ordered by merge time,
 then open PRs. Pre-merge coverage is replaced only when a main report tests that PR's exact
 merge commit; unrelated main runs are omitted. Aggregation PR coverage is never attributed to
 its constituent PRs. PRs without pre-merge reports can still show post-merge coverage.
 `history.html` shows at most one result per phase for each PR, with post-merge first. Run creation time
 selects the newest run within each phase, with numeric attempt ordering for retries. Late
 pre-merge reports cannot displace post-merge results. Closed unmerged PRs are omitted from both
 views and reappear when reopened. All immutable run URLs remain available, including omitted
-runs and retries. A minimal `pull-requests.json` registry records merge SHAs for exact attribution.
+runs and retries; expired source views link to the permanent aggregate summary. A minimal
+`pull-requests.json` registry records merge SHAs for exact attribution.
 The coverage figures describe Rust, not the Python or TypeScript scripts.
 PR closure (including merge) and reopening trigger a metadata-only refresh using trusted `main`
 publisher code. This updates overview ordering and visibility without downloading artifacts or
@@ -101,3 +103,10 @@ Run `python3 -m unittest discover -s tools -p '*_test.py'`, `pre-commit run --al
 existing Cargo and extension checks appropriate to a change. Publishing orchestration is tested
 with mocked GitHub commands, including corrupt/missing assets, existing drafts, immutable retries,
 and failed remote verification. Live publication remains a tag-triggered external integration.
+
+## Published-site storage
+
+[Storage and retention](site-storage.md) describes permanent per-attempt summaries, compressed
+LLVM HTML, seven-day detail retention, original LCOV gzip downloads, and the 250 MB advisory.
+Both publication paths compact before committing and after deployment-only artwork, and fetch
+only a shallow publication snapshot. Neither Git history nor measured run identities are rewritten.

docs/site-storage.md

@@ -0,0 +1,77 @@
+# Published site storage
+
+Coverage, release documentation, and shared assets use one complete-site storage budget.
+`storage-report.json` records byte totals and file counts by top-level directory. Publication warns
+at **250 MB** so growth can be reviewed early. The final payload guard is **9 GB**, allowing some TAR
+packaging headroom below the uploader's absolute limit. GitHub's documented supported published-site
+limit remains [1 GB](https://docs.github.com/en/pages/getting-started-with-github-pages/github-pages-limits);
+the emergency guard is not a supported operating budget.
+
+This adapts [mbo PR #549](https://github.com/mboworks/mbo/pull/549), based on
+[xff PR #955](https://github.com/mboworks/xff/pull/955), merged as
+`91064c283fa90445db48ec598e4e052fb9d37261`.
+
+## Retention and representation
+
+- Coverage summaries, original run metadata,
+  and pre/post-merge attribution are retained indefinitely. No report-count limit evicts history.
+- Detailed source, function, and branch pages expire **seven days** after the report's original
+  completion timestamp. Replays do not renew that timestamp. Undated legacy reports have no
+  recent-detail window. Their summaries and metadata remain available.
+- Older LCOV-only directories without structured summaries or metadata are preserved intact.
+  They cannot enter the retention scheme until their aggregate data and identity can be recovered;
+  the migration does not invent either or discard the only historical report. Their bytes still
+  count toward the site budget.
+- Recent details use compressed manifests and shared source-page bases. Changed pages store edits
+  against one base; there are no dependency chains through earlier reports. LLVM named source
+  anchors are preserved for direct links and scrolling. Collection removes only detail archives
+  and bases without a retained reference.
+- Aggregate counters are captured from LCOV into a permanent per-attempt summary before any
+  source HTML expires. Original LCOV is retained byte-for-byte in `lcov.info.gz`; history reads
+  the permanent summary. Summary JSON is gzip-compressed without changing its original bytes.
+  Report landing pages keep their URLs and load compressed HTML through a shared loader. Old LLVM
+  HTML source URLs route through
+  the site's 404 document to the viewer, preserving report identity and line anchors. Expired
+  details explain the policy and link their aggregate report.
+- The browser uses built-in `DecompressionStream`; packed views require JavaScript. Raw
+  `coverage-summary.json.gz` and uncompressed `metadata.json` remain downloadable without it.
+- Release documentation, coverage thresholds, and measurement identities are not
+  changed by compaction.
+
+## Publication
+
+Both coverage and release-site publishers compact the retained `coverage-pages` tree **before
+committing**. Each then stages the complete site, installs deployment-only artwork, compacts again,
+and measures the final payload before upload. Every publisher uses the existing serialized queue.
+The existing attempt-specific `Rust coverage` gate remains authoritative.
+
+Publication checkouts fetch only the current retained snapshot. Both release and coverage
+restoration explicitly fetch only the shallow publication branch. Compaction does not rewrite
+Git history: earlier committed source details remain in old commits. Deployed bytes, Git object storage, Actions artifacts, and build
+caches are separate budgets.
+
+## Validation and future growth
+
+Python tests cover summary bytes, metadata, expiry boundaries, shared-base collection, missing
+bases, replay behavior, repeated compaction, complete-site size accounting, and both publishers.
+Chromium tests exercise packed summary tables, compressed JSON, source navigation, CSS, line
+anchors, old deep links, expired reports, and the download fallback without JavaScript:
+
+```sh
+python3 -m unittest discover -s tools -p '*_test.py'
+cd tools/coverage_web
+npm ci
+npx playwright install chromium
+npm test
+```
+
+The same browser suite runs in the required release-site CI job. To use an installed Chromium
+browser locally, set `CHROME_PATH` to its executable when running `npm test`.
+
+At 250 MB, inspect the section totals and consider shared data-driven summary tables and a more
+compact source representation. Preserve aggregate measurements and provenance. Moving historical
+data to other storage or rewriting publication Git history requires a separate decision.
+
+LLVM-generated JavaScript navigation controls are removed from packed source views; the viewer
+serves static source, coverage tables, CSS, page links, and line anchors in a sandboxed frame.
+Original report scripts are never executed. Runtime site code has no third-party dependencies.

release-site.json

@@ -15,7 +15,8 @@
     "docs/release-website.md": "docs/release-website.html",
     "GIT_RULES.md": "GIT_RULES.html",
     "STYLE_SH.md": "STYLE_SH.html",
-    "docs/infrastructure.md": "docs/infrastructure.html"
+    "docs/infrastructure.md": "docs/infrastructure.html",
+    "docs/site-storage.md": "docs/site-storage.html"
   },
   "links": [
     {

tools/compact_site.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python3
+# SPDX-FileCopyrightText: Copyright (c) M. Boerger, the MBO Works authors
+# SPDX-License-Identifier: Apache-2.0
+"""Compact the retained publication tree before committing or deploying it."""
+
+import argparse
+import datetime
+import json
+from pathlib import Path
+import shutil
+
+import coverage_storage
+import site_storage
+
+
+def sizes(root):
+    result = {}
+    for path in root.rglob('*'):
+        relative = path.relative_to(root)
+        if '.git' in relative.parts or relative.as_posix() == 'storage-report.json' or not path.is_file():
+            continue
+        section = relative.parts[0] if len(relative.parts) > 1 else 'root'
+        stats = result.setdefault(section, {'files': 0, 'bytes': 0})
+        stats['files'] += 1
+        stats['bytes'] += path.stat().st_size
+    return result
+
+
+def compact(root, now):
+    site_storage.prepare_summaries(root / 'coverage')
+    coverage_storage.compact(root / 'coverage', now)
+    site_storage.compact_summaries(root / 'coverage')
+    source = Path(__file__).parent
+    for name in ('site_storage', 'site_page'):
+        shutil.copyfile(source / (name + '.js'), root / (name.replace('_', '-') + '.js'))
+    if (root / 'coverage').is_dir():
+        shutil.copyfile(source / 'coverage_viewer.js', root / 'coverage/viewer.js')
+        (root / '404.html').write_text(coverage_storage.legacy_router())
+    result = sizes(root)
+    total = sum(entry['bytes'] for entry in result.values())
+    report = {'schema': 1, 'total_bytes': total, 'sections': result,
+              'coverage_detail_days': 7, 'review_bytes': 250_000_000}
+    (root / 'storage-report.json').write_text(json.dumps(report, indent=2) + '\n')
+    print(json.dumps(report, indent=2), flush=True)
+    if total >= report['review_bytes']:
+        print('::warning::Published site reached 250 MB; review storage-report.json and docs/site-storage.md.')
+    if total > 9_000_000_000:
+        raise ValueError('Published site exceeds the Pages deployment ceiling')
+    return report
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument('root', type=Path)
+    args = parser.parse_args()
+    compact(args.root, datetime.datetime.now(datetime.timezone.utc))
+
+
+if __name__ == '__main__':
+    main()

tools/compact_site_test.py

@@ -0,0 +1,71 @@
+# SPDX-FileCopyrightText: Copyright (c) M. Boerger, the MBO Works authors
+# SPDX-License-Identifier: Apache-2.0
+"""Both publishers compact history and measure the complete deployment."""
+
+import contextlib
+import datetime
+import io
+import json
+from pathlib import Path
+import tempfile
+import unittest
+from unittest import mock
+
+import compact_site
+
+
+class CompactSiteTest(unittest.TestCase):
+    def test_counts_entire_payload_without_changing_release_or_benchmark_data(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            retained = {'site/tag/1.2.3/index.html': b'release documentation',
+                        'benchmarks/report.json': b'{"observations":[1,2,3]}',
+                        'coverage/pr/1/index.html': b'legacy aggregate report',
+                        'coverage/pr/1/source.cc.gcov.html': b'legacy source without metadata'}
+            for name, data in {**retained, '.git/objects/private': b'not deployed'}.items():
+                path = root / name
+                path.parent.mkdir(parents=True, exist_ok=True)
+                path.write_bytes(data)
+            with contextlib.redirect_stdout(io.StringIO()):
+                report = compact_site.compact(root, datetime.datetime.now(datetime.timezone.utc))
+            self.assertEqual(report['sections']['site'], {'files': 1, 'bytes': 21})
+            self.assertNotIn('.git', report['sections'])
+            self.assertEqual(report['total_bytes'], sum(p.stat().st_size for p in root.rglob('*')
+                             if p.is_file() and '.git' not in p.parts and p.name != 'storage-report.json'))
+            self.assertEqual(json.loads((root / 'storage-report.json').read_text()), report)
+            for name, data in retained.items():
+                self.assertEqual((root / name).read_bytes(), data)
+            self.assertFalse((root / 'coverage/pr/1/coverage-meta.json').exists())
+
+    def test_advisory_budget_and_packaging_ceiling(self):
+        with tempfile.TemporaryDirectory() as directory:
+            root = Path(directory)
+            now = datetime.datetime.now(datetime.timezone.utc)
+            for size, warns, fails in ((249_999_999, False, False), (250_000_000, True, False),
+                                       (9_000_000_000, True, False), (9_000_000_001, True, True)):
+                with self.subTest(size=size), mock.patch.object(
+                        compact_site, 'sizes', return_value={'site': {'files': 1, 'bytes': size}}):
+                    output = io.StringIO()
+                    with contextlib.redirect_stdout(output):
+                        if fails:
+                            with self.assertRaisesRegex(ValueError, 'deployment ceiling'):
+                                compact_site.compact(root, now)
+                        else:
+                            compact_site.compact(root, now)
+                    self.assertEqual('::warning::' in output.getvalue(), warns)
+
+    def test_publishers_compact_before_commit_and_after_deployment_artwork(self):
+        repository = Path(__file__).resolve().parent.parent
+        workflow = (repository / '.github/workflows/pages.yml').read_text()
+        for source in workflow.split('\n  coverage:\n'):
+            self.assertLess(source.index('compact_site.py site'), source.index('git -C site commit'))
+            self.assertLess(source.index('site_artwork.py'), source.index('compact_site.py public'))
+            self.assertLess(source.index('compact_site.py public'), source.index('uses: actions/upload-pages-artifact@'))
+            self.assertIn('git -C site add --all', source)
+            self.assertIn('git fetch --no-tags --depth=1 origin', source)
+            self.assertNotIn('git fetch origin\n', source)
+
+
+
+if __name__ == '__main__':
+    unittest.main()