Enforce endDate >= startDate and serialize Pages deployments

.github/scripts/test_materialize_dates.py

@@ -0,0 +1,126 @@
+#!/usr/bin/env python3
+"""Integration tests for materialize.py's event date-order check. Runs the
+real script via uv in a temp directory, so it needs uv on PATH (present in CI
+via astral-sh/setup-uv). Run directly:
+python3 .github/scripts/test_materialize_dates.py"""
+import json
+import os
+import pathlib
+import shutil
+import subprocess
+import tempfile
+import unittest
+
+REPO_ROOT = pathlib.Path(__file__).resolve().parents[2]
+MATERIALIZE = REPO_ROOT / "tools" / "materialize.py"
+
+
+def make_series_events(events):
+    """Build a schema-valid series dict from (id, startDate, endDate) tuples.
+
+    Events are listed newest first, matching the layout of the repo's series
+    files."""
+    return {
+        "name": "Testcon",
+        "events": [
+            {
+                "id": event_id,
+                "name": f"Testcon ({event_id})",
+                "url": "https://example.com",
+                "startDate": start_date,
+                "endDate": end_date,
+                "venue": "Test Hall",
+                "locale": "en-US",
+            }
+            for event_id, start_date, end_date in events
+        ],
+    }
+
+
+def make_series(start_date, end_date):
+    """Build a one-event series with the given start and end dates."""
+    return make_series_events([("testcon-2027", start_date, end_date)])
+
+
+def run_materialize(series):
+    """Write ``series`` as a fixture and run materialize.py against it via uv.
+
+    Returns the CompletedProcess. materialize only globs *.json in its cwd, so
+    out/ can live alongside the fixture without being picked up as a series
+    file."""
+    with tempfile.TemporaryDirectory() as data_dir:
+        with open(os.path.join(data_dir, "testcon.json"), "w") as f:
+            json.dump(series, f)
+        out_dir = os.path.join(data_dir, "out")
+        os.mkdir(out_dir)
+        return subprocess.run(
+            ["uv", "run", "--script", str(MATERIALIZE), out_dir],
+            cwd=data_dir,
+            capture_output=True,
+            text=True,
+        )
+
+
+@unittest.skipUnless(
+    shutil.which("uv"), "these tests run materialize.py via uv, which is not on PATH"
+)
+class TestDateOrder(unittest.TestCase):
+    """End-to-end checks of the endDate >= startDate rule in materialize.py."""
+
+    def test_end_after_start_passes(self):
+        """A multi-day event with endDate after startDate validates."""
+        result = run_materialize(make_series("2027-04-02", "2027-04-04"))
+        self.assertEqual(result.returncode, 0, result.stderr)
+
+    def test_single_day_event_passes(self):
+        """A single-day event (endDate == startDate) validates."""
+        result = run_materialize(make_series("2027-04-02", "2027-04-02"))
+        self.assertEqual(result.returncode, 0, result.stderr)
+
+    def test_end_before_start_fails(self):
+        """An event whose endDate precedes startDate fails with a clear message.
+
+        Regression: ainmhicon-2027 was submitted with endDate 2026-04-04
+        against startDate 2027-04-02 and passed validation, because the
+        schema's formatMinimum/$data keyword is an ajv extension that
+        python-jsonschema ignores."""
+        result = run_materialize(make_series("2027-04-02", "2026-04-04"))
+        self.assertEqual(result.returncode, 1, result.stderr)
+        self.assertIn("endDate 2026-04-04 is before startDate 2027-04-02", result.stderr)
+
+    def test_later_event_in_series_fails(self):
+        """The check runs for every event in a series, not just the first.
+
+        Series files list events newest first, so an older event is the
+        likely place for a bad date to hide."""
+        result = run_materialize(
+            make_series_events(
+                [
+                    ("testcon-2027", "2027-04-02", "2027-04-04"),
+                    ("testcon-2026", "2026-04-03", "2026-04-01"),
+                ]
+            )
+        )
+        self.assertEqual(result.returncode, 1, result.stderr)
+        self.assertIn("testcon-2026", result.stderr)
+        self.assertIn(
+            "endDate 2026-04-01 is before startDate 2026-04-03", result.stderr
+        )
+
+    def test_schema_invalid_file_still_fails(self):
+        """A schema-invalid file is reported and skipped, not crashed on.
+
+        Regression for the has_errors fix: without it the file falls through
+        into the event loop and exits 1 via an uncaught KeyError, so the
+        traceback assertion is what discriminates."""
+        series = make_series("2027-04-02", "2027-04-04")
+        del series["events"][0]["locale"]
+        result = run_materialize(series)
+        self.assertEqual(result.returncode, 1, result.stderr)
+        self.assertIn("required property", result.stderr)
+        self.assertIn("locale", result.stderr)
+        self.assertNotIn("Traceback", result.stderr)
+
+
+if __name__ == "__main__":
+    unittest.main()

.github/workflows/deploy.yml

@@ -15,6 +15,15 @@ on:
 
   workflow_dispatch:
 
+# One deployment at a time: Pages rejects a deployment while another is in
+# progress, so back-to-back merges raced and failed. Runs queued between the
+# in-progress one and the newest are skipped (each run deploys full HEAD
+# anyway); in-progress deployments are never cancelled. Same pattern as
+# github.com/actions/starter-workflows/blob/main/pages/static.yml.
+concurrency:
+  group: pages
+  cancel-in-progress: false
+
 permissions:
   contents: read
   pages: write

.github/workflows/import_fancons.yml

@@ -24,6 +24,29 @@ jobs:
           GOOGLE_MAPS_API_KEY: ${{ secrets.GOOGLE_MAPS_API_KEY }}
           FANCONS_CALENDAR_URL: ${{ secrets.FANCONS_CALENDAR_URL }}
           FANCONS_MAP_URL: ${{ secrets.FANCONS_MAP_URL }}
+
+      # Validate before committing so a bad import fails the run instead of landing on main; see import_concat.yml.
+      - name: Canonical formatting
+        id: format
+        run: uv run tools/format.py *.json
+      - name: Prepare the materialize output directory
+        run: mkdir -p "$RUNNER_TEMP/out"
+      - name: Schema validation (materialize)
+        id: validate
+        run: uv run tools/materialize.py "$RUNNER_TEMP/out"
+
+      # Separate messages because a failing format step skips validate, so
+      # naming the wrong one sends whoever reads it to the wrong file.
+      - name: Explain a formatting failure
+        if: failure() && steps.format.outcome == 'failure'
+        run: |
+          echo "::error::tools/format.py failed on the imported data, so nothing was committed and main is unaffected. Schema validation never ran. The import is held back until this is resolved, so treat it as actionable."
+
+      - name: Explain a validation failure
+        if: failure() && steps.validate.outcome == 'failure'
+        run: |
+          echo "::error::Imported data failed tools/materialize.py, which both schema-validates and materializes, so nothing was committed and main is unaffected. This is a data bug, not a flake -- read the errors above to see which of the two failed. The import is held back until it is resolved, so treat it as actionable."
+
       - run: |
           git add .
           git diff-index --quiet HEAD || git commit -m "via import_fancons"

.github/workflows/import_furrynz.yml

@@ -20,6 +20,29 @@ jobs:
           git config --global user.name "cons.fyi GitHub bot"
           git config --global user.email "github@cons.fyi"
       - run: ./tools/data-importers/import_furrynz_all.sh
+
+      # Validate before committing so a bad import fails the run instead of landing on main; see import_concat.yml.
+      - name: Canonical formatting
+        id: format
+        run: uv run tools/format.py *.json
+      - name: Prepare the materialize output directory
+        run: mkdir -p "$RUNNER_TEMP/out"
+      - name: Schema validation (materialize)
+        id: validate
+        run: uv run tools/materialize.py "$RUNNER_TEMP/out"
+
+      # Separate messages because a failing format step skips validate, so
+      # naming the wrong one sends whoever reads it to the wrong file.
+      - name: Explain a formatting failure
+        if: failure() && steps.format.outcome == 'failure'
+        run: |
+          echo "::error::tools/format.py failed on the imported data, so nothing was committed and main is unaffected. Schema validation never ran. The import is held back until this is resolved, so treat it as actionable."
+
+      - name: Explain a validation failure
+        if: failure() && steps.validate.outcome == 'failure'
+        run: |
+          echo "::error::Imported data failed tools/materialize.py, which both schema-validates and materializes, so nothing was committed and main is unaffected. This is a data bug, not a flake -- read the errors above to see which of the two failed. The import is held back until it is resolved, so treat it as actionable."
+
       - run: |
           git add .
           git diff-index --quiet HEAD || git commit -m "via import_furrynz"

.github/workflows/import_rams.yml

@@ -22,6 +22,29 @@ jobs:
       - run: ./tools/data-importers/import_rams.py
         env:
           GOOGLE_MAPS_API_KEY: ${{ secrets.GOOGLE_MAPS_API_KEY }}
+
+      # Validate before committing so a bad import fails the run instead of landing on main; see import_concat.yml.
+      - name: Canonical formatting
+        id: format
+        run: uv run tools/format.py *.json
+      - name: Prepare the materialize output directory
+        run: mkdir -p "$RUNNER_TEMP/out"
+      - name: Schema validation (materialize)
+        id: validate
+        run: uv run tools/materialize.py "$RUNNER_TEMP/out"
+
+      # Separate messages because a failing format step skips validate, so
+      # naming the wrong one sends whoever reads it to the wrong file.
+      - name: Explain a formatting failure
+        if: failure() && steps.format.outcome == 'failure'
+        run: |
+          echo "::error::tools/format.py failed on the imported data, so nothing was committed and main is unaffected. Schema validation never ran. The import is held back until this is resolved, so treat it as actionable."
+
+      - name: Explain a validation failure
+        if: failure() && steps.validate.outcome == 'failure'
+        run: |
+          echo "::error::Imported data failed tools/materialize.py, which both schema-validates and materializes, so nothing was committed and main is unaffected. This is a data bug, not a flake -- read the errors above to see which of the two failed. The import is held back until it is resolved, so treat it as actionable."
+
       - run: |
           git add .
           git diff-index --quiet HEAD || git commit -m "via import_rams"

.github/workflows/validate.yml

@@ -28,3 +28,5 @@ jobs:
           uv run tools/materialize.py "$RUNNER_TEMP/out"
       - name: Reject-parser unit tests
         run: python3 .github/scripts/test_keydates_reject.py
+      - name: Date-order validation tests
+        run: python3 .github/scripts/test_materialize_dates.py

.gitignore

@@ -0,0 +1,2 @@
+__pycache__/
+*.pyc

tools/materialize.py

@@ -119,6 +119,7 @@ def main():
         has_errors = False
         for error in validator.iter_errors(series):
             el.log(series_id, error.json_path, error.message)
+            has_errors = True
         if has_errors:
             continue
 
@@ -127,6 +128,20 @@ def main():
         ):
             assert event is not None
 
+            # schema.json expresses this as formatMinimum/$data, an ajv
+            # extension python-jsonschema ignores, so enforce it here.
+            # Comparing parsed values rather than strings keeps this
+            # independent of which ISO date spellings the format checker
+            # happens to admit.
+            if whenever.Date.parse_iso(event["endDate"]) < whenever.Date.parse_iso(
+                event["startDate"]
+            ):
+                el.log(
+                    f"{series_id}/{event['id']}",
+                    "$.endDate",
+                    f"endDate {event['endDate']} is before startDate {event['startDate']}",
+                )
+
             event_locale_is_zh = event["locale"][:3] == "zh-"
             tls = event.get("translations", {})
 

tools/schema.json

@@ -47,6 +47,7 @@
           "endDate": {
             "type": "string",
             "format": "date",
+            "$comment": "formatMinimum/$data is an ajv extension that python-jsonschema ignores, so it is not enforced by this schema alone; tools/materialize.py enforces endDate >= startDate.",
             "formatMinimum": {
               "$data": "1/startDate"
             }