From 7e8b55fc0205a4a7e7286673cba212de8de2b100 Mon Sep 17 00:00:00 2001 From: Sparky Fen Date: Wed, 12 Aug 2026 14:10:40 -0700 Subject: [PATCH 1/5] Enforce endDate >= startDate in materialize validation The schema expressed this as formatMinimum with a $data pointer, an ajv extension that python-jsonschema silently ignores, so events with an end date before their start date passed CI (ainmhicon 2027 in PR #77 shipped with endDate 2026-04-04 against startDate 2027-04-02 and validated green). Enforce the constraint explicitly in materialize.py's event loop and fix the adjacent has_errors flag, which was never set, so schema-invalid files now skip event processing instead of risking KeyErrors. Tested by .github/scripts/test_materialize_dates.py, which runs the real script against fixture series files and is wired into validate.yml. --- .../keydates_reject.cpython-314.pyc | Bin 0 -> 6152 bytes .github/scripts/test_materialize_dates.py | 77 ++++++++++++++++++ .github/workflows/validate.yml | 2 + tools/materialize.py | 12 +++ 4 files changed, 91 insertions(+) create mode 100644 .github/scripts/__pycache__/keydates_reject.cpython-314.pyc create mode 100644 .github/scripts/test_materialize_dates.py diff --git a/.github/scripts/__pycache__/keydates_reject.cpython-314.pyc b/.github/scripts/__pycache__/keydates_reject.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..db9612b47d90b8ca77e6568b223c8c482af9f832 GIT binary patch literal 6152 zcma)AZ%`aZcJG;;*}t>w@<$*U5QE@=1$I{k{Q-oLISC|TSgi-VXP;N>jdq7+V0UKK zGb;(AT=tTyoMfj8d{PxuQpu4@Qc*6hI$h<9Kjg!?&dIr)R0UaJOdUl6JM2@ew_H|GHef|2qU%%H~6<#-j^tYeinEP=xLjOrVti+TGj}0gkkbrbF zjRfifrBh~3>$I8MbeowoI%DRn&YHPhx0|^`cbK_TcV1$nRo3+7ycr2JwA7SqtkN;M zSkgU$P3Hur4(VQjtwVyn4rTm$#U-EM=zATZ)C&kTpi7P@Z;dSH&FE4^GdfnPI7Sr< zO|X*|^B-{Lov_S#QAuSaUgWeOBV#m(`RE zv#N?)15o<6))_E~4%8CX#ujyD$ z>Uo4*{kwv^ay)k1Tcs)YEdy&Jx!V&K7z zOWPH7entjaNn&!A$9aV(YZbSOrC<)LIhC*{@>x+&&dN%fCvpRMO)e<9czuB9OCsh6 zK|YF}kW)kaU=sSIRlG118l0CEQbR@!nW7qmr3qW1(2#rNx|q#nq=DT@L`9#)DhMdp zbF}9K$Sy5|Fhm{p8cJqNR!Bqe=!qaS6UpndqTXISxEyc{Blr~TEh8(EJCgqqXvlPl z!?@Pf@unDDJR0l|Ug`>v&%N{Cm{oNta{~rvB&^-YiHekgOu{oN&WcJ>(r%>Yv>R8| zOdd`jx&-2(4iyeXyHVuKoH3sIgJXXf1lm)gfsWJ7@kwaw>^ME}CUE?#!fjX{z5}N_ zLY;xu#jicVWJ%9WZD)bn5MmxPP+LIpq6q+vLhb$(Ls8OyU)pa!IVwiAm-?h(mIkol z7F3Voc57hNX_d-()Ke~7r5F|UTIF&+%~+OdPgCQluOHf1BR8w|d8{h*3d1ZHNX ze_yo2;~+G*4#Jt0RzqO*Jgnf(661oCGRcoa%yGnow?hY2hNtl@Gm{w zLtpEBc!>4WU4^co(jkmO(2!`nP<5rWe z7@DU0z7O|=sj070vy?!Utso3HL4f`Z8DrS7dHMm$umNC0W-lIyMrJ;OiEbHOA|Wb@ zs)N`P348#my5X@n`z?a2s~ zz?>|t*987W3yRsU0si0^zMgXl>~j&r^*qF|aKKtyuy=!3U?rH#5n}~s%r3ZM4#6FB z3Z9rt;9`{EjnQF;>JusiUzig7;91Z)w{TST$BA~EY5I^gs~?R}qjwR?dFoNj4W3J- zP$g6!BWDSz$*QXn4(zL|73ze8`|1t}F9`MJy7(@-#dp0VYu!^+g38l*TVmIf$#L0Uh$nx55zk|ubg^%j@qe9~vl)`XrX zF@d*i;%j8qP@dB&HKT8BPt(F-q4^|Jtb-HfqD@wNITwzUx!5mTQ!iTfoZ+s1n=P%v zH-xqr%OQb{*@3pl9A??cp&n=Xgv`Djz)Qfgk0@VW-zrTb)BoKs&(7z`6ExN{eXtL7 zD-FAA08M@cU^^bN=87J*O643hJOraCf^b&q-s2y{d#u{1Wq?)%+C5bpWc%Zmng=HkS}NMtJU&5LK>FyJDxzF(Uh5sWJ9>x?imJT-DQF+4@C zF^B*RiXVu`-Ef6P45#IC8&1pPHW=b?V}jr0{v)2YL0?`lXc7Dn(<7MLy}ZaZNL;)+ z_&<&T`MQe>6bOaDS-eQaP`qSR09a&>eMz8BmSB`z$A%4JlmLy%*%)?BPpNr5z!+{V ze;FbSaBDQf2C;`>(-t(d9lSEDJ0s$>W;#c$o#vH|$!QvuSqPRwgBdj`X0##VL6YDP{ky7J&{g%mL> zEmm8;6@O0F&*v|b*l^Kpwo9;DX3qTE;nP zdp@cceWMbpQzcZVO;rDi(UuKgM+LSCy>g~##e%H`9k&r5`x(y3(VLuq+vI$gZyxSa zYSI=ZinTArC~JjD;9&1<-gh3M(yCy^=KTW*VXjI=h|;GeqAYais6oveEX0@?{AH{W zCffKyq3FlC2Gf!vUC-emqxo6Y*lh^ymfnP+8i9=8{ssiTUfciE$$QlDm6iNE`PEl% zUtc?ZxAO1$KB?>ar^zqOW&Wh2iyECWnGrLY0E17!SlmxY9}vR^v3HT5mrMNM0a9Q0 z6`YvshoQ$8=K58U_Xpk+S9)*vZW2}=)b_9RuJrzJ_iL-Vf4#c@=L278#EJR7Z`e!S zIRNC{W+0=6E0xc|LjwsssB2(sb^rj7lEFb}3UN$A)RMAnP;SE}rcy-lvsqbRq=VT- zzhJ(Okk=41tmh3XWH7nBHXERd#!9R6i@xH60(rlXvN@e!bnzral2QXa91laD57NLF zq|Z+;a%A*w2*u#TP>qRIVxAC!OM!ZWRyD%`uPQQD6?~jDkexvwg27QYm^qc$kp{a0 z(88jUhQNwM+J;XU85=n>b>U)UGBJ8#d<41@fMYVMn1byo3x>V;P6B~gKASVxYgh)c zz-tEy1CtO-VS@VruW*i1Dp~}eVo2t=I9V+C#wIUD&Pt@WS%C@Q<1R9eF@ro-%SciV z;1%9V)U4siD>4Zz0+kpXE9gWZ!*C=4ec^csf(|qE1n|cAdE&s$WO@_S93>GO)C}R7 zyj;P2 zpK`us=W5q_```xmQt_jEy`z7F8z3K!)q`s%Hn>i!{L%*Z;&$bMJL*738wf~o|>TPY4*x-Yz$db0Cy|w$b<%@jmi){N3EYGjuO@F6R zckm%syTvu$=Nea~4X)LkdQID^?Aoa8x_f2YU$f%+p=-7C9`!SJ(?9yqU%BFV$Fa)X z@jhiyRre!1^7@zAJAbsnwHH}$T|d%uukT*xd&BR|zW42)UHP!)L;Hu~dUWc;tLuDh zgZtK}zN+ODYX>%b?OVQ{`@Wug;)d@e;V896Z~9w+qa7Cc`*vy(SG_DO4Q_M(rLnih zmbKONUFt9AH|sjzjs0Z$$J4NY_8@p>v+m3mclOaKRMYt6n-pT*H$$tJH<{*7J>1iN z5C({8y|V4gryOt#yZYS|KdP+X@(1txgYRDZXJ)H!{C?l~x^LnMV&N>FxRJMN`QqBS zwf?&o?ukErZN2JL@v7b8CO+mSez)Tx-M-XB(8r%QoE{sXKd7%8t9E?QQw8OJVGj`6 zc$m;ZFVKJG0)!s#8f$m_f*-DgkB_Q7W38T#4!cOX-7|L5^HGnJlm|M;{p`QB9vSzr z|L$R-Y&a7M0=S7ppwe(85-BwaAw6+Uut3NdA=5w%8W(Y(XxZfP0L)d*<^ahIY7!IA z1;0uNK_%0m$lSz9NF)qKx-RSgfJ`@{c=tF5vuY}zk%n*%`jg8~!$9DHgQ6ZWE{b_v zgDC%J$nzO;eTKXyt^5qRp=QU8z&tzO>fE7e_7vzm>-fRh@11?@Vm)-hu~UJnj~9IZ t4RbX7Pc3 Date: Wed, 12 Aug 2026 14:25:56 -0700 Subject: [PATCH 2/5] Address review findings: gitignore, discriminating test, importer validation - Drop the accidentally committed .pyc and add a .gitignore so compiled bytecode can't be tracked again. - Make test_schema_invalid_file_still_fails actually discriminate: the pre-fix code also exited 1 (via an uncaught KeyError), so the test now asserts the ErrorLogger message and the absence of a traceback. Verified by mutation: reverting the has_errors fix fails the test. - Validate imported data in import_fancons, import_rams and import_furrynz before their push to main, mirroring import_concat, so a bad imported date fails the import instead of breaking deploy. - Document the ajv-only formatMinimum keyword in schema.json and the preconditions of the string date comparison in materialize.py. --- .../__pycache__/keydates_reject.cpython-314.pyc | Bin 6152 -> 0 bytes .github/scripts/test_materialize_dates.py | 13 +++++++++---- .github/workflows/import_fancons.yml | 11 +++++++++++ .github/workflows/import_furrynz.yml | 11 +++++++++++ .github/workflows/import_rams.yml | 11 +++++++++++ .gitignore | 2 ++ tools/materialize.py | 6 ++++-- tools/schema.json | 1 + 8 files changed, 49 insertions(+), 6 deletions(-) delete mode 100644 .github/scripts/__pycache__/keydates_reject.cpython-314.pyc create mode 100644 .gitignore diff --git a/.github/scripts/__pycache__/keydates_reject.cpython-314.pyc b/.github/scripts/__pycache__/keydates_reject.cpython-314.pyc deleted file mode 100644 index db9612b47d90b8ca77e6568b223c8c482af9f832..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6152 zcma)AZ%`aZcJG;;*}t>w@<$*U5QE@=1$I{k{Q-oLISC|TSgi-VXP;N>jdq7+V0UKK zGb;(AT=tTyoMfj8d{PxuQpu4@Qc*6hI$h<9Kjg!?&dIr)R0UaJOdUl6JM2@ew_H|GHef|2qU%%H~6<#-j^tYeinEP=xLjOrVti+TGj}0gkkbrbF zjRfifrBh~3>$I8MbeowoI%DRn&YHPhx0|^`cbK_TcV1$nRo3+7ycr2JwA7SqtkN;M zSkgU$P3Hur4(VQjtwVyn4rTm$#U-EM=zATZ)C&kTpi7P@Z;dSH&FE4^GdfnPI7Sr< zO|X*|^B-{Lov_S#QAuSaUgWeOBV#m(`RE zv#N?)15o<6))_E~4%8CX#ujyD$ z>Uo4*{kwv^ay)k1Tcs)YEdy&Jx!V&K7z zOWPH7entjaNn&!A$9aV(YZbSOrC<)LIhC*{@>x+&&dN%fCvpRMO)e<9czuB9OCsh6 zK|YF}kW)kaU=sSIRlG118l0CEQbR@!nW7qmr3qW1(2#rNx|q#nq=DT@L`9#)DhMdp zbF}9K$Sy5|Fhm{p8cJqNR!Bqe=!qaS6UpndqTXISxEyc{Blr~TEh8(EJCgqqXvlPl z!?@Pf@unDDJR0l|Ug`>v&%N{Cm{oNta{~rvB&^-YiHekgOu{oN&WcJ>(r%>Yv>R8| zOdd`jx&-2(4iyeXyHVuKoH3sIgJXXf1lm)gfsWJ7@kwaw>^ME}CUE?#!fjX{z5}N_ zLY;xu#jicVWJ%9WZD)bn5MmxPP+LIpq6q+vLhb$(Ls8OyU)pa!IVwiAm-?h(mIkol z7F3Voc57hNX_d-()Ke~7r5F|UTIF&+%~+OdPgCQluOHf1BR8w|d8{h*3d1ZHNX ze_yo2;~+G*4#Jt0RzqO*Jgnf(661oCGRcoa%yGnow?hY2hNtl@Gm{w zLtpEBc!>4WU4^co(jkmO(2!`nP<5rWe z7@DU0z7O|=sj070vy?!Utso3HL4f`Z8DrS7dHMm$umNC0W-lIyMrJ;OiEbHOA|Wb@ zs)N`P348#my5X@n`z?a2s~ zz?>|t*987W3yRsU0si0^zMgXl>~j&r^*qF|aKKtyuy=!3U?rH#5n}~s%r3ZM4#6FB z3Z9rt;9`{EjnQF;>JusiUzig7;91Z)w{TST$BA~EY5I^gs~?R}qjwR?dFoNj4W3J- zP$g6!BWDSz$*QXn4(zL|73ze8`|1t}F9`MJy7(@-#dp0VYu!^+g38l*TVmIf$#L0Uh$nx55zk|ubg^%j@qe9~vl)`XrX zF@d*i;%j8qP@dB&HKT8BPt(F-q4^|Jtb-HfqD@wNITwzUx!5mTQ!iTfoZ+s1n=P%v zH-xqr%OQb{*@3pl9A??cp&n=Xgv`Djz)Qfgk0@VW-zrTb)BoKs&(7z`6ExN{eXtL7 zD-FAA08M@cU^^bN=87J*O643hJOraCf^b&q-s2y{d#u{1Wq?)%+C5bpWc%Zmng=HkS}NMtJU&5LK>FyJDxzF(Uh5sWJ9>x?imJT-DQF+4@C zF^B*RiXVu`-Ef6P45#IC8&1pPHW=b?V}jr0{v)2YL0?`lXc7Dn(<7MLy}ZaZNL;)+ z_&<&T`MQe>6bOaDS-eQaP`qSR09a&>eMz8BmSB`z$A%4JlmLy%*%)?BPpNr5z!+{V ze;FbSaBDQf2C;`>(-t(d9lSEDJ0s$>W;#c$o#vH|$!QvuSqPRwgBdj`X0##VL6YDP{ky7J&{g%mL> zEmm8;6@O0F&*v|b*l^Kpwo9;DX3qTE;nP zdp@cceWMbpQzcZVO;rDi(UuKgM+LSCy>g~##e%H`9k&r5`x(y3(VLuq+vI$gZyxSa zYSI=ZinTArC~JjD;9&1<-gh3M(yCy^=KTW*VXjI=h|;GeqAYais6oveEX0@?{AH{W zCffKyq3FlC2Gf!vUC-emqxo6Y*lh^ymfnP+8i9=8{ssiTUfciE$$QlDm6iNE`PEl% zUtc?ZxAO1$KB?>ar^zqOW&Wh2iyECWnGrLY0E17!SlmxY9}vR^v3HT5mrMNM0a9Q0 z6`YvshoQ$8=K58U_Xpk+S9)*vZW2}=)b_9RuJrzJ_iL-Vf4#c@=L278#EJR7Z`e!S zIRNC{W+0=6E0xc|LjwsssB2(sb^rj7lEFb}3UN$A)RMAnP;SE}rcy-lvsqbRq=VT- zzhJ(Okk=41tmh3XWH7nBHXERd#!9R6i@xH60(rlXvN@e!bnzral2QXa91laD57NLF zq|Z+;a%A*w2*u#TP>qRIVxAC!OM!ZWRyD%`uPQQD6?~jDkexvwg27QYm^qc$kp{a0 z(88jUhQNwM+J;XU85=n>b>U)UGBJ8#d<41@fMYVMn1byo3x>V;P6B~gKASVxYgh)c zz-tEy1CtO-VS@VruW*i1Dp~}eVo2t=I9V+C#wIUD&Pt@WS%C@Q<1R9eF@ro-%SciV z;1%9V)U4siD>4Zz0+kpXE9gWZ!*C=4ec^csf(|qE1n|cAdE&s$WO@_S93>GO)C}R7 zyj;P2 zpK`us=W5q_```xmQt_jEy`z7F8z3K!)q`s%Hn>i!{L%*Z;&$bMJL*738wf~o|>TPY4*x-Yz$db0Cy|w$b<%@jmi){N3EYGjuO@F6R zckm%syTvu$=Nea~4X)LkdQID^?Aoa8x_f2YU$f%+p=-7C9`!SJ(?9yqU%BFV$Fa)X z@jhiyRre!1^7@zAJAbsnwHH}$T|d%uukT*xd&BR|zW42)UHP!)L;Hu~dUWc;tLuDh zgZtK}zN+ODYX>%b?OVQ{`@Wug;)d@e;V896Z~9w+qa7Cc`*vy(SG_DO4Q_M(rLnih zmbKONUFt9AH|sjzjs0Z$$J4NY_8@p>v+m3mclOaKRMYt6n-pT*H$$tJH<{*7J>1iN z5C({8y|V4gryOt#yZYS|KdP+X@(1txgYRDZXJ)H!{C?l~x^LnMV&N>FxRJMN`QqBS zwf?&o?ukErZN2JL@v7b8CO+mSez)Tx-M-XB(8r%QoE{sXKd7%8t9E?QQw8OJVGj`6 zc$m;ZFVKJG0)!s#8f$m_f*-DgkB_Q7W38T#4!cOX-7|L5^HGnJlm|M;{p`QB9vSzr z|L$R-Y&a7M0=S7ppwe(85-BwaAw6+Uut3NdA=5w%8W(Y(XxZfP0L)d*<^ahIY7!IA z1;0uNK_%0m$lSz9NF)qKx-RSgfJ`@{c=tF5vuY}zk%n*%`jg8~!$9DHgQ6ZWE{b_v zgDC%J$nzO;eTKXyt^5qRp=QU8z&tzO>fE7e_7vzm>-fRh@11?@Vm)-hu~UJnj~9IZ t4RbX7Pc3= startDate.", "formatMinimum": { "$data": "1/startDate" } From 9fad72f11118b4fb29f394259d1177ce1b5f79aa Mon Sep 17 00:00:00 2001 From: Sparky Fen Date: Wed, 12 Aug 2026 14:48:05 -0700 Subject: [PATCH 3/5] Polish from review round 2 - Mirror import_concat.yml's failure-annotation steps in the three importer workflows so a red run says which stage failed, and shrink the duplicated rationale comment to one line. - Decouple the schema-invalid test from jsonschema's exact message wording, run materialize via uv run --script, and cover the case where a later (non-first) event in a series has the bad dates. - Condense the date-check comment in materialize.py. --- .github/scripts/test_materialize_dates.py | 35 +++++++++++++++++++---- .github/workflows/import_fancons.yml | 18 ++++++++++-- .github/workflows/import_furrynz.yml | 18 ++++++++++-- .github/workflows/import_rams.yml | 18 ++++++++++-- tools/materialize.py | 10 +++---- 5 files changed, 79 insertions(+), 20 deletions(-) diff --git a/.github/scripts/test_materialize_dates.py b/.github/scripts/test_materialize_dates.py index 019f205..0eaf16e 100644 --- a/.github/scripts/test_materialize_dates.py +++ b/.github/scripts/test_materialize_dates.py @@ -14,23 +14,29 @@ MATERIALIZE = REPO_ROOT / "tools" / "materialize.py" -def make_series(start_date, end_date): +def make_series_events(events): + """events: list of (id, startDate, endDate), newest first as in the repo.""" return { "name": "Testcon", "events": [ { - "id": "testcon-2027", - "name": "Testcon 2027", + "id": event_id, + "name": f"Testcon ({event_id})", "url": "https://example.com", "startDate": start_date, "endDate": end_date, "venue": "Test Hall", "locale": "en-US", } + for event_id, start_date, end_date in events ], } +def make_series(start_date, end_date): + return make_series_events([("testcon-2027", start_date, end_date)]) + + def run_materialize(series): # materialize only globs *.json in its cwd, so out/ can live alongside # the fixture without being picked up as a series file @@ -40,7 +46,7 @@ def run_materialize(series): out_dir = os.path.join(data_dir, "out") os.mkdir(out_dir) return subprocess.run( - ["uv", "run", str(MATERIALIZE), out_dir], + ["uv", "run", "--script", str(MATERIALIZE), out_dir], cwd=data_dir, capture_output=True, text=True, @@ -65,6 +71,24 @@ def test_end_before_start_fails(self): self.assertEqual(result.returncode, 1, result.stderr) self.assertIn("endDate 2026-04-04 is before startDate 2027-04-02", result.stderr) + def test_later_event_in_series_fails(self): + # the check has to run for every event, not just the first: series + # files list events newest first, so an older event is the likely + # place for a bad date to hide + result = run_materialize( + make_series_events( + [ + ("testcon-2027", "2027-04-02", "2027-04-04"), + ("testcon-2026", "2026-04-03", "2026-04-01"), + ] + ) + ) + self.assertEqual(result.returncode, 1, result.stderr) + self.assertIn("testcon-2026", result.stderr) + self.assertIn( + "endDate 2026-04-01 is before startDate 2026-04-03", result.stderr + ) + def test_schema_invalid_file_still_fails(self): # regression for the has_errors fix: a file that fails schema # validation must be skipped and reported, not fall through into the @@ -74,7 +98,8 @@ def test_schema_invalid_file_still_fails(self): del series["events"][0]["locale"] result = run_materialize(series) self.assertEqual(result.returncode, 1, result.stderr) - self.assertIn("'locale' is a required property", result.stderr) + self.assertIn("required property", result.stderr) + self.assertIn("locale", result.stderr) self.assertNotIn("Traceback", result.stderr) diff --git a/.github/workflows/import_fancons.yml b/.github/workflows/import_fancons.yml index c0b87fa..48eb508 100644 --- a/.github/workflows/import_fancons.yml +++ b/.github/workflows/import_fancons.yml @@ -25,16 +25,28 @@ jobs: FANCONS_CALENDAR_URL: ${{ secrets.FANCONS_CALENDAR_URL }} FANCONS_MAP_URL: ${{ secrets.FANCONS_MAP_URL }} - # This job pushes straight to main and validate.yml only triggers on - # pull_request, so nothing checks what the bot writes. Validate here and - # fail the run instead of committing (same as import_concat.yml). + # Validate before committing so a bad import fails the run instead of landing on main; see import_concat.yml. - name: Canonical formatting + id: format run: uv run tools/format.py *.json - name: Prepare the materialize output directory run: mkdir -p "$RUNNER_TEMP/out" - name: Schema validation (materialize) + id: validate run: uv run tools/materialize.py "$RUNNER_TEMP/out" + # Separate messages because a failing format step skips validate, so + # naming the wrong one sends whoever reads it to the wrong file. + - name: Explain a formatting failure + if: failure() && steps.format.outcome == 'failure' + run: | + echo "::error::tools/format.py failed on the imported data, so nothing was committed and main is unaffected. Schema validation never ran. The import is held back until this is resolved, so treat it as actionable." + + - name: Explain a validation failure + if: failure() && steps.validate.outcome == 'failure' + run: | + echo "::error::Imported data failed tools/materialize.py, which both schema-validates and materializes, so nothing was committed and main is unaffected. This is a data bug, not a flake -- read the errors above to see which of the two failed. The import is held back until it is resolved, so treat it as actionable." + - run: | git add . git diff-index --quiet HEAD || git commit -m "via import_fancons" diff --git a/.github/workflows/import_furrynz.yml b/.github/workflows/import_furrynz.yml index a4d0b58..72bd5ee 100644 --- a/.github/workflows/import_furrynz.yml +++ b/.github/workflows/import_furrynz.yml @@ -21,16 +21,28 @@ jobs: git config --global user.email "github@cons.fyi" - run: ./tools/data-importers/import_furrynz_all.sh - # This job pushes straight to main and validate.yml only triggers on - # pull_request, so nothing checks what the bot writes. Validate here and - # fail the run instead of committing (same as import_concat.yml). + # Validate before committing so a bad import fails the run instead of landing on main; see import_concat.yml. - name: Canonical formatting + id: format run: uv run tools/format.py *.json - name: Prepare the materialize output directory run: mkdir -p "$RUNNER_TEMP/out" - name: Schema validation (materialize) + id: validate run: uv run tools/materialize.py "$RUNNER_TEMP/out" + # Separate messages because a failing format step skips validate, so + # naming the wrong one sends whoever reads it to the wrong file. + - name: Explain a formatting failure + if: failure() && steps.format.outcome == 'failure' + run: | + echo "::error::tools/format.py failed on the imported data, so nothing was committed and main is unaffected. Schema validation never ran. The import is held back until this is resolved, so treat it as actionable." + + - name: Explain a validation failure + if: failure() && steps.validate.outcome == 'failure' + run: | + echo "::error::Imported data failed tools/materialize.py, which both schema-validates and materializes, so nothing was committed and main is unaffected. This is a data bug, not a flake -- read the errors above to see which of the two failed. The import is held back until it is resolved, so treat it as actionable." + - run: | git add . git diff-index --quiet HEAD || git commit -m "via import_furrynz" diff --git a/.github/workflows/import_rams.yml b/.github/workflows/import_rams.yml index 088a526..5721fd2 100644 --- a/.github/workflows/import_rams.yml +++ b/.github/workflows/import_rams.yml @@ -23,16 +23,28 @@ jobs: env: GOOGLE_MAPS_API_KEY: ${{ secrets.GOOGLE_MAPS_API_KEY }} - # This job pushes straight to main and validate.yml only triggers on - # pull_request, so nothing checks what the bot writes. Validate here and - # fail the run instead of committing (same as import_concat.yml). + # Validate before committing so a bad import fails the run instead of landing on main; see import_concat.yml. - name: Canonical formatting + id: format run: uv run tools/format.py *.json - name: Prepare the materialize output directory run: mkdir -p "$RUNNER_TEMP/out" - name: Schema validation (materialize) + id: validate run: uv run tools/materialize.py "$RUNNER_TEMP/out" + # Separate messages because a failing format step skips validate, so + # naming the wrong one sends whoever reads it to the wrong file. + - name: Explain a formatting failure + if: failure() && steps.format.outcome == 'failure' + run: | + echo "::error::tools/format.py failed on the imported data, so nothing was committed and main is unaffected. Schema validation never ran. The import is held back until this is resolved, so treat it as actionable." + + - name: Explain a validation failure + if: failure() && steps.validate.outcome == 'failure' + run: | + echo "::error::Imported data failed tools/materialize.py, which both schema-validates and materializes, so nothing was committed and main is unaffected. This is a data bug, not a flake -- read the errors above to see which of the two failed. The import is held back until it is resolved, so treat it as actionable." + - run: | git add . git diff-index --quiet HEAD || git commit -m "via import_rams" diff --git a/tools/materialize.py b/tools/materialize.py index aae6dc9..b703caa 100755 --- a/tools/materialize.py +++ b/tools/materialize.py @@ -128,12 +128,10 @@ def main(): ): assert event is not None - # schema.json expresses this as formatMinimum with a $data - # pointer, an ajv extension that python-jsonschema silently - # ignores, so it has to be enforced here. Comparing the raw - # strings is only safe because schema-invalid files are skipped - # above: the format checker guarantees canonical YYYY-MM-DD, - # which sorts the same lexically as chronologically. + # schema.json expresses this as formatMinimum/$data, an ajv + # extension python-jsonschema ignores, so enforce it here. Raw + # string comparison is safe: schema-invalid files were skipped + # above, so the format checker guarantees canonical YYYY-MM-DD. if event["endDate"] < event["startDate"]: el.log( f"{series_id}/{event['id']}", From 60326999ddb66707fb5b6b27b6ebd80181ccc3e9 Mon Sep 17 00:00:00 2001 From: Sparky Fen Date: Thu, 13 Aug 2026 12:01:27 -0700 Subject: [PATCH 4/5] Serialize Pages deployments and harden the date check - deploy.yml gets the standard Pages concurrency group (one deployment at a time, queued intermediates skipped, in-progress never cancelled; same pattern as actions/starter-workflows pages/static.yml). Merging four PRs back to back on 2026-08-12 raced the Pages API and two runs failed with 'deployment request failed due to in progress deployment'. - Compare event dates as parsed values instead of raw strings, so the check no longer depends on the format checker admitting only canonical YYYY-MM-DD spellings. - Skip the materialize tests with a clear message when uv is not on PATH instead of tracebacking. --- .github/scripts/test_materialize_dates.py | 4 ++++ .github/workflows/deploy.yml | 9 +++++++++ tools/materialize.py | 11 +++++++---- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/.github/scripts/test_materialize_dates.py b/.github/scripts/test_materialize_dates.py index 0eaf16e..b0a70b6 100644 --- a/.github/scripts/test_materialize_dates.py +++ b/.github/scripts/test_materialize_dates.py @@ -6,6 +6,7 @@ import json import os import pathlib +import shutil import subprocess import tempfile import unittest @@ -53,6 +54,9 @@ def run_materialize(series): ) +@unittest.skipUnless( + shutil.which("uv"), "these tests run materialize.py via uv, which is not on PATH" +) class TestDateOrder(unittest.TestCase): def test_end_after_start_passes(self): result = run_materialize(make_series("2027-04-02", "2027-04-04")) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index 3aa1cbe..ff82be5 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -15,6 +15,15 @@ on: workflow_dispatch: +# One deployment at a time: Pages rejects a deployment while another is in +# progress, so back-to-back merges raced and failed. Runs queued between the +# in-progress one and the newest are skipped (each run deploys full HEAD +# anyway); in-progress deployments are never cancelled. Same pattern as +# github.com/actions/starter-workflows/blob/main/pages/static.yml. +concurrency: + group: pages + cancel-in-progress: false + permissions: contents: read pages: write diff --git a/tools/materialize.py b/tools/materialize.py index b703caa..c45d7ef 100755 --- a/tools/materialize.py +++ b/tools/materialize.py @@ -129,10 +129,13 @@ def main(): assert event is not None # schema.json expresses this as formatMinimum/$data, an ajv - # extension python-jsonschema ignores, so enforce it here. Raw - # string comparison is safe: schema-invalid files were skipped - # above, so the format checker guarantees canonical YYYY-MM-DD. - if event["endDate"] < event["startDate"]: + # extension python-jsonschema ignores, so enforce it here. + # Comparing parsed values rather than strings keeps this + # independent of which ISO date spellings the format checker + # happens to admit. + if whenever.Date.parse_iso(event["endDate"]) < whenever.Date.parse_iso( + event["startDate"] + ): el.log( f"{series_id}/{event['id']}", "$.endDate", From 6663b8b0b48f440c6f54808dab3f906300b91517 Mon Sep 17 00:00:00 2001 From: Sparky Fen Date: Mon, 17 Aug 2026 13:42:41 -0700 Subject: [PATCH 5/5] Add docstrings to the materialize date-order tests --- .github/scripts/test_materialize_dates.py | 43 +++++++++++++++-------- 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/.github/scripts/test_materialize_dates.py b/.github/scripts/test_materialize_dates.py index b0a70b6..14a074c 100644 --- a/.github/scripts/test_materialize_dates.py +++ b/.github/scripts/test_materialize_dates.py @@ -16,7 +16,10 @@ def make_series_events(events): - """events: list of (id, startDate, endDate), newest first as in the repo.""" + """Build a schema-valid series dict from (id, startDate, endDate) tuples. + + Events are listed newest first, matching the layout of the repo's series + files.""" return { "name": "Testcon", "events": [ @@ -35,12 +38,16 @@ def make_series_events(events): def make_series(start_date, end_date): + """Build a one-event series with the given start and end dates.""" return make_series_events([("testcon-2027", start_date, end_date)]) def run_materialize(series): - # materialize only globs *.json in its cwd, so out/ can live alongside - # the fixture without being picked up as a series file + """Write ``series`` as a fixture and run materialize.py against it via uv. + + Returns the CompletedProcess. materialize only globs *.json in its cwd, so + out/ can live alongside the fixture without being picked up as a series + file.""" with tempfile.TemporaryDirectory() as data_dir: with open(os.path.join(data_dir, "testcon.json"), "w") as f: json.dump(series, f) @@ -58,27 +65,34 @@ def run_materialize(series): shutil.which("uv"), "these tests run materialize.py via uv, which is not on PATH" ) class TestDateOrder(unittest.TestCase): + """End-to-end checks of the endDate >= startDate rule in materialize.py.""" + def test_end_after_start_passes(self): + """A multi-day event with endDate after startDate validates.""" result = run_materialize(make_series("2027-04-02", "2027-04-04")) self.assertEqual(result.returncode, 0, result.stderr) def test_single_day_event_passes(self): + """A single-day event (endDate == startDate) validates.""" result = run_materialize(make_series("2027-04-02", "2027-04-02")) self.assertEqual(result.returncode, 0, result.stderr) def test_end_before_start_fails(self): - # regression: ainmhicon-2027 was submitted with endDate 2026-04-04 - # against startDate 2027-04-02 and passed validation, because the - # schema's formatMinimum/$data keyword is an ajv extension that - # python-jsonschema ignores + """An event whose endDate precedes startDate fails with a clear message. + + Regression: ainmhicon-2027 was submitted with endDate 2026-04-04 + against startDate 2027-04-02 and passed validation, because the + schema's formatMinimum/$data keyword is an ajv extension that + python-jsonschema ignores.""" result = run_materialize(make_series("2027-04-02", "2026-04-04")) self.assertEqual(result.returncode, 1, result.stderr) self.assertIn("endDate 2026-04-04 is before startDate 2027-04-02", result.stderr) def test_later_event_in_series_fails(self): - # the check has to run for every event, not just the first: series - # files list events newest first, so an older event is the likely - # place for a bad date to hide + """The check runs for every event in a series, not just the first. + + Series files list events newest first, so an older event is the + likely place for a bad date to hide.""" result = run_materialize( make_series_events( [ @@ -94,10 +108,11 @@ def test_later_event_in_series_fails(self): ) def test_schema_invalid_file_still_fails(self): - # regression for the has_errors fix: a file that fails schema - # validation must be skipped and reported, not fall through into the - # event loop. Without the fix it also exits 1, but via an uncaught - # KeyError, so the traceback assertion is what discriminates. + """A schema-invalid file is reported and skipped, not crashed on. + + Regression for the has_errors fix: without it the file falls through + into the event loop and exits 1 via an uncaught KeyError, so the + traceback assertion is what discriminates.""" series = make_series("2027-04-02", "2027-04-04") del series["events"][0]["locale"] result = run_materialize(series)