From d16c8b0ee77e5db303522998a09b74793a966184 Mon Sep 17 00:00:00 2001 From: nick-gorman Date: Thu, 11 Jun 2026 12:25:50 +1000 Subject: [PATCH 1/6] Make the new-format templater output set granularity-invariant MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The new-format templater now emits the same table set at every granularity: timeslices and costs_connection are wired in, and the custom-constraint tables are emitted header-only at nem_regions / single_region instead of being absent. This reverses the gating from 9366945 — the PLEXOS constraints are sub-regional export-group limits with no meaningful representation once sub-regions are collapsed, but writing "all columns, no rows" tables means list_templater_output_files needs no granularity awareness and downstream consumers never check for missing tables. costs_connection is also now tracked in the output list — it was being written but untracked, the same class of bug 9366945 fixed for the constraint tables. Co-Authored-By: Claude Fable 5 --- src/ispypsa/templater/create_template.py | 34 ++--- .../custom_constraints_from_plexos.py | 28 ++++ ...create_ispypsa_inputs_new_table_formats.py | 44 +++++-- .../test_create_ispypsa_inputs_template.py | 121 ++++++++++++++---- 4 files changed, 171 insertions(+), 56 deletions(-) diff --git a/src/ispypsa/templater/create_template.py b/src/ispypsa/templater/create_template.py index bf26486a..2861c85d 100644 --- a/src/ispypsa/templater/create_template.py +++ b/src/ispypsa/templater/create_template.py @@ -6,6 +6,7 @@ from ispypsa.feature_flags import FEATURE_FLAGS from ispypsa.templater.connection_and_build_costs import _template_connection_costs from ispypsa.templater.custom_constraints_from_plexos import ( + empty_custom_constraint_tables, template_custom_constraints_from_plexos, ) from ispypsa.templater.dynamic_generator_properties import ( @@ -44,6 +45,10 @@ _template_new_generators_static_properties, ) from ispypsa.templater.storage import _template_battery_properties +from ispypsa.templater.timeslices import ( + _template_timeslices, + load_timeslice_calendar, +) from ispypsa.templater.transmission import _template_network_transmission _BASE_TEMPLATE_OUTPUTS = [ @@ -74,8 +79,9 @@ ] # Outputs from the new-format templater branch. Granularity-invariant: the -# same five tables are emitted for sub_regions, nem_regions, and single_region -# (only their contents differ). +# same tables are emitted for sub_regions, nem_regions, and single_region +# (only their contents differ — the custom-constraint tables are header-only +# at coarser granularities). # FEATURE_FLAG_CLEANUP[use_new_table_format]: rename to _TEMPLATE_OUTPUTS and # delete _BASE_TEMPLATE_OUTPUTS above. _NEW_FORMAT_TEMPLATE_OUTPUTS = [ @@ -84,15 +90,8 @@ "network_transmission_path_limits", "network_expansion_options", "network_transmission_path_expansion_costs", -] - -# Custom constraints are templated only at sub_regions granularity (see the gate -# in create_ispypsa_inputs_template) — coarser granularities collapse the -# sub-region nodes, sub-regional flow paths and REZ-located units the constraints -# reference, leaving nothing to constrain. Listed as outputs only at that -# granularity so the create_ispypsa_inputs task tracks them where they are -# written and does not expect them where they never are. -_CUSTOM_CONSTRAINT_OUTPUTS = [ + "timeslices", + "costs_connection", "custom_constraints", "custom_constraints_lhs", "custom_constraints_rhs", @@ -219,6 +218,10 @@ def create_ispypsa_inputs_template( ) template["network_expansion_options"] = expansion_options template["network_transmission_path_expansion_costs"] = expansion_costs + template["timeslices"] = _template_timeslices( + load_timeslice_calendar(iasr_workbook_version), + manually_extracted_tables["reference_year_sequence"], + ) # todo: replace with actual generators_new_entrant once that templating # function is written — passing empty placeholder for now so costs_connection @@ -246,14 +249,17 @@ def create_ispypsa_inputs_template( # their LHS references sub-region nodes, sub-regional flow paths, and # REZ-located units that only exist as distinct entities at sub_regions # granularity. Once sub-regions are collapsed (nem_regions / - # single_region) they have no meaningful representation, so only emit - # them for sub_regions. + # single_region) they have no meaningful representation, so the tables + # are emitted header-only ("all columns, no rows") there, keeping the + # output set granularity-invariant. if regional_granularity == "sub_regions": template.update( template_custom_constraints_from_plexos( iasr_tables, iasr_workbook_version=iasr_workbook_version ) ) + else: + template.update(empty_custom_constraint_tables()) return template template = {} @@ -347,8 +353,6 @@ def list_templater_output_files(regional_granularity, output_path=None): # granularity-specific file removals. if FEATURE_FLAGS["use_new_table_format"]: files = _NEW_FORMAT_TEMPLATE_OUTPUTS.copy() - if regional_granularity == "sub_regions": - files += _CUSTOM_CONSTRAINT_OUTPUTS else: files = _BASE_TEMPLATE_OUTPUTS.copy() if regional_granularity in ["sub_regions", "single_region"]: diff --git a/src/ispypsa/templater/custom_constraints_from_plexos.py b/src/ispypsa/templater/custom_constraints_from_plexos.py index c2a4672e..80a84e54 100644 --- a/src/ispypsa/templater/custom_constraints_from_plexos.py +++ b/src/ispypsa/templater/custom_constraints_from_plexos.py @@ -234,6 +234,34 @@ _EXCLUDED_PARENT_CLASSES = ("Purchaser",) _AREA_SUFFIX_PATTERN = re.compile(r" Area\d+$") +# Output columns of the three custom-constraint template tables. +_CUSTOM_CONSTRAINTS_COLUMNS = ["constraint_id", "direction"] +_CUSTOM_CONSTRAINTS_LHS_COLUMNS = [ + "constraint_id", + "term_type", + "variable_name", + "coefficient", + "date_from", +] +_CUSTOM_CONSTRAINTS_RHS_COLUMNS = ["constraint_id", "timeslice", "rhs", "date_from"] + + +def empty_custom_constraint_tables() -> dict[str, pd.DataFrame]: + """Header-only versions of the three custom-constraint template tables. + + Emitted at nem_regions / single_region granularity, where the sub-region + nodes, sub-regional flow paths, and REZ-located units the constraints + reference are collapsed, leaving nothing to constrain. Emitting "all + columns, no rows" tables keeps the templater's output set + granularity-invariant, so downstream consumers never need to check for + missing tables. + """ + return { + "custom_constraints": pd.DataFrame(columns=_CUSTOM_CONSTRAINTS_COLUMNS), + "custom_constraints_lhs": pd.DataFrame(columns=_CUSTOM_CONSTRAINTS_LHS_COLUMNS), + "custom_constraints_rhs": pd.DataFrame(columns=_CUSTOM_CONSTRAINTS_RHS_COLUMNS), + } + def template_custom_constraints_from_plexos( iasr_tables: dict[str, pd.DataFrame], diff --git a/tests/test_cli/test_create_ispypsa_inputs_new_table_formats.py b/tests/test_cli/test_create_ispypsa_inputs_new_table_formats.py index 79a967b4..1de4d8da 100644 --- a/tests/test_cli/test_create_ispypsa_inputs_new_table_formats.py +++ b/tests/test_cli/test_create_ispypsa_inputs_new_table_formats.py @@ -71,13 +71,15 @@ "network_transmission_path_limits", "network_expansion_options", "network_transmission_path_expansion_costs", + "timeslices", "costs_connection", ] # Custom constraints are templated only at sub_regions (coarser granularities -# collapse the entities they reference). Detailed content lives in -# test_custom_constraints_from_plexos.py; here we check the CLI writes them at -# sub_regions and omits them otherwise. +# collapse the entities they reference) but written at every granularity — +# header-only at nem_regions / single_region. Detailed content lives in +# test_custom_constraints_from_plexos.py; here we check the CLI writes them +# populated at sub_regions and empty otherwise. _CUSTOM_CONSTRAINT_OUTPUTS = [ "custom_constraints", "custom_constraints_lhs", @@ -286,6 +288,15 @@ def test_create_ispypsa_inputs_task_new_format( "single_region": 1209, } +# Per-reference-year window patterns decoded from the shipped RefYear5000 +# calendar and reference_year_sequence (drift-detection; +# granularity-invariant). 14 distinct ids, not 15: TAS Hot Day never +# activates in the calendar, so tas_peak_demand has no windows. 15 reference +# years: 2011 to 2025 in year-ending convention. +_NUM_TIMESLICE_PATTERN_ROWS_75 = 678 +_NUM_TIMESLICE_IDS_75 = 14 +_NUM_REFERENCE_YEARS_75 = 15 + @pytest.mark.parametrize("granularity", ["sub_regions", "nem_regions", "single_region"]) def test_create_ispypsa_inputs_new_format( @@ -355,18 +366,25 @@ def test_create_ispypsa_inputs_new_format( assert set(costs["expansion_id"]) == set(options["expansion_id"]) assert len(costs) == _EXPECTED_EXPANSION_COST_ROWS_75[granularity] - # custom_constraints — written only at sub_regions. Detailed content is - # covered by test_custom_constraints_from_plexos.py; here we assert the CLI - # emits the three tables at sub_regions with no orphan LHS/RHS rows, and - # omits them entirely at coarser granularities. + # timeslices — per-reference-year patterns decoded from the shipped + # RefYear5000 calendar, identical at every granularity. + timeslices = pd.read_csv(output_dir / "timeslices.csv") + assert len(timeslices) == _NUM_TIMESLICE_PATTERN_ROWS_75 + assert timeslices["timeslice_id"].nunique() == _NUM_TIMESLICE_IDS_75 + assert timeslices["reference_year"].nunique() == _NUM_REFERENCE_YEARS_75 + + # custom_constraints — templated only at sub_regions but written at every + # granularity (header-only at coarser ones). Detailed content is covered + # by test_custom_constraints_from_plexos.py; here we assert the CLI emits + # the three tables populated at sub_regions with no orphan LHS/RHS rows, + # and empty otherwise. + verify_output_files(output_dir, _CUSTOM_CONSTRAINT_OUTPUTS) + constraints = pd.read_csv(output_dir / "custom_constraints.csv") + lhs = pd.read_csv(output_dir / "custom_constraints_lhs.csv") + rhs = pd.read_csv(output_dir / "custom_constraints_rhs.csv") if granularity == "sub_regions": - verify_output_files(output_dir, _CUSTOM_CONSTRAINT_OUTPUTS) - constraints = pd.read_csv(output_dir / "custom_constraints.csv") - lhs = pd.read_csv(output_dir / "custom_constraints_lhs.csv") - rhs = pd.read_csv(output_dir / "custom_constraints_rhs.csv") constraint_ids = set(constraints["constraint_id"]) assert set(lhs["constraint_id"]) <= constraint_ids assert set(rhs["constraint_id"]) <= constraint_ids else: - for name in _CUSTOM_CONSTRAINT_OUTPUTS: - assert not (output_dir / f"{name}.csv").exists() + assert constraints.empty and lhs.empty and rhs.empty diff --git a/tests/test_templater/test_create_ispypsa_inputs_template.py b/tests/test_templater/test_create_ispypsa_inputs_template.py index 9675f1ed..7bf86f06 100644 --- a/tests/test_templater/test_create_ispypsa_inputs_template.py +++ b/tests/test_templater/test_create_ispypsa_inputs_template.py @@ -57,21 +57,35 @@ def _stub_custom_constraints_tables() -> dict[str, pd.DataFrame]: } -def test_list_templater_output_files_includes_custom_constraints_only_at_sub_regions(): - """Custom-constraint tables are declared as task outputs only at sub_regions. - - They are templated (and written) only at sub_regions, so they must appear in - the output-file list there — otherwise the create_ispypsa_inputs task leaves - files it writes untracked — and must be absent at nem_regions / - single_region, where declaring them would make the task expect files that are - never written. +def _stub_timeslice_calendar() -> pd.DataFrame: + """Minimal raw timeslice calendar: two windows that tile planning year + FY2026 — a hot-day peak and a winter window that wraps back to it, so the + decoded windows leave no day uncovered (the tiling invariant the templater + enforces). Patched in place of the packaged PLEXOS calendar; full decode + behaviour is covered by the per-module tests in ``test_timeslices.py``. """ - custom_constraint_files = { - "custom_constraints", - "custom_constraints_lhs", - "custom_constraints_rhs", - } + return pd.DataFrame( + { + "DATETIME": ["18/11/2025", "20/11/2025", "20/11/2025", "18/11/2026"], + "NAME": ["NSW Hot Day", "NSW Hot Day", "NSW Winter", "NSW Winter"], + "TIMESLICE": [-1, 0, -1, 0], + } + ) + + +def _stub_reference_year_sequence() -> pd.DataFrame: + """Assigns the stub calendar's single planning year a reference year.""" + return pd.DataFrame({"planning_year": [2026], "reference_year": [2015]}) + + +def test_list_templater_output_files_granularity_invariant(): + """The new-format templater declares the same output set at every granularity. + The custom-constraint tables are header-only at nem_regions / + single_region but still written, so the create_ispypsa_inputs task must + track them everywhere — a granularity-dependent list would leave written + files untracked or expect files that are never written. + """ with patch( "ispypsa.templater.create_template.FEATURE_FLAGS", {"use_new_table_format": True}, @@ -80,9 +94,14 @@ def test_list_templater_output_files_includes_custom_constraints_only_at_sub_reg nem_regions = set(list_templater_output_files("nem_regions")) single_region = set(list_templater_output_files("single_region")) - assert custom_constraint_files <= sub_regions - assert custom_constraint_files.isdisjoint(nem_regions) - assert custom_constraint_files.isdisjoint(single_region) + assert sub_regions == nem_regions == single_region + assert { + "custom_constraints", + "custom_constraints_lhs", + "custom_constraints_rhs", + "timeslices", + "costs_connection", + } <= sub_regions def test_create_ispypsa_inputs_template_sub_regions( @@ -239,6 +258,10 @@ def test_create_ispypsa_inputs_template_new_format(csv_str_to_df): "ispypsa.templater.create_template.template_custom_constraints_from_plexos", return_value=_stub_custom_constraints_tables(), ), + patch( + "ispypsa.templater.create_template.load_timeslice_calendar", + return_value=_stub_timeslice_calendar(), + ), ): result = create_ispypsa_inputs_template( scenario="Step Change", @@ -264,6 +287,7 @@ def test_create_ispypsa_inputs_template_new_format(csv_str_to_df): # wiring runs. Output stays empty: generators/storage are placeholder-empty. manually_extracted_tables={ "connection_capacity_non_vre": connection_capacity_non_vre, + "reference_year_sequence": _stub_reference_year_sequence(), }, iasr_workbook_version="ignored-by-patch", ) @@ -304,6 +328,17 @@ def test_create_ispypsa_inputs_template_new_format(csv_str_to_df): # 3 expansion_ids x 2 years assert len(expansion_costs) == 6 + timeslices = result["timeslices"] + assert set(timeslices.columns) == { + "timeslice_id", + "reference_year", + "start_month_day", + "end_month_day", + } + # The stub calendar's two windows decode into two tiling window patterns + # (nsw_peak_demand and nsw_winter_reference) for FY2026's reference year. + assert len(timeslices) == 2 + assert "costs_connection" in result costs_connection = result["costs_connection"] assert set(costs_connection.columns) == { @@ -427,6 +462,10 @@ def test_create_ispypsa_inputs_template_new_format_nem_regions(csv_str_to_df): "ispypsa.templater.create_template.template_custom_constraints_from_plexos", return_value=_stub_custom_constraints_tables(), ) as mock_template_custom_constraints, + patch( + "ispypsa.templater.create_template.load_timeslice_calendar", + return_value=_stub_timeslice_calendar(), + ), ): result = create_ispypsa_inputs_template( scenario="Step Change", @@ -449,6 +488,7 @@ def test_create_ispypsa_inputs_template_new_format_nem_regions(csv_str_to_df): }, manually_extracted_tables={ "connection_capacity_non_vre": connection_capacity_non_vre, + "reference_year_sequence": _stub_reference_year_sequence(), }, iasr_workbook_version="ignored-by-patch", ) @@ -490,13 +530,28 @@ def test_create_ispypsa_inputs_template_new_format_nem_regions(csv_str_to_df): assert costs_connection.empty # Custom constraints from PLEXOS are sub-regional export limits with no - # meaningful representation once sub-regions are collapsed, so the templater - # skips them at nem_regions granularity. The mock turns a regression in that - # gate into a clean assertion failure instead of a PLEXOS-extract disk read. + # meaningful representation once sub-regions are collapsed, so at + # nem_regions the templater emits them header-only instead of templating + # them. The mock turns a regression in that gate into a clean assertion + # failure instead of a PLEXOS-extract disk read. mock_template_custom_constraints.assert_not_called() - assert "custom_constraints" not in result - assert "custom_constraints_lhs" not in result - assert "custom_constraints_rhs" not in result + assert set(result["custom_constraints"].columns) == {"constraint_id", "direction"} + assert result["custom_constraints"].empty + assert set(result["custom_constraints_lhs"].columns) == { + "constraint_id", + "term_type", + "variable_name", + "coefficient", + "date_from", + } + assert result["custom_constraints_lhs"].empty + assert set(result["custom_constraints_rhs"].columns) == { + "constraint_id", + "timeslice", + "rhs", + "date_from", + } + assert result["custom_constraints_rhs"].empty def test_create_ispypsa_inputs_template_new_format_single_region(csv_str_to_df): @@ -570,6 +625,10 @@ def test_create_ispypsa_inputs_template_new_format_single_region(csv_str_to_df): "ispypsa.templater.create_template.template_custom_constraints_from_plexos", return_value=_stub_custom_constraints_tables(), ) as mock_template_custom_constraints, + patch( + "ispypsa.templater.create_template.load_timeslice_calendar", + return_value=_stub_timeslice_calendar(), + ), ): result = create_ispypsa_inputs_template( scenario="Step Change", @@ -590,6 +649,7 @@ def test_create_ispypsa_inputs_template_new_format_single_region(csv_str_to_df): }, manually_extracted_tables={ "connection_capacity_non_vre": connection_capacity_non_vre, + "reference_year_sequence": _stub_reference_year_sequence(), }, iasr_workbook_version="ignored-by-patch", ) @@ -630,10 +690,15 @@ def test_create_ispypsa_inputs_template_new_format_single_region(csv_str_to_df): assert connection_costs.empty # Custom constraints from PLEXOS are sub-regional export limits with no - # meaningful representation at single_region, so the templater skips them. - # The mock turns a regression in that gate into a clean assertion failure - # instead of a PLEXOS-extract disk read. + # meaningful representation at single_region, so the templater emits them + # header-only instead of templating them. The mock turns a regression in + # that gate into a clean assertion failure instead of a PLEXOS-extract + # disk read. mock_template_custom_constraints.assert_not_called() - assert "custom_constraints" not in result - assert "custom_constraints_lhs" not in result - assert "custom_constraints_rhs" not in result + for table in [ + "custom_constraints", + "custom_constraints_lhs", + "custom_constraints_rhs", + ]: + assert result[table].empty + assert len(result[table].columns) > 0 From 9cf41da01864434426d91faea147c245644f55b3 Mon Sep 17 00:00:00 2001 From: nick-gorman Date: Wed, 24 Jun 2026 13:08:11 +1000 Subject: [PATCH 2/6] Build the new-format templater test stubs with csv_str_to_df MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The three _stub_* helpers built their DataFrames by hand (dict-of-lists), the outliers in a file that otherwise uses the csv_str_to_df convention. Convert them so the fixtures read as row-oriented tables — the timeslice calendar especially, since test_timeslices.py already writes that exact shape with csv_str_to_df. The helpers are module-level, so the fixture is threaded in as a parameter rather than imported. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../test_create_ispypsa_inputs_template.py | 76 +++++++++---------- 1 file changed, 36 insertions(+), 40 deletions(-) diff --git a/tests/test_templater/test_create_ispypsa_inputs_template.py b/tests/test_templater/test_create_ispypsa_inputs_template.py index 7bf86f06..d5145137 100644 --- a/tests/test_templater/test_create_ispypsa_inputs_template.py +++ b/tests/test_templater/test_create_ispypsa_inputs_template.py @@ -23,7 +23,7 @@ ] -def _stub_custom_constraints_tables() -> dict[str, pd.DataFrame]: +def _stub_custom_constraints_tables(csv_str_to_df) -> dict[str, pd.DataFrame]: """Identifiable, non-empty stand-in for ``template_custom_constraints_from_plexos``. Has known row counts (1 constraint, 2 LHS terms, 1 RHS row). The sub_regions @@ -34,48 +34,44 @@ def _stub_custom_constraints_tables() -> dict[str, pd.DataFrame]: per-module tests in ``test_custom_constraints_from_plexos.py``. """ return { - "custom_constraints": pd.DataFrame( - {"constraint_id": ["SWQLD1"], "direction": ["<="]} - ), - "custom_constraints_lhs": pd.DataFrame( - { - "constraint_id": ["SWQLD1", "SWQLD1"], - "term_type": ["generator_output", "storage_output"], - "variable_name": ["BW01", "Q8 Battery - 2h"], - "coefficient": [0.5, 1.0], - "date_from": [pd.NA, pd.NA], - } - ), - "custom_constraints_rhs": pd.DataFrame( - { - "constraint_id": ["SWQLD1"], - "timeslice": ["qld_winter_reference"], - "rhs": [3000.0], - "date_from": [pd.NA], - } - ), + "custom_constraints": csv_str_to_df(""" + constraint_id, direction + SWQLD1, <= + """), + "custom_constraints_lhs": csv_str_to_df(""" + constraint_id, term_type, variable_name, coefficient, date_from + SWQLD1, generator_output, BW01, 0.5, + SWQLD1, storage_output, Q8 Battery - 2h, 1.0, + """), + "custom_constraints_rhs": csv_str_to_df(""" + constraint_id, timeslice, rhs, date_from + SWQLD1, qld_winter_reference, 3000.0, + """), } -def _stub_timeslice_calendar() -> pd.DataFrame: +def _stub_timeslice_calendar(csv_str_to_df) -> pd.DataFrame: """Minimal raw timeslice calendar: two windows that tile planning year FY2026 — a hot-day peak and a winter window that wraps back to it, so the decoded windows leave no day uncovered (the tiling invariant the templater enforces). Patched in place of the packaged PLEXOS calendar; full decode behaviour is covered by the per-module tests in ``test_timeslices.py``. """ - return pd.DataFrame( - { - "DATETIME": ["18/11/2025", "20/11/2025", "20/11/2025", "18/11/2026"], - "NAME": ["NSW Hot Day", "NSW Hot Day", "NSW Winter", "NSW Winter"], - "TIMESLICE": [-1, 0, -1, 0], - } - ) + return csv_str_to_df(""" + DATETIME, NAME, TIMESLICE + 18/11/2025, NSW Hot Day, -1 + 20/11/2025, NSW Hot Day, 0 + 20/11/2025, NSW Winter, -1 + 18/11/2026, NSW Winter, 0 + """) -def _stub_reference_year_sequence() -> pd.DataFrame: +def _stub_reference_year_sequence(csv_str_to_df) -> pd.DataFrame: """Assigns the stub calendar's single planning year a reference year.""" - return pd.DataFrame({"planning_year": [2026], "reference_year": [2015]}) + return csv_str_to_df(""" + planning_year, reference_year + 2026, 2015 + """) def test_list_templater_output_files_granularity_invariant(): @@ -256,11 +252,11 @@ def test_create_ispypsa_inputs_template_new_format(csv_str_to_df): ), patch( "ispypsa.templater.create_template.template_custom_constraints_from_plexos", - return_value=_stub_custom_constraints_tables(), + return_value=_stub_custom_constraints_tables(csv_str_to_df), ), patch( "ispypsa.templater.create_template.load_timeslice_calendar", - return_value=_stub_timeslice_calendar(), + return_value=_stub_timeslice_calendar(csv_str_to_df), ), ): result = create_ispypsa_inputs_template( @@ -287,7 +283,7 @@ def test_create_ispypsa_inputs_template_new_format(csv_str_to_df): # wiring runs. Output stays empty: generators/storage are placeholder-empty. manually_extracted_tables={ "connection_capacity_non_vre": connection_capacity_non_vre, - "reference_year_sequence": _stub_reference_year_sequence(), + "reference_year_sequence": _stub_reference_year_sequence(csv_str_to_df), }, iasr_workbook_version="ignored-by-patch", ) @@ -460,11 +456,11 @@ def test_create_ispypsa_inputs_template_new_format_nem_regions(csv_str_to_df): ), patch( "ispypsa.templater.create_template.template_custom_constraints_from_plexos", - return_value=_stub_custom_constraints_tables(), + return_value=_stub_custom_constraints_tables(csv_str_to_df), ) as mock_template_custom_constraints, patch( "ispypsa.templater.create_template.load_timeslice_calendar", - return_value=_stub_timeslice_calendar(), + return_value=_stub_timeslice_calendar(csv_str_to_df), ), ): result = create_ispypsa_inputs_template( @@ -488,7 +484,7 @@ def test_create_ispypsa_inputs_template_new_format_nem_regions(csv_str_to_df): }, manually_extracted_tables={ "connection_capacity_non_vre": connection_capacity_non_vre, - "reference_year_sequence": _stub_reference_year_sequence(), + "reference_year_sequence": _stub_reference_year_sequence(csv_str_to_df), }, iasr_workbook_version="ignored-by-patch", ) @@ -623,11 +619,11 @@ def test_create_ispypsa_inputs_template_new_format_single_region(csv_str_to_df): ), patch( "ispypsa.templater.create_template.template_custom_constraints_from_plexos", - return_value=_stub_custom_constraints_tables(), + return_value=_stub_custom_constraints_tables(csv_str_to_df), ) as mock_template_custom_constraints, patch( "ispypsa.templater.create_template.load_timeslice_calendar", - return_value=_stub_timeslice_calendar(), + return_value=_stub_timeslice_calendar(csv_str_to_df), ), ): result = create_ispypsa_inputs_template( @@ -649,7 +645,7 @@ def test_create_ispypsa_inputs_template_new_format_single_region(csv_str_to_df): }, manually_extracted_tables={ "connection_capacity_non_vre": connection_capacity_non_vre, - "reference_year_sequence": _stub_reference_year_sequence(), + "reference_year_sequence": _stub_reference_year_sequence(csv_str_to_df), }, iasr_workbook_version="ignored-by-patch", ) From 81c7bc294d01dd8fd5d56688981a2d76889cb568 Mon Sep 17 00:00:00 2001 From: nick-gorman Date: Wed, 24 Jun 2026 13:15:49 +1000 Subject: [PATCH 3/6] Check the coarse-granularity templater tests consistently The nem_regions and single_region tests both assert the new-format output at a collapsed granularity, but had diverged: single_region only checked the custom-constraint tables had *some* columns where nem_regions pinned the exact column set, and neither asserted timeslices at all. timeslices is granularity-invariant (decoded from the calendar + reference_year_sequence, independent of regional_granularity), so omitting it left the invariance this PR establishes unchecked at the coarse granularities. Pin the exact custom-constraint column sets in single_region and assert timeslices (two patterns, same as sub_regions) in both. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../test_create_ispypsa_inputs_template.py | 52 +++++++++++++++---- 1 file changed, 42 insertions(+), 10 deletions(-) diff --git a/tests/test_templater/test_create_ispypsa_inputs_template.py b/tests/test_templater/test_create_ispypsa_inputs_template.py index d5145137..be2a2317 100644 --- a/tests/test_templater/test_create_ispypsa_inputs_template.py +++ b/tests/test_templater/test_create_ispypsa_inputs_template.py @@ -515,6 +515,17 @@ def test_create_ispypsa_inputs_template_new_format_nem_regions(csv_str_to_df): assert set(expansion_costs["expansion_id"]) == {"NSW-QLD", "N3-NSW"} # 2 expansion_ids x 2 years assert len(expansion_costs) == 4 + + # timeslices is granularity-invariant — same two patterns as at sub_regions. + timeslices = result["timeslices"] + assert set(timeslices.columns) == { + "timeslice_id", + "reference_year", + "start_month_day", + "end_month_day", + } + assert len(timeslices) == 2 + costs_connection = result["costs_connection"] assert set(costs_connection.columns) == { "geo_id", @@ -675,15 +686,26 @@ def test_create_ispypsa_inputs_template_new_format_single_region(csv_str_to_df): assert set(expansion_costs["expansion_id"]) == {"N3-NEM"} # 1 expansion_id x 2 years assert len(expansion_costs) == 2 - connection_costs = result["costs_connection"] - assert set(connection_costs.columns) == { + + # timeslices is granularity-invariant — same two patterns as at sub_regions. + timeslices = result["timeslices"] + assert set(timeslices.columns) == { + "timeslice_id", + "reference_year", + "start_month_day", + "end_month_day", + } + assert len(timeslices) == 2 + + costs_connection = result["costs_connection"] + assert set(costs_connection.columns) == { "geo_id", "technology", "year", "connection_cost", "system_strength_cost", } - assert connection_costs.empty + assert costs_connection.empty # Custom constraints from PLEXOS are sub-regional export limits with no # meaningful representation at single_region, so the templater emits them @@ -691,10 +713,20 @@ def test_create_ispypsa_inputs_template_new_format_single_region(csv_str_to_df): # that gate into a clean assertion failure instead of a PLEXOS-extract # disk read. mock_template_custom_constraints.assert_not_called() - for table in [ - "custom_constraints", - "custom_constraints_lhs", - "custom_constraints_rhs", - ]: - assert result[table].empty - assert len(result[table].columns) > 0 + assert set(result["custom_constraints"].columns) == {"constraint_id", "direction"} + assert result["custom_constraints"].empty + assert set(result["custom_constraints_lhs"].columns) == { + "constraint_id", + "term_type", + "variable_name", + "coefficient", + "date_from", + } + assert result["custom_constraints_lhs"].empty + assert set(result["custom_constraints_rhs"].columns) == { + "constraint_id", + "timeslice", + "rhs", + "date_from", + } + assert result["custom_constraints_rhs"].empty From d0c60064cac5e26f328413a5555af775d6f11166 Mon Sep 17 00:00:00 2001 From: nick-gorman Date: Wed, 24 Jun 2026 13:38:39 +1000 Subject: [PATCH 4/6] Pin the CLI custom-constraint output's size The new-format CLI test pins a row count plus referential integrity for every output table except custom_constraints, which only checked that the LHS/RHS rows carried no orphan constraint_ids. Add a principled constraint count (one per CONSTRAINT_NAMES in scripts/extract_plexos_constraints.py) and a drift-detection count for the LHS coefficient rows, so a regression in the populated sub_regions output is caught here rather than only in the per-module tests. Co-Authored-By: Claude Opus 4.8 (1M context) --- ...create_ispypsa_inputs_new_table_formats.py | 27 ++++++++++++++++--- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/tests/test_cli/test_create_ispypsa_inputs_new_table_formats.py b/tests/test_cli/test_create_ispypsa_inputs_new_table_formats.py index 1de4d8da..b8cb41ee 100644 --- a/tests/test_cli/test_create_ispypsa_inputs_new_table_formats.py +++ b/tests/test_cli/test_create_ispypsa_inputs_new_table_formats.py @@ -297,6 +297,18 @@ def test_create_ispypsa_inputs_task_new_format( _NUM_TIMESLICE_IDS_75 = 14 _NUM_REFERENCE_YEARS_75 = 15 +# Custom constraints extracted from the PLEXOS model — one ISPyPSA constraint +# per entry in CONSTRAINT_NAMES in scripts/extract_plexos_constraints.py (the +# authoritative set: 14 export-group limits + 1 GPG constraint). All survive +# templating at sub_regions; coarser granularities collapse the sub-regional +# entities they reference, so the tables are emitted header-only there. +_NUM_CUSTOM_CONSTRAINTS_75 = 15 + +# Drift-detection: total LHS coefficient rows across all constraints. No clean +# formula — it is the per-constraint participant count in the PLEXOS model. +# Refresh by rerunning the test and pasting the failure value here. +_NUM_CUSTOM_CONSTRAINT_LHS_ROWS_75 = 1023 + @pytest.mark.parametrize("granularity", ["sub_regions", "nem_regions", "single_region"]) def test_create_ispypsa_inputs_new_format( @@ -374,15 +386,22 @@ def test_create_ispypsa_inputs_new_format( assert timeslices["reference_year"].nunique() == _NUM_REFERENCE_YEARS_75 # custom_constraints — templated only at sub_regions but written at every - # granularity (header-only at coarser ones). Detailed content is covered - # by test_custom_constraints_from_plexos.py; here we assert the CLI emits - # the three tables populated at sub_regions with no orphan LHS/RHS rows, - # and empty otherwise. + # granularity (header-only at coarser ones). Detailed content is covered by + # test_custom_constraints_from_plexos.py; here we pin the populated output's + # size (constraint count + LHS rows) and check no orphan LHS/RHS rows. + # Empty at coarser granularities. verify_output_files(output_dir, _CUSTOM_CONSTRAINT_OUTPUTS) constraints = pd.read_csv(output_dir / "custom_constraints.csv") lhs = pd.read_csv(output_dir / "custom_constraints_lhs.csv") rhs = pd.read_csv(output_dir / "custom_constraints_rhs.csv") if granularity == "sub_regions": + # One ISPyPSA constraint per PLEXOS constraint extracted (the + # CONSTRAINT_NAMES set in scripts/extract_plexos_constraints.py), plus a + # drift-detection count for the LHS coefficient rows. + assert constraints["constraint_id"].nunique() == _NUM_CUSTOM_CONSTRAINTS_75 + assert len(constraints) == _NUM_CUSTOM_CONSTRAINTS_75 + assert len(lhs) == _NUM_CUSTOM_CONSTRAINT_LHS_ROWS_75 + # No orphan LHS/RHS rows — every term and limit belongs to a constraint. constraint_ids = set(constraints["constraint_id"]) assert set(lhs["constraint_id"]) <= constraint_ids assert set(rhs["constraint_id"]) <= constraint_ids From b168aab2efe8349b962a0a6708e4bcd819b75dc8 Mon Sep 17 00:00:00 2001 From: nick-gorman Date: Wed, 24 Jun 2026 14:15:14 +1000 Subject: [PATCH 5/6] Project the constraint builders onto the shared column constants empty_custom_constraint_tables() declared the three tables' column sets as module constants, but the populated builders still hardcoded their own copies of the same lists. Nothing coupled the two, so a future change to a builder's output columns would silently diverge from the header-only tables emitted at coarse granularities, breaking the granularity-invariance this branch establishes. Project every builder onto the shared constants so the populated and empty tables have one definition, and split the CLI test's empty-case assertion so a failure names the offending table. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../custom_constraints_from_plexos.py | 29 ++++++------------- ...create_ispypsa_inputs_new_table_formats.py | 4 ++- 2 files changed, 12 insertions(+), 21 deletions(-) diff --git a/src/ispypsa/templater/custom_constraints_from_plexos.py b/src/ispypsa/templater/custom_constraints_from_plexos.py index 80a84e54..26e9783b 100644 --- a/src/ispypsa/templater/custom_constraints_from_plexos.py +++ b/src/ispypsa/templater/custom_constraints_from_plexos.py @@ -234,7 +234,10 @@ _EXCLUDED_PARENT_CLASSES = ("Purchaser",) _AREA_SUFFIX_PATTERN = re.compile(r" Area\d+$") -# Output columns of the three custom-constraint template tables. +# Output columns of the three custom-constraint template tables. Both the +# populated builders below and empty_custom_constraint_tables() project onto +# these, so the header-only tables emitted at coarse granularities carry +# exactly the same columns as the populated sub_regions tables. _CUSTOM_CONSTRAINTS_COLUMNS = ["constraint_id", "direction"] _CUSTOM_CONSTRAINTS_LHS_COLUMNS = [ "constraint_id", @@ -386,7 +389,7 @@ def _build_custom_constraints(constraints: pd.DataFrame) -> pd.DataFrame: ), "direction": directions, } - ).reset_index(drop=True) + )[_CUSTOM_CONSTRAINTS_COLUMNS].reset_index(drop=True) def _raise_on_unmapped_sense(sense_rows: pd.DataFrame, directions: pd.Series) -> None: @@ -455,9 +458,7 @@ def _build_custom_constraints_lhs( lhs = _drop_unresolved_terms(lhs) lhs = lhs.rename(columns={"value": "coefficient"}) lhs["constraint_id"] = lhs["constraint_name"].map(_strip_constraint_prefix) - lhs = lhs[ - ["constraint_id", "term_type", "variable_name", "coefficient", "date_from"] - ] + lhs = lhs[_CUSTOM_CONSTRAINTS_LHS_COLUMNS] lhs = _inject_iasr_new_entrant_batteries(lhs, new_entrants) lhs = _dedupe_lhs_terms(lhs) return lhs.reset_index(drop=True) @@ -1150,15 +1151,7 @@ def _battery_rows_for_triggers( # Short-circuit the empty case: an empty triggered carries a float64 # location column, which would raise on the object-keyed merge below. if triggered.empty: - return pd.DataFrame( - columns=[ - "constraint_id", - "term_type", - "variable_name", - "coefficient", - "date_from", - ] - ) + return pd.DataFrame(columns=_CUSTOM_CONSTRAINTS_LHS_COLUMNS) # Inner merge: expand each triggered (constraint_id, location) into one row # per new-entrant battery at that location. Triggered locations that hold no # batteries (location absent from batteries_by_location) drop out here. @@ -1173,9 +1166,7 @@ def _battery_rows_for_triggers( ) rows["coefficient"] = rows["coefficient"].fillna(1.0) rows["term_type"] = "storage_output" - return rows[ - ["constraint_id", "term_type", "variable_name", "coefficient", "date_from"] - ].reset_index(drop=True) + return rows[_CUSTOM_CONSTRAINTS_LHS_COLUMNS].reset_index(drop=True) def _log_injected_batteries(injected: pd.DataFrame) -> None: @@ -1256,9 +1247,7 @@ def _build_custom_constraints_rhs(rhs_values: pd.DataFrame) -> pd.DataFrame: rhs["constraint_id"] = rhs["constraint_name"].map(_strip_constraint_prefix) rhs["timeslice"] = rhs["tags"].map(_tag_to_timeslice) rhs = rhs.rename(columns={"value": "rhs"}) - return rhs[["constraint_id", "timeslice", "rhs", "date_from"]].reset_index( - drop=True - ) + return rhs[_CUSTOM_CONSTRAINTS_RHS_COLUMNS].reset_index(drop=True) def _tag_to_timeslice(tag: str) -> str: diff --git a/tests/test_cli/test_create_ispypsa_inputs_new_table_formats.py b/tests/test_cli/test_create_ispypsa_inputs_new_table_formats.py index b8cb41ee..55ebe973 100644 --- a/tests/test_cli/test_create_ispypsa_inputs_new_table_formats.py +++ b/tests/test_cli/test_create_ispypsa_inputs_new_table_formats.py @@ -406,4 +406,6 @@ def test_create_ispypsa_inputs_new_format( assert set(lhs["constraint_id"]) <= constraint_ids assert set(rhs["constraint_id"]) <= constraint_ids else: - assert constraints.empty and lhs.empty and rhs.empty + assert constraints.empty + assert lhs.empty + assert rhs.empty From ab64e0af3fa94bd420edf43bfaebc51887110b57 Mon Sep 17 00:00:00 2001 From: nick-gorman Date: Wed, 24 Jun 2026 14:17:13 +1000 Subject: [PATCH 6/6] Drop the redundant header-only rationale comment The same explanation now lives in empty_custom_constraint_tables()'s docstring, which the else-branch calls. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/ispypsa/templater/create_template.py | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/ispypsa/templater/create_template.py b/src/ispypsa/templater/create_template.py index 2861c85d..c4ea7327 100644 --- a/src/ispypsa/templater/create_template.py +++ b/src/ispypsa/templater/create_template.py @@ -245,13 +245,7 @@ def create_ispypsa_inputs_template( storage_new_entrant, sub_regional_geography, ) - # Custom constraints from PLEXOS are sub-regional export-group limits: - # their LHS references sub-region nodes, sub-regional flow paths, and - # REZ-located units that only exist as distinct entities at sub_regions - # granularity. Once sub-regions are collapsed (nem_regions / - # single_region) they have no meaningful representation, so the tables - # are emitted header-only ("all columns, no rows") there, keeping the - # output set granularity-invariant. + if regional_granularity == "sub_regions": template.update( template_custom_constraints_from_plexos(