Make the new-format templater output set granularity-invariant

src/ispypsa/templater/create_template.py

@@ -6,6 +6,7 @@
 from ispypsa.feature_flags import FEATURE_FLAGS
 from ispypsa.templater.connection_and_build_costs import _template_connection_costs
 from ispypsa.templater.custom_constraints_from_plexos import (
+    empty_custom_constraint_tables,
     template_custom_constraints_from_plexos,
 )
 from ispypsa.templater.dynamic_generator_properties import (
@@ -44,6 +45,10 @@
     _template_new_generators_static_properties,
 )
 from ispypsa.templater.storage import _template_battery_properties
+from ispypsa.templater.timeslices import (
+    _template_timeslices,
+    load_timeslice_calendar,
+)
 from ispypsa.templater.transmission import _template_network_transmission
 
 _BASE_TEMPLATE_OUTPUTS = [
@@ -74,8 +79,9 @@
 ]
 
 # Outputs from the new-format templater branch. Granularity-invariant: the
-# same five tables are emitted for sub_regions, nem_regions, and single_region
-# (only their contents differ).
+# same tables are emitted for sub_regions, nem_regions, and single_region
+# (only their contents differ — the custom-constraint tables are header-only
+# at coarser granularities).
 # FEATURE_FLAG_CLEANUP[use_new_table_format]: rename to _TEMPLATE_OUTPUTS and
 # delete _BASE_TEMPLATE_OUTPUTS above.
 _NEW_FORMAT_TEMPLATE_OUTPUTS = [
@@ -84,15 +90,8 @@
     "network_transmission_path_limits",
     "network_expansion_options",
     "network_transmission_path_expansion_costs",
-]
-
-# Custom constraints are templated only at sub_regions granularity (see the gate
-# in create_ispypsa_inputs_template) — coarser granularities collapse the
-# sub-region nodes, sub-regional flow paths and REZ-located units the constraints
-# reference, leaving nothing to constrain. Listed as outputs only at that
-# granularity so the create_ispypsa_inputs task tracks them where they are
-# written and does not expect them where they never are.
-_CUSTOM_CONSTRAINT_OUTPUTS = [
+    "timeslices",
+    "costs_connection",
     "custom_constraints",
     "custom_constraints_lhs",
     "custom_constraints_rhs",
@@ -219,6 +218,10 @@ def create_ispypsa_inputs_template(
         )
         template["network_expansion_options"] = expansion_options
         template["network_transmission_path_expansion_costs"] = expansion_costs
+        template["timeslices"] = _template_timeslices(
+            load_timeslice_calendar(iasr_workbook_version),
+            manually_extracted_tables["reference_year_sequence"],
+        )
 
         # todo: replace with actual generators_new_entrant once that templating
         # function is written — passing empty placeholder for now so costs_connection
@@ -242,18 +245,15 @@ def create_ispypsa_inputs_template(
             storage_new_entrant,
             sub_regional_geography,
         )
-        # Custom constraints from PLEXOS are sub-regional export-group limits:
-        # their LHS references sub-region nodes, sub-regional flow paths, and
-        # REZ-located units that only exist as distinct entities at sub_regions
-        # granularity. Once sub-regions are collapsed (nem_regions /
-        # single_region) they have no meaningful representation, so only emit
-        # them for sub_regions.
+
         if regional_granularity == "sub_regions":
             template.update(
                 template_custom_constraints_from_plexos(
                     iasr_tables, iasr_workbook_version=iasr_workbook_version
                 )
             )
+        else:
+            template.update(empty_custom_constraint_tables())
         return template
 
     template = {}
@@ -347,8 +347,6 @@ def list_templater_output_files(regional_granularity, output_path=None):
     # granularity-specific file removals.
     if FEATURE_FLAGS["use_new_table_format"]:
         files = _NEW_FORMAT_TEMPLATE_OUTPUTS.copy()
-        if regional_granularity == "sub_regions":
-            files += _CUSTOM_CONSTRAINT_OUTPUTS
     else:
         files = _BASE_TEMPLATE_OUTPUTS.copy()
         if regional_granularity in ["sub_regions", "single_region"]:

src/ispypsa/templater/custom_constraints_from_plexos.py

@@ -234,6 +234,37 @@
 _EXCLUDED_PARENT_CLASSES = ("Purchaser",)
 _AREA_SUFFIX_PATTERN = re.compile(r" Area\d+$")
 
+# Output columns of the three custom-constraint template tables. Both the
+# populated builders below and empty_custom_constraint_tables() project onto
+# these, so the header-only tables emitted at coarse granularities carry
+# exactly the same columns as the populated sub_regions tables.
+_CUSTOM_CONSTRAINTS_COLUMNS = ["constraint_id", "direction"]
+_CUSTOM_CONSTRAINTS_LHS_COLUMNS = [
+    "constraint_id",
+    "term_type",
+    "variable_name",
+    "coefficient",
+    "date_from",
+]
+_CUSTOM_CONSTRAINTS_RHS_COLUMNS = ["constraint_id", "timeslice", "rhs", "date_from"]
+
+
+def empty_custom_constraint_tables() -> dict[str, pd.DataFrame]:
+    """Header-only versions of the three custom-constraint template tables.
+
+    Emitted at nem_regions / single_region granularity, where the sub-region
+    nodes, sub-regional flow paths, and REZ-located units the constraints
+    reference are collapsed, leaving nothing to constrain. Emitting "all
+    columns, no rows" tables keeps the templater's output set
+    granularity-invariant, so downstream consumers never need to check for
+    missing tables.
+    """
+    return {
+        "custom_constraints": pd.DataFrame(columns=_CUSTOM_CONSTRAINTS_COLUMNS),
+        "custom_constraints_lhs": pd.DataFrame(columns=_CUSTOM_CONSTRAINTS_LHS_COLUMNS),
+        "custom_constraints_rhs": pd.DataFrame(columns=_CUSTOM_CONSTRAINTS_RHS_COLUMNS),
+    }
+
 
 def template_custom_constraints_from_plexos(
     iasr_tables: dict[str, pd.DataFrame],
@@ -358,7 +389,7 @@ def _build_custom_constraints(constraints: pd.DataFrame) -> pd.DataFrame:
             ),
             "direction": directions,
         }
-    ).reset_index(drop=True)
+    )[_CUSTOM_CONSTRAINTS_COLUMNS].reset_index(drop=True)
 
 
 def _raise_on_unmapped_sense(sense_rows: pd.DataFrame, directions: pd.Series) -> None:
@@ -427,9 +458,7 @@ def _build_custom_constraints_lhs(
     lhs = _drop_unresolved_terms(lhs)
     lhs = lhs.rename(columns={"value": "coefficient"})
     lhs["constraint_id"] = lhs["constraint_name"].map(_strip_constraint_prefix)
-    lhs = lhs[
-        ["constraint_id", "term_type", "variable_name", "coefficient", "date_from"]
-    ]
+    lhs = lhs[_CUSTOM_CONSTRAINTS_LHS_COLUMNS]
     lhs = _inject_iasr_new_entrant_batteries(lhs, new_entrants)
     lhs = _dedupe_lhs_terms(lhs)
     return lhs.reset_index(drop=True)
@@ -1122,15 +1151,7 @@ def _battery_rows_for_triggers(
     # Short-circuit the empty case: an empty triggered carries a float64
     # location column, which would raise on the object-keyed merge below.
     if triggered.empty:
-        return pd.DataFrame(
-            columns=[
-                "constraint_id",
-                "term_type",
-                "variable_name",
-                "coefficient",
-                "date_from",
-            ]
-        )
+        return pd.DataFrame(columns=_CUSTOM_CONSTRAINTS_LHS_COLUMNS)
     # Inner merge: expand each triggered (constraint_id, location) into one row
     # per new-entrant battery at that location. Triggered locations that hold no
     # batteries (location absent from batteries_by_location) drop out here.
@@ -1145,9 +1166,7 @@ def _battery_rows_for_triggers(
     )
     rows["coefficient"] = rows["coefficient"].fillna(1.0)
     rows["term_type"] = "storage_output"
-    return rows[
-        ["constraint_id", "term_type", "variable_name", "coefficient", "date_from"]
-    ].reset_index(drop=True)
+    return rows[_CUSTOM_CONSTRAINTS_LHS_COLUMNS].reset_index(drop=True)
 
 
 def _log_injected_batteries(injected: pd.DataFrame) -> None:
@@ -1228,9 +1247,7 @@ def _build_custom_constraints_rhs(rhs_values: pd.DataFrame) -> pd.DataFrame:
     rhs["constraint_id"] = rhs["constraint_name"].map(_strip_constraint_prefix)
     rhs["timeslice"] = rhs["tags"].map(_tag_to_timeslice)
     rhs = rhs.rename(columns={"value": "rhs"})
-    return rhs[["constraint_id", "timeslice", "rhs", "date_from"]].reset_index(
-        drop=True
-    )
+    return rhs[_CUSTOM_CONSTRAINTS_RHS_COLUMNS].reset_index(drop=True)
 
 
 def _tag_to_timeslice(tag: str) -> str:

tests/test_cli/test_create_ispypsa_inputs_new_table_formats.py

@@ -71,13 +71,15 @@
     "network_transmission_path_limits",
     "network_expansion_options",
     "network_transmission_path_expansion_costs",
+    "timeslices",
     "costs_connection",
 ]
 
 # Custom constraints are templated only at sub_regions (coarser granularities
-# collapse the entities they reference). Detailed content lives in
-# test_custom_constraints_from_plexos.py; here we check the CLI writes them at
-# sub_regions and omits them otherwise.
+# collapse the entities they reference) but written at every granularity —
+# header-only at nem_regions / single_region. Detailed content lives in
+# test_custom_constraints_from_plexos.py; here we check the CLI writes them
+# populated at sub_regions and empty otherwise.
 _CUSTOM_CONSTRAINT_OUTPUTS = [
     "custom_constraints",
     "custom_constraints_lhs",
@@ -286,6 +288,27 @@ def test_create_ispypsa_inputs_task_new_format(
     "single_region": 1209,
 }
 
+# Per-reference-year window patterns decoded from the shipped RefYear5000
+# calendar and reference_year_sequence (drift-detection;
+# granularity-invariant). 14 distinct ids, not 15: TAS Hot Day never
+# activates in the calendar, so tas_peak_demand has no windows. 15 reference
+# years: 2011 to 2025 in year-ending convention.
+_NUM_TIMESLICE_PATTERN_ROWS_75 = 678
+_NUM_TIMESLICE_IDS_75 = 14
+_NUM_REFERENCE_YEARS_75 = 15
+
+# Custom constraints extracted from the PLEXOS model — one ISPyPSA constraint
+# per entry in CONSTRAINT_NAMES in scripts/extract_plexos_constraints.py (the
+# authoritative set: 14 export-group limits + 1 GPG constraint). All survive
+# templating at sub_regions; coarser granularities collapse the sub-regional
+# entities they reference, so the tables are emitted header-only there.
+_NUM_CUSTOM_CONSTRAINTS_75 = 15
+
+# Drift-detection: total LHS coefficient rows across all constraints. No clean
+# formula — it is the per-constraint participant count in the PLEXOS model.
+# Refresh by rerunning the test and pasting the failure value here.
+_NUM_CUSTOM_CONSTRAINT_LHS_ROWS_75 = 1023
+
 
 @pytest.mark.parametrize("granularity", ["sub_regions", "nem_regions", "single_region"])
 def test_create_ispypsa_inputs_new_format(
@@ -355,18 +378,34 @@ def test_create_ispypsa_inputs_new_format(
     assert set(costs["expansion_id"]) == set(options["expansion_id"])
     assert len(costs) == _EXPECTED_EXPANSION_COST_ROWS_75[granularity]
 
-    # custom_constraints — written only at sub_regions. Detailed content is
-    # covered by test_custom_constraints_from_plexos.py; here we assert the CLI
-    # emits the three tables at sub_regions with no orphan LHS/RHS rows, and
-    # omits them entirely at coarser granularities.
+    # timeslices — per-reference-year patterns decoded from the shipped
+    # RefYear5000 calendar, identical at every granularity.
+    timeslices = pd.read_csv(output_dir / "timeslices.csv")
+    assert len(timeslices) == _NUM_TIMESLICE_PATTERN_ROWS_75
+    assert timeslices["timeslice_id"].nunique() == _NUM_TIMESLICE_IDS_75
+    assert timeslices["reference_year"].nunique() == _NUM_REFERENCE_YEARS_75
+
+    # custom_constraints — templated only at sub_regions but written at every
+    # granularity (header-only at coarser ones). Detailed content is covered by
+    # test_custom_constraints_from_plexos.py; here we pin the populated output's
+    # size (constraint count + LHS rows) and check no orphan LHS/RHS rows.
+    # Empty at coarser granularities.
+    verify_output_files(output_dir, _CUSTOM_CONSTRAINT_OUTPUTS)
+    constraints = pd.read_csv(output_dir / "custom_constraints.csv")
+    lhs = pd.read_csv(output_dir / "custom_constraints_lhs.csv")
+    rhs = pd.read_csv(output_dir / "custom_constraints_rhs.csv")
     if granularity == "sub_regions":
-        verify_output_files(output_dir, _CUSTOM_CONSTRAINT_OUTPUTS)
-        constraints = pd.read_csv(output_dir / "custom_constraints.csv")
-        lhs = pd.read_csv(output_dir / "custom_constraints_lhs.csv")
-        rhs = pd.read_csv(output_dir / "custom_constraints_rhs.csv")
+        # One ISPyPSA constraint per PLEXOS constraint extracted (the
+        # CONSTRAINT_NAMES set in scripts/extract_plexos_constraints.py), plus a
+        # drift-detection count for the LHS coefficient rows.
+        assert constraints["constraint_id"].nunique() == _NUM_CUSTOM_CONSTRAINTS_75
+        assert len(constraints) == _NUM_CUSTOM_CONSTRAINTS_75
+        assert len(lhs) == _NUM_CUSTOM_CONSTRAINT_LHS_ROWS_75
+        # No orphan LHS/RHS rows — every term and limit belongs to a constraint.
         constraint_ids = set(constraints["constraint_id"])
         assert set(lhs["constraint_id"]) <= constraint_ids
         assert set(rhs["constraint_id"]) <= constraint_ids
     else:
-        for name in _CUSTOM_CONSTRAINT_OUTPUTS:
-            assert not (output_dir / f"{name}.csv").exists()
+        assert constraints.empty
+        assert lhs.empty
+        assert rhs.empty