From ce8575f450dc2e8387056512660b08479a30aa33 Mon Sep 17 00:00:00 2001 From: nick-gorman Date: Thu, 11 Jun 2026 12:25:50 +1000 Subject: [PATCH 1/6] Make the new-format templater output set granularity-invariant MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The new-format templater now emits the same table set at every granularity: timeslices and costs_connection are wired in, and the custom-constraint tables are emitted header-only at nem_regions / single_region instead of being absent. This reverses the gating from 9366945 — the PLEXOS constraints are sub-regional export-group limits with no meaningful representation once sub-regions are collapsed, but writing "all columns, no rows" tables means list_templater_output_files needs no granularity awareness and downstream consumers never check for missing tables. costs_connection is also now tracked in the output list — it was being written but untracked, the same class of bug 9366945 fixed for the constraint tables. Co-Authored-By: Claude Fable 5 --- src/ispypsa/templater/create_template.py | 34 ++--- .../custom_constraints_from_plexos.py | 28 +++++ ...create_ispypsa_inputs_new_table_formats.py | 44 +++++-- .../test_create_ispypsa_inputs_template.py | 118 +++++++++++++----- 4 files changed, 168 insertions(+), 56 deletions(-) diff --git a/src/ispypsa/templater/create_template.py b/src/ispypsa/templater/create_template.py index bf26486a..2861c85d 100644 --- a/src/ispypsa/templater/create_template.py +++ b/src/ispypsa/templater/create_template.py @@ -6,6 +6,7 @@ from ispypsa.feature_flags import FEATURE_FLAGS from ispypsa.templater.connection_and_build_costs import _template_connection_costs from ispypsa.templater.custom_constraints_from_plexos import ( + empty_custom_constraint_tables, template_custom_constraints_from_plexos, ) from ispypsa.templater.dynamic_generator_properties import ( @@ -44,6 +45,10 @@ _template_new_generators_static_properties, ) from ispypsa.templater.storage import _template_battery_properties +from ispypsa.templater.timeslices import ( + _template_timeslices, + load_timeslice_calendar, +) from ispypsa.templater.transmission import _template_network_transmission _BASE_TEMPLATE_OUTPUTS = [ @@ -74,8 +79,9 @@ ] # Outputs from the new-format templater branch. Granularity-invariant: the -# same five tables are emitted for sub_regions, nem_regions, and single_region -# (only their contents differ). +# same tables are emitted for sub_regions, nem_regions, and single_region +# (only their contents differ — the custom-constraint tables are header-only +# at coarser granularities). # FEATURE_FLAG_CLEANUP[use_new_table_format]: rename to _TEMPLATE_OUTPUTS and # delete _BASE_TEMPLATE_OUTPUTS above. _NEW_FORMAT_TEMPLATE_OUTPUTS = [ @@ -84,15 +90,8 @@ "network_transmission_path_limits", "network_expansion_options", "network_transmission_path_expansion_costs", -] - -# Custom constraints are templated only at sub_regions granularity (see the gate -# in create_ispypsa_inputs_template) — coarser granularities collapse the -# sub-region nodes, sub-regional flow paths and REZ-located units the constraints -# reference, leaving nothing to constrain. Listed as outputs only at that -# granularity so the create_ispypsa_inputs task tracks them where they are -# written and does not expect them where they never are. -_CUSTOM_CONSTRAINT_OUTPUTS = [ + "timeslices", + "costs_connection", "custom_constraints", "custom_constraints_lhs", "custom_constraints_rhs", @@ -219,6 +218,10 @@ def create_ispypsa_inputs_template( ) template["network_expansion_options"] = expansion_options template["network_transmission_path_expansion_costs"] = expansion_costs + template["timeslices"] = _template_timeslices( + load_timeslice_calendar(iasr_workbook_version), + manually_extracted_tables["reference_year_sequence"], + ) # todo: replace with actual generators_new_entrant once that templating # function is written — passing empty placeholder for now so costs_connection @@ -246,14 +249,17 @@ def create_ispypsa_inputs_template( # their LHS references sub-region nodes, sub-regional flow paths, and # REZ-located units that only exist as distinct entities at sub_regions # granularity. Once sub-regions are collapsed (nem_regions / - # single_region) they have no meaningful representation, so only emit - # them for sub_regions. + # single_region) they have no meaningful representation, so the tables + # are emitted header-only ("all columns, no rows") there, keeping the + # output set granularity-invariant. if regional_granularity == "sub_regions": template.update( template_custom_constraints_from_plexos( iasr_tables, iasr_workbook_version=iasr_workbook_version ) ) + else: + template.update(empty_custom_constraint_tables()) return template template = {} @@ -347,8 +353,6 @@ def list_templater_output_files(regional_granularity, output_path=None): # granularity-specific file removals. if FEATURE_FLAGS["use_new_table_format"]: files = _NEW_FORMAT_TEMPLATE_OUTPUTS.copy() - if regional_granularity == "sub_regions": - files += _CUSTOM_CONSTRAINT_OUTPUTS else: files = _BASE_TEMPLATE_OUTPUTS.copy() if regional_granularity in ["sub_regions", "single_region"]: diff --git a/src/ispypsa/templater/custom_constraints_from_plexos.py b/src/ispypsa/templater/custom_constraints_from_plexos.py index c2a4672e..80a84e54 100644 --- a/src/ispypsa/templater/custom_constraints_from_plexos.py +++ b/src/ispypsa/templater/custom_constraints_from_plexos.py @@ -234,6 +234,34 @@ _EXCLUDED_PARENT_CLASSES = ("Purchaser",) _AREA_SUFFIX_PATTERN = re.compile(r" Area\d+$") +# Output columns of the three custom-constraint template tables. +_CUSTOM_CONSTRAINTS_COLUMNS = ["constraint_id", "direction"] +_CUSTOM_CONSTRAINTS_LHS_COLUMNS = [ + "constraint_id", + "term_type", + "variable_name", + "coefficient", + "date_from", +] +_CUSTOM_CONSTRAINTS_RHS_COLUMNS = ["constraint_id", "timeslice", "rhs", "date_from"] + + +def empty_custom_constraint_tables() -> dict[str, pd.DataFrame]: + """Header-only versions of the three custom-constraint template tables. + + Emitted at nem_regions / single_region granularity, where the sub-region + nodes, sub-regional flow paths, and REZ-located units the constraints + reference are collapsed, leaving nothing to constrain. Emitting "all + columns, no rows" tables keeps the templater's output set + granularity-invariant, so downstream consumers never need to check for + missing tables. + """ + return { + "custom_constraints": pd.DataFrame(columns=_CUSTOM_CONSTRAINTS_COLUMNS), + "custom_constraints_lhs": pd.DataFrame(columns=_CUSTOM_CONSTRAINTS_LHS_COLUMNS), + "custom_constraints_rhs": pd.DataFrame(columns=_CUSTOM_CONSTRAINTS_RHS_COLUMNS), + } + def template_custom_constraints_from_plexos( iasr_tables: dict[str, pd.DataFrame], diff --git a/tests/test_cli/test_create_ispypsa_inputs_new_table_formats.py b/tests/test_cli/test_create_ispypsa_inputs_new_table_formats.py index 79a967b4..1de4d8da 100644 --- a/tests/test_cli/test_create_ispypsa_inputs_new_table_formats.py +++ b/tests/test_cli/test_create_ispypsa_inputs_new_table_formats.py @@ -71,13 +71,15 @@ "network_transmission_path_limits", "network_expansion_options", "network_transmission_path_expansion_costs", + "timeslices", "costs_connection", ] # Custom constraints are templated only at sub_regions (coarser granularities -# collapse the entities they reference). Detailed content lives in -# test_custom_constraints_from_plexos.py; here we check the CLI writes them at -# sub_regions and omits them otherwise. +# collapse the entities they reference) but written at every granularity — +# header-only at nem_regions / single_region. Detailed content lives in +# test_custom_constraints_from_plexos.py; here we check the CLI writes them +# populated at sub_regions and empty otherwise. _CUSTOM_CONSTRAINT_OUTPUTS = [ "custom_constraints", "custom_constraints_lhs", @@ -286,6 +288,15 @@ def test_create_ispypsa_inputs_task_new_format( "single_region": 1209, } +# Per-reference-year window patterns decoded from the shipped RefYear5000 +# calendar and reference_year_sequence (drift-detection; +# granularity-invariant). 14 distinct ids, not 15: TAS Hot Day never +# activates in the calendar, so tas_peak_demand has no windows. 15 reference +# years: 2011 to 2025 in year-ending convention. +_NUM_TIMESLICE_PATTERN_ROWS_75 = 678 +_NUM_TIMESLICE_IDS_75 = 14 +_NUM_REFERENCE_YEARS_75 = 15 + @pytest.mark.parametrize("granularity", ["sub_regions", "nem_regions", "single_region"]) def test_create_ispypsa_inputs_new_format( @@ -355,18 +366,25 @@ def test_create_ispypsa_inputs_new_format( assert set(costs["expansion_id"]) == set(options["expansion_id"]) assert len(costs) == _EXPECTED_EXPANSION_COST_ROWS_75[granularity] - # custom_constraints — written only at sub_regions. Detailed content is - # covered by test_custom_constraints_from_plexos.py; here we assert the CLI - # emits the three tables at sub_regions with no orphan LHS/RHS rows, and - # omits them entirely at coarser granularities. + # timeslices — per-reference-year patterns decoded from the shipped + # RefYear5000 calendar, identical at every granularity. + timeslices = pd.read_csv(output_dir / "timeslices.csv") + assert len(timeslices) == _NUM_TIMESLICE_PATTERN_ROWS_75 + assert timeslices["timeslice_id"].nunique() == _NUM_TIMESLICE_IDS_75 + assert timeslices["reference_year"].nunique() == _NUM_REFERENCE_YEARS_75 + + # custom_constraints — templated only at sub_regions but written at every + # granularity (header-only at coarser ones). Detailed content is covered + # by test_custom_constraints_from_plexos.py; here we assert the CLI emits + # the three tables populated at sub_regions with no orphan LHS/RHS rows, + # and empty otherwise. + verify_output_files(output_dir, _CUSTOM_CONSTRAINT_OUTPUTS) + constraints = pd.read_csv(output_dir / "custom_constraints.csv") + lhs = pd.read_csv(output_dir / "custom_constraints_lhs.csv") + rhs = pd.read_csv(output_dir / "custom_constraints_rhs.csv") if granularity == "sub_regions": - verify_output_files(output_dir, _CUSTOM_CONSTRAINT_OUTPUTS) - constraints = pd.read_csv(output_dir / "custom_constraints.csv") - lhs = pd.read_csv(output_dir / "custom_constraints_lhs.csv") - rhs = pd.read_csv(output_dir / "custom_constraints_rhs.csv") constraint_ids = set(constraints["constraint_id"]) assert set(lhs["constraint_id"]) <= constraint_ids assert set(rhs["constraint_id"]) <= constraint_ids else: - for name in _CUSTOM_CONSTRAINT_OUTPUTS: - assert not (output_dir / f"{name}.csv").exists() + assert constraints.empty and lhs.empty and rhs.empty diff --git a/tests/test_templater/test_create_ispypsa_inputs_template.py b/tests/test_templater/test_create_ispypsa_inputs_template.py index 9675f1ed..ecc04dcc 100644 --- a/tests/test_templater/test_create_ispypsa_inputs_template.py +++ b/tests/test_templater/test_create_ispypsa_inputs_template.py @@ -57,21 +57,33 @@ def _stub_custom_constraints_tables() -> dict[str, pd.DataFrame]: } -def test_list_templater_output_files_includes_custom_constraints_only_at_sub_regions(): - """Custom-constraint tables are declared as task outputs only at sub_regions. - - They are templated (and written) only at sub_regions, so they must appear in - the output-file list there — otherwise the create_ispypsa_inputs task leaves - files it writes untracked — and must be absent at nem_regions / - single_region, where declaring them would make the task expect files that are - never written. +def _stub_timeslice_calendar() -> pd.DataFrame: + """Minimal raw timeslice calendar: one closed window in planning year + FY2026. Patched in place of the packaged PLEXOS calendar; full decode + behaviour is covered by the per-module tests in ``test_timeslices.py``. """ - custom_constraint_files = { - "custom_constraints", - "custom_constraints_lhs", - "custom_constraints_rhs", - } + return pd.DataFrame( + { + "DATETIME": ["18/11/2025", "20/11/2025"], + "NAME": ["NSW Hot Day", "NSW Hot Day"], + "TIMESLICE": [-1, 0], + } + ) + + +def _stub_reference_year_sequence() -> pd.DataFrame: + """Assigns the stub calendar's single planning year a reference year.""" + return pd.DataFrame({"planning_year": [2026], "reference_year": [2015]}) + + +def test_list_templater_output_files_granularity_invariant(): + """The new-format templater declares the same output set at every granularity. + The custom-constraint tables are header-only at nem_regions / + single_region but still written, so the create_ispypsa_inputs task must + track them everywhere — a granularity-dependent list would leave written + files untracked or expect files that are never written. + """ with patch( "ispypsa.templater.create_template.FEATURE_FLAGS", {"use_new_table_format": True}, @@ -80,9 +92,14 @@ def test_list_templater_output_files_includes_custom_constraints_only_at_sub_reg nem_regions = set(list_templater_output_files("nem_regions")) single_region = set(list_templater_output_files("single_region")) - assert custom_constraint_files <= sub_regions - assert custom_constraint_files.isdisjoint(nem_regions) - assert custom_constraint_files.isdisjoint(single_region) + assert sub_regions == nem_regions == single_region + assert { + "custom_constraints", + "custom_constraints_lhs", + "custom_constraints_rhs", + "timeslices", + "costs_connection", + } <= sub_regions def test_create_ispypsa_inputs_template_sub_regions( @@ -239,6 +256,10 @@ def test_create_ispypsa_inputs_template_new_format(csv_str_to_df): "ispypsa.templater.create_template.template_custom_constraints_from_plexos", return_value=_stub_custom_constraints_tables(), ), + patch( + "ispypsa.templater.create_template.load_timeslice_calendar", + return_value=_stub_timeslice_calendar(), + ), ): result = create_ispypsa_inputs_template( scenario="Step Change", @@ -264,6 +285,7 @@ def test_create_ispypsa_inputs_template_new_format(csv_str_to_df): # wiring runs. Output stays empty: generators/storage are placeholder-empty. manually_extracted_tables={ "connection_capacity_non_vre": connection_capacity_non_vre, + "reference_year_sequence": _stub_reference_year_sequence(), }, iasr_workbook_version="ignored-by-patch", ) @@ -304,6 +326,16 @@ def test_create_ispypsa_inputs_template_new_format(csv_str_to_df): # 3 expansion_ids x 2 years assert len(expansion_costs) == 6 + timeslices = result["timeslices"] + assert set(timeslices.columns) == { + "timeslice_id", + "reference_year", + "start_month_day", + "end_month_day", + } + # The stub calendar's single on/off pair decodes into one window pattern. + assert len(timeslices) == 1 + assert "costs_connection" in result costs_connection = result["costs_connection"] assert set(costs_connection.columns) == { @@ -427,6 +459,10 @@ def test_create_ispypsa_inputs_template_new_format_nem_regions(csv_str_to_df): "ispypsa.templater.create_template.template_custom_constraints_from_plexos", return_value=_stub_custom_constraints_tables(), ) as mock_template_custom_constraints, + patch( + "ispypsa.templater.create_template.load_timeslice_calendar", + return_value=_stub_timeslice_calendar(), + ), ): result = create_ispypsa_inputs_template( scenario="Step Change", @@ -449,6 +485,7 @@ def test_create_ispypsa_inputs_template_new_format_nem_regions(csv_str_to_df): }, manually_extracted_tables={ "connection_capacity_non_vre": connection_capacity_non_vre, + "reference_year_sequence": _stub_reference_year_sequence(), }, iasr_workbook_version="ignored-by-patch", ) @@ -490,13 +527,28 @@ def test_create_ispypsa_inputs_template_new_format_nem_regions(csv_str_to_df): assert costs_connection.empty # Custom constraints from PLEXOS are sub-regional export limits with no - # meaningful representation once sub-regions are collapsed, so the templater - # skips them at nem_regions granularity. The mock turns a regression in that - # gate into a clean assertion failure instead of a PLEXOS-extract disk read. + # meaningful representation once sub-regions are collapsed, so at + # nem_regions the templater emits them header-only instead of templating + # them. The mock turns a regression in that gate into a clean assertion + # failure instead of a PLEXOS-extract disk read. mock_template_custom_constraints.assert_not_called() - assert "custom_constraints" not in result - assert "custom_constraints_lhs" not in result - assert "custom_constraints_rhs" not in result + assert set(result["custom_constraints"].columns) == {"constraint_id", "direction"} + assert result["custom_constraints"].empty + assert set(result["custom_constraints_lhs"].columns) == { + "constraint_id", + "term_type", + "variable_name", + "coefficient", + "date_from", + } + assert result["custom_constraints_lhs"].empty + assert set(result["custom_constraints_rhs"].columns) == { + "constraint_id", + "timeslice", + "rhs", + "date_from", + } + assert result["custom_constraints_rhs"].empty def test_create_ispypsa_inputs_template_new_format_single_region(csv_str_to_df): @@ -570,6 +622,10 @@ def test_create_ispypsa_inputs_template_new_format_single_region(csv_str_to_df): "ispypsa.templater.create_template.template_custom_constraints_from_plexos", return_value=_stub_custom_constraints_tables(), ) as mock_template_custom_constraints, + patch( + "ispypsa.templater.create_template.load_timeslice_calendar", + return_value=_stub_timeslice_calendar(), + ), ): result = create_ispypsa_inputs_template( scenario="Step Change", @@ -590,6 +646,7 @@ def test_create_ispypsa_inputs_template_new_format_single_region(csv_str_to_df): }, manually_extracted_tables={ "connection_capacity_non_vre": connection_capacity_non_vre, + "reference_year_sequence": _stub_reference_year_sequence(), }, iasr_workbook_version="ignored-by-patch", ) @@ -630,10 +687,15 @@ def test_create_ispypsa_inputs_template_new_format_single_region(csv_str_to_df): assert connection_costs.empty # Custom constraints from PLEXOS are sub-regional export limits with no - # meaningful representation at single_region, so the templater skips them. - # The mock turns a regression in that gate into a clean assertion failure - # instead of a PLEXOS-extract disk read. + # meaningful representation at single_region, so the templater emits them + # header-only instead of templating them. The mock turns a regression in + # that gate into a clean assertion failure instead of a PLEXOS-extract + # disk read. mock_template_custom_constraints.assert_not_called() - assert "custom_constraints" not in result - assert "custom_constraints_lhs" not in result - assert "custom_constraints_rhs" not in result + for table in [ + "custom_constraints", + "custom_constraints_lhs", + "custom_constraints_rhs", + ]: + assert result[table].empty + assert len(result[table].columns) > 0 From c73e9a3b17c6280bb221c7d90d5de458df55d57a Mon Sep 17 00:00:00 2001 From: nick-gorman Date: Thu, 11 Jun 2026 12:26:25 +1000 Subject: [PATCH 2/6] Translate the new-format network tables to buses and links MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit network_geography becomes buses (REZ buses dropped when rezs is attached_to_parent_node), and the path/limit/expansion tables become links. The winter_reference limit is the link's static p_nom — winter is the calendar's default season — with the other timeslices' forward and reverse limits emitted as per-unit values in a link_timeslice_limits table for pypsa_build to expand into p_max_pu/p_min_pu series (per-unit values can exceed 1.0 when a season's limit tops winter's). Asymmetric reverse limits land in p_min_pu rather than separate links. Expansion options become one extendable link per (path, investment period) with annuitised capital costs, gated by transmission_expansion / rez_transmission_expansion according to whether the path connects a REZ. Paths with no capacity data take rez_to_sub_region_transmission_default_limit, replicating the existing REZ behaviour. Co-Authored-By: Claude Fable 5 --- src/ispypsa/translator/network.py | 424 ++++++++++++++++++++++++++ tests/test_translator/test_network.py | 297 ++++++++++++++++++ 2 files changed, 721 insertions(+) create mode 100644 src/ispypsa/translator/network.py create mode 100644 tests/test_translator/test_network.py diff --git a/src/ispypsa/translator/network.py b/src/ispypsa/translator/network.py new file mode 100644 index 00000000..e3d6d543 --- /dev/null +++ b/src/ispypsa/translator/network.py @@ -0,0 +1,424 @@ +"""Translates the new-format network tables into PyPSA friendly buses and links. + +Consumes ``network_geography``, ``network_transmission_paths``, +``network_transmission_path_limits``, ``network_expansion_options`` and +``network_transmission_path_expansion_costs``, handling flow paths and REZ +connections through one unified pipeline (both are just paths). +""" + +import numpy as np +import pandas as pd + +from ispypsa.config import ModelConfig +from ispypsa.translator.helpers import _annuitised_investment_costs + +_LINK_COLUMNS = [ + "isp_name", + "name", + "carrier", + "bus0", + "bus1", + "p_nom", + "p_min_pu", + "build_year", + "lifetime", + "capital_cost", + "p_nom_extendable", + "isp_type", +] + +_LINK_TIMESLICE_LIMIT_COLUMNS = ["name", "attribute", "timeslice", "value"] + + +def _translate_network_geography_to_buses( + network_geography: pd.DataFrame, rezs: str +) -> pd.DataFrame: + """Creates one PyPSA bus per geography in the model. + + REZ buses are only created when REZs are modelled as discrete nodes; + with ``rezs="attached_to_parent_node"`` REZ-located components connect + straight to the parent geography's bus. + + I/O Example: + network_geography: + geo_id geo_type region_id subregion_id + NQ subregion QLD NQ + Q1 rez QLD NQ + + rezs = "discrete_nodes" returns: + name + NQ + Q1 + + rezs = "attached_to_parent_node" returns: + name + NQ + """ + buses = network_geography + if rezs != "discrete_nodes": + buses = buses[buses["geo_type"] != "rez"] + buses = buses.loc[:, ["geo_id"]].rename(columns={"geo_id": "name"}) + return buses.reset_index(drop=True) + + +def _translate_network_to_links( + ispypsa_tables: dict[str, pd.DataFrame], + config: ModelConfig, +) -> tuple[pd.DataFrame, pd.DataFrame]: + """Translates the network tables into PyPSA links and per-timeslice limits. + + Existing links carry the winter_reference limit as p_nom; limits for the + other demand conditions are returned per unit of p_nom in the + link_timeslice_limits table. Expansion links are built per investment + period from the unified expansion options/costs tables (their total + capacity is capped by the expansion-limit custom constraints built in + ispypsa.translator.constraints). + + Returns: + Tuple of (links, link_timeslice_limits) in PyPSA friendly format. + """ + rez_ids = _rez_geo_ids(ispypsa_tables["network_geography"]) + paths = _drop_rez_paths_if_not_modelled( + ispypsa_tables["network_transmission_paths"], rez_ids, config.network.nodes.rezs + ) + limits = ispypsa_tables["network_transmission_path_limits"] + static_limits = _extract_static_limits(limits) + existing_links = _build_existing_links( + paths, + static_limits, + rez_ids, + config.network.rez_to_sub_region_transmission_default_limit, + config.temporal.range.start_year, + ) + link_timeslice_limits = _translate_timeslice_limits_to_pu(limits, existing_links) + options = _pivot_physical_expansion_options( + ispypsa_tables["network_expansion_options"] + ) + options = _filter_options_to_enabled_expansion( + options, + existing_links, + config.network.transmission_expansion, + config.network.rez_transmission_expansion, + ) + costs = _prepare_expansion_costs( + ispypsa_tables["network_transmission_path_expansion_costs"], + config.temporal.capacity_expansion.investment_periods, + config.temporal.year_type, + config.wacc, + config.network.annuitisation_lifetime, + ) + expansion_links = _build_expansion_links(existing_links, options, costs) + links = pd.concat([existing_links, expansion_links], ignore_index=True) + return links.loc[:, _LINK_COLUMNS], link_timeslice_limits + + +def _rez_geo_ids(network_geography: pd.DataFrame) -> set[str]: + """geo_ids of the REZ entries in the geography. + + I/O Example: + geo_id=NQ/geo_type=subregion, geo_id=Q1/geo_type=rez -> {"Q1"} + """ + return set(network_geography.loc[network_geography["geo_type"] == "rez", "geo_id"]) + + +def _drop_rez_paths_if_not_modelled( + paths: pd.DataFrame, rez_ids: set[str], rezs: str +) -> pd.DataFrame: + """Drops REZ-to-parent paths when REZs are not modelled as discrete nodes. + + I/O Example: + paths (path_ids): CQ-NQ, Q1-NQ (Q1 is a REZ) + + rezs = "attached_to_parent_node" returns paths: CQ-NQ + rezs = "discrete_nodes" returns paths unchanged. + """ + if rezs == "discrete_nodes": + return paths + return paths[~paths["geo_from"].isin(rez_ids)] + + +def _extract_static_limits(limits: pd.DataFrame) -> pd.DataFrame: + """Reduces the per-timeslice limits to one static capacity per path and + direction: the winter_reference limit. + + Winter is used as the link's p_nom because it is the demand condition left + over when no summer timeslice is active — limits for the other conditions + are applied per unit of it. Collapsed all-NaN rows (paths with no limit + data) contribute nothing; paths absent from the result get the default + limit downstream. + + I/O Example: + limits: + path_id direction timeslice capacity + CQ-NQ forward qld_peak_demand 1200 + CQ-NQ forward qld_winter_reference 1400 + CQ-NQ reverse qld_winter_reference 1910 + N1-CNSW , , # collapsed row + + returns: + path_id direction capacity + CQ-NQ forward 1400 + CQ-NQ reverse 1910 + """ + winter = limits[limits["timeslice"].str.endswith("_winter_reference", na=False)] + return winter.loc[:, ["path_id", "direction", "capacity"]] + + +def _build_existing_links( + paths: pd.DataFrame, + static_limits: pd.DataFrame, + rez_ids: set[str], + default_limit: float, + start_year: int, +) -> pd.DataFrame: + """Builds one existing (non-extendable) PyPSA link per transmission path. + + I/O Example: + paths: + path_id geo_from geo_to carrier + CQ-NQ CQ NQ AC + N1-CNSW N1 CNSW AC # REZ path, no limit data + + static_limits: + path_id direction capacity + CQ-NQ forward 1400 + CQ-NQ reverse 1910 + + returns (abridged): + isp_name name bus0 bus1 p_nom p_min_pu isp_type + CQ-NQ CQ-NQ_existing CQ NQ 1400 -1.364 flow_path + N1-CNSW N1-CNSW_existing N1 CNSW 100000 -1.0 rez # default limit + """ + links = paths.rename( + columns={"path_id": "isp_name", "geo_from": "bus0", "geo_to": "bus1"} + ) + links = _add_static_capacities(links, static_limits, default_limit) + links["name"] = links["isp_name"] + "_existing" + links["isp_type"] = np.where(links["bus0"].isin(rez_ids), "rez", "flow_path") + links["build_year"] = start_year - 1 + links["lifetime"] = np.inf + links["capital_cost"] = np.nan + links["p_nom_extendable"] = False + return links + + +def _add_static_capacities( + links: pd.DataFrame, static_limits: pd.DataFrame, default_limit: float +) -> pd.DataFrame: + """Merges the static forward limit as p_nom and the reverse limit as p_min_pu. + + Paths with no static limit in a direction get the default: ``default_limit`` + as p_nom (the path is constraint-modelled with no explicit physical limit) + and a symmetric -1.0 as p_min_pu. Zero-capacity paths (new parallel + corridors) also get a symmetric p_min_pu, avoiding a 0/0 division. + + I/O Example: + links (isp_name): CQ-NQ, N1-CNSW + static_limits: CQ-NQ forward 1400, CQ-NQ reverse 1910 + + returns: + isp_name p_nom p_min_pu + CQ-NQ 1400 -1.364 # -1910/1400 + N1-CNSW 100000 -1.0 # defaults + """ + directions = static_limits.pivot( + index="path_id", columns="direction", values="capacity" + ).reindex(columns=["forward", "reverse"]) + links = links.merge( + directions, left_on="isp_name", right_index=True, how="left" + ).rename(columns={"forward": "p_nom"}) + links["p_min_pu"] = np.where( + links["reverse"].notna() & (links["p_nom"] > 0), + -1.0 * links["reverse"] / links["p_nom"], + -1.0, + ) + links["p_nom"] = links["p_nom"].fillna(default_limit) + return links.drop(columns=["reverse"]) + + +def _translate_timeslice_limits_to_pu( + limits: pd.DataFrame, existing_links: pd.DataFrame +) -> pd.DataFrame: + """Expresses each per-timeslice limit per unit of its link's p_nom. + + Forward limits become p_max_pu values and reverse limits p_min_pu values. + pypsa_build expands them into per-snapshot series via the + timeslice_snapshots mapping; snapshots outside any of a link's tagged + timeslices keep the static (winter_reference) limit. Zero-p_nom links + (new parallel corridors) are skipped — all their limits are zero and the + per-unit form is undefined. + + I/O Example: + limits: + path_id direction timeslice capacity + CQ-NQ forward qld_peak_demand 1200 + CQ-NQ forward qld_winter_reference 1400 + CQ-NQ reverse qld_peak_demand 1440 + + existing_links (abridged): + isp_name name p_nom + CQ-NQ CQ-NQ_existing 1400 + + returns: + name attribute timeslice value + CQ-NQ_existing p_max_pu qld_peak_demand 0.857 + CQ-NQ_existing p_max_pu qld_winter_reference 1.0 + CQ-NQ_existing p_min_pu qld_peak_demand -1.029 + """ + rows = limits.dropna(subset=["timeslice", "capacity"]) + rows = rows.merge( + existing_links.loc[:, ["isp_name", "name", "p_nom"]], + left_on="path_id", + right_on="isp_name", + ) + rows = rows[rows["p_nom"] > 0] + rows["attribute"] = rows["direction"].map( + {"forward": "p_max_pu", "reverse": "p_min_pu"} + ) + sign = rows["direction"].map({"forward": 1.0, "reverse": -1.0}) + rows["value"] = sign * rows["capacity"] / rows["p_nom"] + return rows.loc[:, _LINK_TIMESLICE_LIMIT_COLUMNS].reset_index(drop=True) + + +def _pivot_physical_expansion_options(expansion_options: pd.DataFrame) -> pd.DataFrame: + """Pairs each physical path's forward and reverse expansion rows into one row. + + constraint_relaxation rows are not physical paths — they are translated + into custom-constraint relaxation generators by + ispypsa.translator.constraints. + + I/O Example: + expansion_options: + expansion_id expansion_type allowed_expansion expansion_option + CQ-NQ forward 1000 Option 1 + CQ-NQ reverse 900 Option 1 + SWQLD1 constraint_relaxation 500 Option 2 + + returns: + expansion_id forward_capacity reverse_capacity + CQ-NQ 1000 900 + """ + physical = expansion_options[ + expansion_options["expansion_type"].isin(["forward", "reverse"]) + ] + options = ( + physical.pivot( + index="expansion_id", columns="expansion_type", values="allowed_expansion" + ) + .reindex(columns=["forward", "reverse"]) + .rename(columns={"forward": "forward_capacity", "reverse": "reverse_capacity"}) + ) + options.columns.name = None + return options.reset_index() + + +def _filter_options_to_enabled_expansion( + options: pd.DataFrame, + existing_links: pd.DataFrame, + transmission_expansion: bool, + rez_transmission_expansion: bool, +) -> pd.DataFrame: + """Keeps the expansion options enabled by the network config flags. + + ``transmission_expansion`` gates paths between (sub)regions; + ``rez_transmission_expansion`` gates REZ connection paths. Options for + paths not in the model (e.g. REZ paths dropped under + attached_to_parent_node) drop out here too, as they match no link. + + I/O Example: + options (expansion_ids): CQ-NQ, Q1-NQ + existing_links: CQ-NQ isp_type=flow_path, Q1-NQ isp_type=rez + + transmission_expansion=True, rez_transmission_expansion=False + returns options: CQ-NQ + """ + rez_path_ids = set( + existing_links.loc[existing_links["isp_type"] == "rez", "isp_name"] + ) + flow_path_ids = set( + existing_links.loc[existing_links["isp_type"] == "flow_path", "isp_name"] + ) + enabled = set() + if transmission_expansion: + enabled |= flow_path_ids + if rez_transmission_expansion: + enabled |= rez_path_ids + return options[options["expansion_id"].isin(enabled)] + + +def _prepare_expansion_costs( + expansion_costs: pd.DataFrame, + investment_periods: list[int], + year_type: str, + wacc: float, + asset_lifetime: int, +) -> pd.DataFrame: + """Filters the long-format expansion costs to the investment periods and + annuitises them. + + The ``year`` column holds financial-year ending years as ints, matching + the investment period labels used with ``year_type="fy"``. + + I/O Example: + expansion_costs: + expansion_id year cost + CQ-NQ 2025 1000 + CQ-NQ 2026 1010 + + investment_periods=[2026], wacc=0.07, asset_lifetime=30 returns: + expansion_id year capital_cost + CQ-NQ 2026 81.4 + """ + if year_type != "fy": + raise NotImplementedError( + f"Network expansion costs are not implemented for year_type: {year_type}" + ) + costs = expansion_costs[expansion_costs["year"].isin(investment_periods)].copy() + costs["capital_cost"] = costs["cost"].apply( + lambda cost: _annuitised_investment_costs(cost, wacc, asset_lifetime) + ) + return costs.loc[:, ["expansion_id", "year", "capital_cost"]] + + +def _build_expansion_links( + existing_links: pd.DataFrame, + options: pd.DataFrame, + costs: pd.DataFrame, +) -> pd.DataFrame: + """Builds one extendable PyPSA link per expandable path and investment period. + + Each expansion link is unbounded on its own — the expansion-limit custom + constraints built in ispypsa.translator.constraints cap the p_nom built + across a path's expansion links at the selected option's forward capacity. + Asymmetric options are modelled with p_min_pu = -reverse/forward, so + however much forward capacity is built, reverse capacity scales in the + option's proportion. + + I/O Example: + options: + expansion_id forward_capacity reverse_capacity + CQ-NQ 1000 900 + + costs: + expansion_id year capital_cost + CQ-NQ 2026 81.4 + + existing_links (abridged): CQ-NQ bus0=CQ bus1=NQ carrier=AC isp_type=flow_path + + returns (abridged): + isp_name name p_nom p_nom_extendable p_min_pu build_year capital_cost + CQ-NQ CQ-NQ_exp_2026 0.0 True -0.9 2026 81.4 + """ + links = costs.merge(options, on="expansion_id") + links = links.merge( + existing_links.loc[:, ["isp_name", "bus0", "bus1", "carrier", "isp_type"]], + left_on="expansion_id", + right_on="isp_name", + ) + links["name"] = links["expansion_id"] + "_exp_" + links["year"].astype(str) + links["p_nom"] = 0.0 + links["p_nom_extendable"] = True + links["p_min_pu"] = -1.0 * links["reverse_capacity"] / links["forward_capacity"] + links["build_year"] = links["year"] + links["lifetime"] = np.inf + return links diff --git a/tests/test_translator/test_network.py b/tests/test_translator/test_network.py new file mode 100644 index 00000000..8c1ae216 --- /dev/null +++ b/tests/test_translator/test_network.py @@ -0,0 +1,297 @@ +import pandas as pd +import pytest + +from ispypsa.translator.helpers import _annuitised_investment_costs +from ispypsa.translator.network import ( + _translate_network_geography_to_buses, + _translate_network_to_links, +) + +# Annuitised $1/MW at the sample_model_config's wacc (0.06) and annuitisation +# lifetime (25) — expansion cost expectations are multiples of this. +_ANNUITY_PER_DOLLAR = _annuitised_investment_costs(1.0, 0.06, 25) + + +def test_translate_network_geography_to_buses_discrete_nodes(csv_str_to_df): + network_geography = csv_str_to_df(""" + geo_id, geo_type, region_id + NQ, subregion, QLD + CNSW, subregion, NSW + Q1, rez, QLD + """) + + result = _translate_network_geography_to_buses(network_geography, "discrete_nodes") + + expected = csv_str_to_df(""" + name + NQ + CNSW + Q1 + """) + pd.testing.assert_frame_equal(result, expected) + + +def test_translate_network_geography_to_buses_attached_to_parent_node(csv_str_to_df): + network_geography = csv_str_to_df(""" + geo_id, geo_type, region_id + NQ, subregion, QLD + Q1, rez, QLD + """) + + result = _translate_network_geography_to_buses( + network_geography, "attached_to_parent_node" + ) + + expected = csv_str_to_df(""" + name + NQ + """) + pd.testing.assert_frame_equal(result, expected) + + +def _network_tables(csv_str_to_df) -> dict[str, pd.DataFrame]: + """New-format network tables with one flow path (CQ-NQ, asymmetric winter + limits, expandable), one REZ path with limits (Q1-NQ) and one collapsed + REZ path with no limit data (N1-CNSW).""" + tables = {} + tables["network_geography"] = csv_str_to_df(""" + geo_id, geo_type, region_id + NQ, subregion, QLD + CQ, subregion, QLD + CNSW, subregion, NSW + Q1, rez, QLD + N1, rez, NSW + """) + tables["network_transmission_paths"] = csv_str_to_df(""" + path_id, geo_from, geo_to, carrier + CQ-NQ, CQ, NQ, AC + Q1-NQ, Q1, NQ, AC + N1-CNSW, N1, CNSW, AC + """) + tables["network_transmission_path_limits"] = csv_str_to_df(""" + path_id, direction, timeslice, capacity + CQ-NQ, forward, qld_peak_demand, 1200 + CQ-NQ, forward, qld_summer_typical, 1300 + CQ-NQ, forward, qld_winter_reference, 1400 + CQ-NQ, reverse, qld_peak_demand, 1440 + CQ-NQ, reverse, qld_summer_typical, 1600 + CQ-NQ, reverse, qld_winter_reference, 1910 + Q1-NQ, forward, qld_winter_reference, 750 + Q1-NQ, reverse, qld_winter_reference, 750 + N1-CNSW, , , + """) + tables["network_expansion_options"] = csv_str_to_df(""" + expansion_id, expansion_type, allowed_expansion, expansion_option + CQ-NQ, forward, 1000, CQ-NQ Option 1 + CQ-NQ, reverse, 900, CQ-NQ Option 1 + Q1-NQ, forward, 500, Q1 Option 1 + Q1-NQ, reverse, 500, Q1 Option 1 + SWQLD1, constraint_relaxation, 400, SWQLD1 Option 2 + """) + # 2025 is outside the sample config's investment periods (2026, 2028) and + # is expected to be filtered out. + tables["network_transmission_path_expansion_costs"] = csv_str_to_df(""" + expansion_id, year, cost + CQ-NQ, 2025, 900000 + CQ-NQ, 2026, 1000000 + Q1-NQ, 2026, 500000 + SWQLD1, 2026, 400000 + """) + return tables + + +def test_translate_network_to_links(csv_str_to_df, sample_model_config): + ispypsa_tables = _network_tables(csv_str_to_df) + + links, link_timeslice_limits = _translate_network_to_links( + ispypsa_tables, sample_model_config + ) + + expected_links = csv_str_to_df(f""" + isp_name, name, carrier, bus0, bus1, p_nom, p_min_pu, build_year, lifetime, capital_cost, p_nom_extendable, isp_type + CQ-NQ, CQ-NQ_existing, AC, CQ, NQ, 1400, -1.364286, 2025, inf, , False, flow_path + Q1-NQ, Q1-NQ_existing, AC, Q1, NQ, 750, -1.0, 2025, inf, , False, rez + N1-CNSW, N1-CNSW_existing,AC, N1, CNSW, 1000000, -1.0, 2025, inf, , False, rez + CQ-NQ, CQ-NQ_exp_2026, AC, CQ, NQ, 0.0, -0.9, 2026, inf, {1000000 * _ANNUITY_PER_DOLLAR}, True, flow_path + Q1-NQ, Q1-NQ_exp_2026, AC, Q1, NQ, 0.0, -1.0, 2026, inf, {500000 * _ANNUITY_PER_DOLLAR}, True, rez + """) + pd.testing.assert_frame_equal( + links.sort_values("name").reset_index(drop=True), + expected_links.sort_values("name").reset_index(drop=True), + check_dtype=False, + rtol=1e-5, + ) + + expected_limits = csv_str_to_df(""" + name, attribute, timeslice, value + CQ-NQ_existing, p_max_pu, qld_peak_demand, 0.857143 + CQ-NQ_existing, p_max_pu, qld_summer_typical, 0.928571 + CQ-NQ_existing, p_max_pu, qld_winter_reference, 1.0 + CQ-NQ_existing, p_min_pu, qld_peak_demand, -1.028571 + CQ-NQ_existing, p_min_pu, qld_summer_typical, -1.142857 + CQ-NQ_existing, p_min_pu, qld_winter_reference, -1.364286 + Q1-NQ_existing, p_max_pu, qld_winter_reference, 1.0 + Q1-NQ_existing, p_min_pu, qld_winter_reference, -1.0 + """) + pd.testing.assert_frame_equal( + link_timeslice_limits.sort_values( + ["name", "attribute", "timeslice"] + ).reset_index(drop=True), + expected_limits.sort_values(["name", "attribute", "timeslice"]).reset_index( + drop=True + ), + rtol=1e-5, + ) + + +def test_translate_network_to_links_expansion_disabled( + csv_str_to_df, sample_model_config +): + ispypsa_tables = _network_tables(csv_str_to_df) + sample_model_config.network.transmission_expansion = False + + links, _ = _translate_network_to_links(ispypsa_tables, sample_model_config) + + # The flow path's expansion link is gone; the REZ path's remains. + assert sorted(links["name"]) == [ + "CQ-NQ_existing", + "N1-CNSW_existing", + "Q1-NQ_existing", + "Q1-NQ_exp_2026", + ] + + +def test_translate_network_to_links_rez_expansion_disabled( + csv_str_to_df, sample_model_config +): + ispypsa_tables = _network_tables(csv_str_to_df) + sample_model_config.network.rez_transmission_expansion = False + + links, _ = _translate_network_to_links(ispypsa_tables, sample_model_config) + + # The REZ path's expansion link is gone; the flow path's remains. + assert sorted(links["name"]) == [ + "CQ-NQ_existing", + "CQ-NQ_exp_2026", + "N1-CNSW_existing", + "Q1-NQ_existing", + ] + + +def test_translate_network_to_links_rezs_attached_to_parent_node( + csv_str_to_df, sample_model_config +): + ispypsa_tables = _network_tables(csv_str_to_df) + sample_model_config.network.nodes.rezs = "attached_to_parent_node" + + links, link_timeslice_limits = _translate_network_to_links( + ispypsa_tables, sample_model_config + ) + + # REZ paths (and their expansion links and timeslice limits) are dropped. + assert sorted(links["name"]) == ["CQ-NQ_existing", "CQ-NQ_exp_2026"] + assert set(link_timeslice_limits["name"]) == {"CQ-NQ_existing"} + + +def test_translate_network_to_links_zero_capacity_parallel_path( + csv_str_to_df, sample_model_config +): + """New parallel corridors arrive as zero-capacity paths: p_nom 0 with a + symmetric p_min_pu (no 0/0 division), and no per-unit timeslice rows.""" + ispypsa_tables = _network_tables(csv_str_to_df) + ispypsa_tables["network_transmission_paths"] = csv_str_to_df(""" + path_id, geo_from, geo_to, carrier + CNSW-SNW, CNSW, SNW, AC + """) + ispypsa_tables["network_transmission_path_limits"] = csv_str_to_df(""" + path_id, direction, timeslice, capacity + CNSW-SNW, forward, nsw_winter_reference, 0 + CNSW-SNW, reverse, nsw_winter_reference, 0 + """) + ispypsa_tables["network_expansion_options"] = pd.DataFrame( + columns=[ + "expansion_id", + "expansion_type", + "allowed_expansion", + "expansion_option", + ] + ) + + links, link_timeslice_limits = _translate_network_to_links( + ispypsa_tables, sample_model_config + ) + + expected_links = csv_str_to_df(""" + isp_name, name, carrier, bus0, bus1, p_nom, p_min_pu, build_year, lifetime, capital_cost, p_nom_extendable, isp_type + CNSW-SNW, CNSW-SNW_existing, AC, CNSW, SNW, 0, -1.0, 2025, inf, , False, flow_path + """) + pd.testing.assert_frame_equal(links, expected_links, check_dtype=False) + + expected_limits = csv_str_to_df(""" + name, attribute, timeslice, value + """) + pd.testing.assert_frame_equal( + link_timeslice_limits, expected_limits, check_dtype=False + ) + + +def test_translate_network_to_links_empty_expansion_tables( + csv_str_to_df, sample_model_config +): + ispypsa_tables = _network_tables(csv_str_to_df) + ispypsa_tables["network_expansion_options"] = pd.DataFrame( + columns=[ + "expansion_id", + "expansion_type", + "allowed_expansion", + "expansion_option", + ] + ) + ispypsa_tables["network_transmission_path_expansion_costs"] = pd.DataFrame( + columns=["expansion_id", "year", "cost"] + ) + + links, _ = _translate_network_to_links(ispypsa_tables, sample_model_config) + + assert sorted(links["name"]) == [ + "CQ-NQ_existing", + "N1-CNSW_existing", + "Q1-NQ_existing", + ] + + +def test_translate_network_to_links_empty_paths(csv_str_to_df, sample_model_config): + ispypsa_tables = _network_tables(csv_str_to_df) + ispypsa_tables["network_transmission_paths"] = pd.DataFrame( + columns=["path_id", "geo_from", "geo_to", "carrier"] + ) + ispypsa_tables["network_transmission_path_limits"] = pd.DataFrame( + columns=["path_id", "direction", "timeslice", "capacity"] + ) + ispypsa_tables["network_expansion_options"] = pd.DataFrame( + columns=[ + "expansion_id", + "expansion_type", + "allowed_expansion", + "expansion_option", + ] + ) + ispypsa_tables["network_transmission_path_expansion_costs"] = pd.DataFrame( + columns=["expansion_id", "year", "cost"] + ) + + links, link_timeslice_limits = _translate_network_to_links( + ispypsa_tables, sample_model_config + ) + + expected_links = csv_str_to_df(""" + isp_name, name, carrier, bus0, bus1, p_nom, p_min_pu, build_year, lifetime, capital_cost, p_nom_extendable, isp_type + """) + pd.testing.assert_frame_equal(links, expected_links, check_dtype=False) + + expected_limits = csv_str_to_df(""" + name, attribute, timeslice, value + """) + pd.testing.assert_frame_equal( + link_timeslice_limits, expected_limits, check_dtype=False + ) From 93b5d1434481877f95ee779ab251f700c5d2142d Mon Sep 17 00:00:00 2001 From: nick-gorman Date: Thu, 11 Jun 2026 12:27:01 +1000 Subject: [PATCH 3/6] Re-sequence timeslice patterns to the configured reference_year_cycle MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The translator assigns a reference year to each model financial year with the identical construct_reference_year_mapping call the trace pipeline makes, so timeslice windows and demand/VRE traces always come from the same weather years — the binding interaction between hot-day limits and hot-day demand survives any configured cycle. The year before the first model year also gets a pattern (the cycle's last reference year, cycle-consistent) because winter windows run April-October and must cover the first model year's July-September snapshots. Month-day boundaries of 29 February (reference year 2024 carries them) clamp to 28 February in non-leap model years. The wrap decision for exclusive ends compares the month-day strings rather than the placed dates, so clamping a one-day 02-28..02-29 window collapses it to empty instead of stretching it out a year. A configured cycle requesting reference years the timeslices table has no patterns for raises rather than silently never binding. Co-Authored-By: Claude Fable 5 --- src/ispypsa/translator/timeslices.py | 240 ++++++++++++++ .../test_timeslice_snapshots.py | 310 ++++++++++++++++++ 2 files changed, 550 insertions(+) create mode 100644 src/ispypsa/translator/timeslices.py create mode 100644 tests/test_translator/test_timeslice_snapshots.py diff --git a/src/ispypsa/translator/timeslices.py b/src/ispypsa/translator/timeslices.py new file mode 100644 index 00000000..1e9a68ff --- /dev/null +++ b/src/ispypsa/translator/timeslices.py @@ -0,0 +1,240 @@ +"""Re-sequences the timeslice patterns and maps them onto model snapshots. + +The templated ``timeslices`` table gives one month-day window pattern per +reference (weather) year. This module assigns a reference year to each model +financial year using the configured reference_year_cycle — the identical +assignment the trace pipeline uses, so timeslices stay consistent with the +demand and VRE traces — expands the assigned patterns into absolute date +windows, and joins them onto the model's snapshots. The resulting mapping is +what pypsa_build uses to expand per-timeslice link limits into series and to +restrict custom constraints to the snapshots their RHS values apply to. +""" + +import calendar +import logging + +import pandas as pd +from isp_trace_parser import construct_reference_year_mapping + +from ispypsa.config import ModelConfig + +logger = logging.getLogger(__name__) + +_TIMESLICE_SNAPSHOT_COLUMNS = ["timeslice_id", "investment_periods", "snapshots"] + + +def _create_timeslice_snapshot_mapping( + timeslices: pd.DataFrame, snapshots: pd.DataFrame, config: ModelConfig +) -> pd.DataFrame: + """Maps each snapshot to the timeslices active on its date under the + configured reference_year_cycle. + + I/O Example: + timeslices: + timeslice_id reference_year start_month_day end_month_day + nsw_peak_demand 2018 01-13 01-14 + vic_peak_demand 2018 01-20 01-21 # no snapshots inside + + snapshots (config: start_year 2025, reference_year_cycle [2018]): + investment_periods snapshots + 2025 2025-01-13 12:00:00 + 2025 2025-01-15 12:00:00 + + returns: + timeslice_id investment_periods snapshots + nsw_peak_demand 2025 2025-01-13 12:00:00 + """ + if config.temporal.year_type != "fy": + raise NotImplementedError( + "Timeslice re-sequencing is only implemented for fy year_type; " + "AEMO's timeslice calendar is built on financial years." + ) + if timeslices.empty: + return pd.DataFrame(columns=_TIMESLICE_SNAPSHOT_COLUMNS) + reference_year_mapping = _map_model_years_to_reference_years(config) + _raise_on_reference_years_without_patterns(reference_year_mapping, timeslices) + windows = _expand_patterns_to_absolute_windows(timeslices, reference_year_mapping) + mapped = [ + _snapshots_in_window(snapshots, window) for window in windows.itertuples() + ] + return _concat_window_snapshots(mapped) + + +def _map_model_years_to_reference_years(config: ModelConfig) -> dict[int, int]: + """The model-year -> reference-year assignment, via the identical + construct_reference_year_mapping call the trace pipeline makes, plus the + year before the first model year (cycle-consistent: the cycle's last + reference year precedes its first) so windows starting in the prior + financial year — winter runs April to October — cover the first model + year's July-September snapshots. + + I/O Example: + start_year 2025, end_year 2027, reference_year_cycle [2011, 2018] + -> {2024: 2018, 2025: 2011, 2026: 2018, 2027: 2011} + """ + cycle = config.temporal.capacity_expansion.reference_year_cycle + mapping = construct_reference_year_mapping( + start_year=config.temporal.range.start_year, + end_year=config.temporal.range.end_year, + reference_years=cycle, + ) + return {config.temporal.range.start_year - 1: cycle[-1], **mapping} + + +def _raise_on_reference_years_without_patterns( + reference_year_mapping: dict[int, int], timeslices: pd.DataFrame +) -> None: + """Raise if the configured cycle uses reference years the timeslices + table has no patterns for — silently producing no windows would let + timeslice-tagged limits and constraints never bind.""" + missing = sorted( + set(reference_year_mapping.values()) - set(timeslices["reference_year"]) + ) + if missing: + raise ValueError( + f"Configured reference_year_cycle includes reference years with " + f"no timeslice window patterns: {missing}" + ) + + +def _expand_patterns_to_absolute_windows( + timeslices: pd.DataFrame, reference_year_mapping: dict[int, int] +) -> pd.DataFrame: + """Expands each model year's assigned pattern into absolute date windows. + + I/O Example: + timeslices: + timeslice_id reference_year start_month_day end_month_day + nsw_peak_demand 2018 11-18 11-20 + + reference_year_mapping {2026: 2018} -> + timeslice_id start_date end_date + nsw_peak_demand 2025-11-18 2025-11-20 + """ + expanded = [] + for model_year, reference_year in reference_year_mapping.items(): + pattern = timeslices[timeslices["reference_year"] == reference_year] + expanded.append(_place_pattern_in_financial_year(pattern, model_year)) + return pd.concat(expanded, ignore_index=True) + + +def _place_pattern_in_financial_year( + pattern: pd.DataFrame, model_year: int +) -> pd.DataFrame: + """Turns one pattern's month-day windows into absolute dates in the + financial year ending model_year: starts in July-December land in + model_year - 1, January-June in model_year, and each end is its first + occurrence after the start (ends may fall past 30 June, e.g. winter + April-October). + + I/O Example (model_year=2026): + timeslice_id start_month_day end_month_day + nsw_peak_demand 11-18 11-20 + nsw_summer_typical 11-20 03-20 # end wraps past new year + nsw_winter_reference 04-01 10-01 # end past 30 June + + returns: + timeslice_id start_date end_date + nsw_peak_demand 2025-11-18 2025-11-20 + nsw_summer_typical 2025-11-20 2026-03-20 + nsw_winter_reference 2026-04-01 2026-10-01 + """ + placed = pattern.copy() + placed["start_date"] = placed["start_month_day"].apply( + lambda month_day: _place_start_in_financial_year(month_day, model_year) + ) + placed["end_date"] = placed.apply( + lambda row: _resolve_end_date( + row["end_month_day"], row["start_month_day"], row["start_date"] + ), + axis=1, + ) + return placed[["timeslice_id", "start_date", "end_date"]] + + +def _place_start_in_financial_year(month_day: str, model_year: int) -> pd.Timestamp: + """'11-18' in FY2026 -> 2025-11-18; '04-01' in FY2026 -> 2026-04-01.""" + month = int(month_day.split("-")[0]) + year = model_year - 1 if month >= 7 else model_year + return _timestamp_with_leap_day_clamp(year, month_day) + + +def _resolve_end_date( + end_month_day: str, start_month_day: str, start: pd.Timestamp +) -> pd.Timestamp: + """An end later in the calendar year than the start lands in the start's + year; otherwise it wraps into the next. The comparison uses the original + month-day strings (which compare lexically in chronological order) + rather than the placed dates, so leap-day clamping a one-day 02-28 to + 02-29 window in a non-leap year collapses it to an empty window instead + of wrapping its end a year out. + + I/O Example: + ('03-20', '11-18', 2025-11-18) -> 2026-03-20 # wraps + ('10-01', '04-01', 2026-04-01) -> 2026-10-01 + ('02-29', '02-28', 2025-02-28) -> 2025-02-28 # clamped: empty window + """ + year = start.year if end_month_day > start_month_day else start.year + 1 + return _timestamp_with_leap_day_clamp(year, end_month_day) + + +def _timestamp_with_leap_day_clamp(year: int, month_day: str) -> pd.Timestamp: + """'02-29' is clamped to 28 February in non-leap years — patterns from + leap reference years land in non-leap model years under re-sequencing. + + I/O Example: + (2025, '02-29') -> 2025-02-28; (2024, '02-29') -> 2024-02-29 + """ + month, day = (int(part) for part in month_day.split("-")) + if (month, day) == (2, 29) and not calendar.isleap(year): + day = 28 + return pd.Timestamp(year=year, month=month, day=day) + + +def _snapshots_in_window(snapshots: pd.DataFrame, window) -> pd.DataFrame: + """Selects the snapshots inside one window, tagged with its timeslice. + + I/O Example: + snapshots 2025-01-13 12:00 and 2025-01-15 12:00, + window nsw_peak_demand [2025-01-13, 2025-01-14) + -> the 2025-01-13 12:00 snapshot tagged nsw_peak_demand + """ + in_window = (snapshots["snapshots"] >= window.start_date) & ( + snapshots["snapshots"] < window.end_date + ) + tagged = snapshots.loc[in_window, ["investment_periods", "snapshots"]].copy() + tagged["timeslice_id"] = window.timeslice_id + return tagged + + +def _concat_window_snapshots(mapped: list[pd.DataFrame]) -> pd.DataFrame: + """Combines the per-window snapshot selections into one mapping table.""" + if not mapped: + return pd.DataFrame(columns=_TIMESLICE_SNAPSHOT_COLUMNS) + mapping = pd.concat(mapped, ignore_index=True) + return mapping.loc[:, _TIMESLICE_SNAPSHOT_COLUMNS] + + +def _log_referenced_timeslices_without_snapshots( + timeslice_snapshots: pd.DataFrame, + link_timeslice_limits: pd.DataFrame, + custom_constraints_rhs: pd.DataFrame, +) -> None: + """Logs the timeslices referenced by a limit or constraint but mapped to + no snapshots — those limits and constraints will never apply. + + This is expected when snapshot aggregation (e.g. representative weeks) + selects no snapshots inside a timeslice's windows, and for calendar + timeslices that never activate (tas_peak_demand in the Draft 2026 ISP + calendar), but the user should know the affected inputs will not bind. + """ + referenced = set(link_timeslice_limits["timeslice"]) | set( + custom_constraints_rhs["timeslice"].dropna() + ) + without_snapshots = referenced - set(timeslice_snapshots["timeslice_id"]) + if without_snapshots: + logger.warning( + f"Timeslices referenced by transmission limits or custom constraints " + f"but with no snapshots in the model (these limits and constraints " + f"will never apply): {sorted(without_snapshots)}" + ) diff --git a/tests/test_translator/test_timeslice_snapshots.py b/tests/test_translator/test_timeslice_snapshots.py new file mode 100644 index 00000000..4561d4e2 --- /dev/null +++ b/tests/test_translator/test_timeslice_snapshots.py @@ -0,0 +1,310 @@ +import pandas as pd +import pytest + +from ispypsa.translator.timeslices import ( + _create_timeslice_snapshot_mapping, + _log_referenced_timeslices_without_snapshots, +) + +# sample_model_config: start_year 2026, end_year 2028, year_type fy, +# reference_year_cycle [2024]. + + +def _snapshots(csv_str_to_df, csv_str: str) -> pd.DataFrame: + snapshots = csv_str_to_df(csv_str) + snapshots["snapshots"] = pd.to_datetime(snapshots["snapshots"]) + return snapshots + + +def test_pattern_expanded_into_every_model_year(csv_str_to_df, sample_model_config): + timeslices = csv_str_to_df(""" + timeslice_id, reference_year, start_month_day, end_month_day + nsw_peak_demand, 2024, 01-13, 01-14 + """) + snapshots = _snapshots( + csv_str_to_df, + """ + investment_periods, snapshots + 2026, 2026-01-13 12:00:00 + 2026, 2026-01-14 12:00:00 + 2026, 2027-01-13 12:00:00 + 2028, 2028-01-13 12:00:00 + """, + ) + + result = _create_timeslice_snapshot_mapping( + timeslices, snapshots, sample_model_config + ) + + expected = _snapshots( + csv_str_to_df, + """ + timeslice_id, investment_periods, snapshots + nsw_peak_demand, 2026, 2026-01-13 12:00:00 + nsw_peak_demand, 2026, 2027-01-13 12:00:00 + nsw_peak_demand, 2028, 2028-01-13 12:00:00 + """, + ) + pd.testing.assert_frame_equal( + result.sort_values("snapshots").reset_index(drop=True), + expected.sort_values("snapshots").reset_index(drop=True), + ) + + +def test_cycle_assigns_different_patterns_to_different_model_years( + csv_str_to_df, sample_model_config +): + sample_model_config.temporal.capacity_expansion.reference_year_cycle = [2024, 2018] + timeslices = csv_str_to_df(""" + timeslice_id, reference_year, start_month_day, end_month_day + nsw_peak_demand, 2024, 01-13, 01-14 + nsw_peak_demand, 2018, 02-01, 02-03 + """) + # FY2026 -> 2024, FY2027 -> 2018, FY2028 -> 2024. Snapshots sit on both + # patterns' dates in FY2026 and FY2027; only the assigned pattern tags. + snapshots = _snapshots( + csv_str_to_df, + """ + investment_periods, snapshots + 2026, 2026-01-13 12:00:00 + 2026, 2026-02-02 12:00:00 + 2026, 2027-01-13 12:00:00 + 2026, 2027-02-02 12:00:00 + """, + ) + + result = _create_timeslice_snapshot_mapping( + timeslices, snapshots, sample_model_config + ) + + expected = _snapshots( + csv_str_to_df, + """ + timeslice_id, investment_periods, snapshots + nsw_peak_demand, 2026, 2026-01-13 12:00:00 + nsw_peak_demand, 2026, 2027-02-02 12:00:00 + """, + ) + pd.testing.assert_frame_equal( + result.sort_values("snapshots").reset_index(drop=True), + expected.sort_values("snapshots").reset_index(drop=True), + ) + + +def test_prior_year_winter_window_covers_first_model_year_july( + csv_str_to_df, sample_model_config +): + sample_model_config.temporal.capacity_expansion.reference_year_cycle = [2024, 2018] + # The year before the first model year takes the cycle's last reference + # year (2018), whose winter window [04-01, 10-01) spills into the first + # model year's July-September. 2018's winter ends 10-01 so the October + # snapshot is uncovered (2024's FY2026 winter only starts 2026-04-15). + timeslices = csv_str_to_df(""" + timeslice_id, reference_year, start_month_day, end_month_day + nsw_winter_reference, 2018, 04-01, 10-01 + nsw_winter_reference, 2024, 04-15, 10-15 + """) + snapshots = _snapshots( + csv_str_to_df, + """ + investment_periods, snapshots + 2026, 2025-07-15 12:00:00 + 2026, 2025-10-10 12:00:00 + """, + ) + + result = _create_timeslice_snapshot_mapping( + timeslices, snapshots, sample_model_config + ) + + expected = _snapshots( + csv_str_to_df, + """ + timeslice_id, investment_periods, snapshots + nsw_winter_reference, 2026, 2025-07-15 12:00:00 + """, + ) + pd.testing.assert_frame_equal(result.reset_index(drop=True), expected) + + +def test_end_month_day_wraps_past_new_year(csv_str_to_df, sample_model_config): + timeslices = csv_str_to_df(""" + timeslice_id, reference_year, start_month_day, end_month_day + nsw_summer_typical, 2024, 11-20, 03-20 + """) + snapshots = _snapshots( + csv_str_to_df, + """ + investment_periods, snapshots + 2026, 2025-11-20 12:00:00 + 2026, 2026-03-19 12:00:00 + 2026, 2026-03-21 12:00:00 + """, + ) + + result = _create_timeslice_snapshot_mapping( + timeslices, snapshots, sample_model_config + ) + + expected = _snapshots( + csv_str_to_df, + """ + timeslice_id, investment_periods, snapshots + nsw_summer_typical, 2026, 2025-11-20 12:00:00 + nsw_summer_typical, 2026, 2026-03-19 12:00:00 + """, + ) + pd.testing.assert_frame_equal(result.reset_index(drop=True), expected) + + +def test_leap_day_windows_clamped_in_non_leap_years(csv_str_to_df, sample_model_config): + # Reference year 2024's pattern carries leap-day boundaries. In leap + # FY2028 both windows apply as-is. In non-leap FY2026 the one-day + # [02-28, 02-29) window collapses to empty (the 28th is NOT tagged + # vic_peak_demand) while the [02-29, 03-02) window clamps its start to + # the 28th. + timeslices = csv_str_to_df(""" + timeslice_id, reference_year, start_month_day, end_month_day + vic_peak_demand, 2024, 02-28, 02-29 + nsw_peak_demand, 2024, 02-29, 03-02 + """) + snapshots = _snapshots( + csv_str_to_df, + """ + investment_periods, snapshots + 2026, 2026-02-28 12:00:00 + 2028, 2028-02-28 12:00:00 + 2028, 2028-02-29 12:00:00 + """, + ) + + result = _create_timeslice_snapshot_mapping( + timeslices, snapshots, sample_model_config + ) + + expected = _snapshots( + csv_str_to_df, + """ + timeslice_id, investment_periods, snapshots + nsw_peak_demand, 2026, 2026-02-28 12:00:00 + vic_peak_demand, 2028, 2028-02-28 12:00:00 + nsw_peak_demand, 2028, 2028-02-29 12:00:00 + """, + ) + pd.testing.assert_frame_equal( + result.sort_values(["snapshots", "timeslice_id"]).reset_index(drop=True), + expected.sort_values(["snapshots", "timeslice_id"]).reset_index(drop=True), + ) + + +def test_raises_on_reference_years_without_patterns(csv_str_to_df, sample_model_config): + timeslices = csv_str_to_df(""" + timeslice_id, reference_year, start_month_day, end_month_day + nsw_peak_demand, 2018, 01-13, 01-14 + """) + snapshots = _snapshots( + csv_str_to_df, + """ + investment_periods, snapshots + 2026, 2026-01-13 12:00:00 + """, + ) + + with pytest.raises(ValueError, match=r"no timeslice window patterns: \[2024\]"): + _create_timeslice_snapshot_mapping(timeslices, snapshots, sample_model_config) + + +def test_raises_for_calendar_year_type(csv_str_to_df, sample_model_config): + sample_model_config.temporal.year_type = "calendar" + timeslices = csv_str_to_df(""" + timeslice_id, reference_year, start_month_day, end_month_day + nsw_peak_demand, 2024, 01-13, 01-14 + """) + snapshots = _snapshots( + csv_str_to_df, + """ + investment_periods, snapshots + 2026, 2026-01-13 12:00:00 + """, + ) + + with pytest.raises(NotImplementedError, match="only implemented for fy"): + _create_timeslice_snapshot_mapping(timeslices, snapshots, sample_model_config) + + +def test_empty_timeslices_table(csv_str_to_df, sample_model_config): + timeslices = pd.DataFrame( + columns=["timeslice_id", "reference_year", "start_month_day", "end_month_day"] + ) + snapshots = _snapshots( + csv_str_to_df, + """ + investment_periods, snapshots + 2026, 2026-01-13 12:00:00 + """, + ) + + result = _create_timeslice_snapshot_mapping( + timeslices, snapshots, sample_model_config + ) + + expected = csv_str_to_df(""" + timeslice_id, investment_periods, snapshots + """) + pd.testing.assert_frame_equal(result, expected, check_dtype=False) + + +def test_logs_referenced_timeslices_without_snapshots(csv_str_to_df, caplog): + timeslice_snapshots = _snapshots( + csv_str_to_df, + """ + timeslice_id, investment_periods, snapshots + nsw_peak_demand, 2026, 2026-01-13 12:00:00 + """, + ) + link_timeslice_limits = csv_str_to_df(""" + name, attribute, timeslice, value + CQ-NQ_existing, p_max_pu, nsw_peak_demand, 0.8 + CQ-NQ_existing, p_max_pu, tas_peak_demand, 0.9 + """) + custom_constraints_rhs = csv_str_to_df(""" + constraint_name, investment_period, timeslice, rhs, constraint_type + SWQLD1, 2026, vic_peak_demand, 3000, <= + CQ-NQ_expansion_limit, , , 1000, <= + """) + + with caplog.at_level("WARNING"): + _log_referenced_timeslices_without_snapshots( + timeslice_snapshots, link_timeslice_limits, custom_constraints_rhs + ) + + assert ( + "Timeslices referenced by transmission limits or custom constraints " + "but with no snapshots in the model (these limits and constraints " + "will never apply): ['tas_peak_demand', 'vic_peak_demand']" + ) in caplog.text + + +def test_no_log_when_all_referenced_timeslices_have_snapshots(csv_str_to_df, caplog): + timeslice_snapshots = _snapshots( + csv_str_to_df, + """ + timeslice_id, investment_periods, snapshots + nsw_peak_demand, 2026, 2026-01-13 12:00:00 + """, + ) + link_timeslice_limits = csv_str_to_df(""" + name, attribute, timeslice, value + CQ-NQ_existing, p_max_pu, nsw_peak_demand, 0.8 + """) + custom_constraints_rhs = csv_str_to_df(""" + constraint_name, investment_period, timeslice, rhs, constraint_type + SWQLD1, 2026, nsw_peak_demand, 3000, <= + """) + + with caplog.at_level("WARNING"): + _log_referenced_timeslices_without_snapshots( + timeslice_snapshots, link_timeslice_limits, custom_constraints_rhs + ) + + assert caplog.text == "" From 6391117cef0779f454a1335c72761b4ed7a9675b Mon Sep 17 00:00:00 2001 From: nick-gorman Date: Thu, 11 Jun 2026 12:27:57 +1000 Subject: [PATCH 4/6] Translate the new-format custom constraints with temporal scope MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each RHS value becomes one constraint instance per investment period (and per timeslice where tagged), with date_from resolved as the value active at the start of each period. LHS terms keep generator and battery names even though those components aren't translated yet — pypsa_build skips-and-logs missing components, and the logging silences itself once generator translation lands. The generators/batteries parameters are accepted now as the future name-mapping hook. Constraint-relaxation expansion options become dummy generators on bus_for_custom_constraint_gens, with their LHS terms filtered per period by build_year so a generator built for 2040 can't relax a 2030 constraint instance — an improvement over the old single-constraint formulation, which couldn't express this. Extendable links and relaxation generators get "_expansion_limit" constraints capping total p_nom at the option's allowed expansion; the suffix keeps their names clear of the parent constraints'. Duplicate input rows and RHS rows without LHS pairs raise rather than collapse silently; the "load" term type maps to Load/p, which pypsa_build logs as unimplemented and skips, matching the old path. Co-Authored-By: Claude Fable 5 --- src/ispypsa/translator/constraints.py | 587 ++++++++++++++++++++++ src/ispypsa/translator/mappings.py | 5 + tests/test_translator/test_constraints.py | 327 ++++++++++++ 3 files changed, 919 insertions(+) create mode 100644 src/ispypsa/translator/constraints.py create mode 100644 tests/test_translator/test_constraints.py diff --git a/src/ispypsa/translator/constraints.py b/src/ispypsa/translator/constraints.py new file mode 100644 index 00000000..1c6cff04 --- /dev/null +++ b/src/ispypsa/translator/constraints.py @@ -0,0 +1,587 @@ +"""Translates the new-format custom-constraint tables into PyPSA friendly form. + +Consumes ``custom_constraints`` (constraint senses), ``custom_constraints_lhs`` +(terms) and ``custom_constraints_rhs`` (limit values), plus the unified +network expansion tables for constraint-relaxation generators and +expansion-limit constraints. + +The PyPSA friendly outputs carry two temporal columns the old format lacked: + +- ``investment_period``: time-varying inputs (``date_from``) are resolved by + holding, for each investment period, the value active at the period's + start. Rows with a NaN investment_period apply in every period. +- ``timeslice`` (RHS only): the constraint binds only at snapshots inside the + timeslice's windows (see ispypsa.translator.timeslices). NaN means the + limit applies at all snapshots. +""" + +import logging + +import numpy as np +import pandas as pd + +from ispypsa.config import ModelConfig +from ispypsa.translator.mappings import ( + _CUSTOM_CONSTRAINT_TERM_TYPE_TO_ATTRIBUTE_TYPE, + _CUSTOM_CONSTRAINT_TERM_TYPE_TO_COMPONENT_TYPE, +) +from ispypsa.translator.network import ( + _pivot_physical_expansion_options, + _prepare_expansion_costs, +) + +logger = logging.getLogger(__name__) + +_LHS_COLUMNS = [ + "constraint_name", + "investment_period", + "variable_name", + "component", + "attribute", + "coefficient", +] + +_RHS_COLUMNS = [ + "constraint_name", + "investment_period", + "timeslice", + "rhs", + "constraint_type", +] + +_GENERATOR_COLUMNS = [ + "name", + "isp_name", + "bus", + "p_nom", + "p_nom_extendable", + "build_year", + "lifetime", + "capital_cost", +] + +_DIRECTION_TO_CONSTRAINT_TYPE = {"<=": "<=", ">=": ">=", "=": "=="} + +# Working column orders before constraint_id is renamed to constraint_name. +_INTERNAL_LHS_COLUMNS = ["constraint_id"] + [ + c for c in _LHS_COLUMNS if c != "constraint_name" +] +_INTERNAL_RHS_COLUMNS = ["constraint_id"] + [ + c for c in _RHS_COLUMNS if c != "constraint_name" +] + + +def _raise_on_duplicate_input_rows( + table: pd.DataFrame, keys: list[str], label: str +) -> None: + """Raise on input rows sharing the same key — duplicates would otherwise + silently collapse to one arbitrary row during date_from resolution.""" + duplicates = table[table.duplicated(subset=keys, keep=False)] + if not duplicates.empty: + raise ValueError( + f"Duplicate custom constraint {label} rows for: " + f"{sorted(set(duplicates['constraint_id']))}" + ) + + +def _concat_non_empty(frames: list[pd.DataFrame], columns: list[str]) -> pd.DataFrame: + """Concatenates the non-empty frames (concatenating all-empty frames is + deprecated by pandas), returning a header-only frame when all are empty.""" + non_empty = [frame for frame in frames if not frame.empty] + if not non_empty: + return pd.DataFrame(columns=columns) + return pd.concat(non_empty, ignore_index=True) + + +def _translate_custom_constraints_from_network_tables( + ispypsa_tables: dict[str, pd.DataFrame], + links: pd.DataFrame, + config: ModelConfig, + generators: pd.DataFrame | None = None, + batteries: pd.DataFrame | None = None, +) -> dict[str, pd.DataFrame]: + """Translates the custom-constraint tables and builds the endogenous + expansion-limit constraints. + + Generator and battery terms pass through with their IASR IDs as + variable_names: pypsa_build skips (and logs) terms whose components are + not in the model. The ``generators`` and ``batteries`` arguments are + accepted so the templated generator/battery tables can be used to map + those IDs to model component names once generator templating lands; they + are currently unused. + + Args: + ispypsa_tables: dictionary of new-format `ISPyPSA` input tables; + consumes custom_constraints, custom_constraints_lhs, + custom_constraints_rhs, network_expansion_options and + network_transmission_path_expansion_costs. + links: PyPSA friendly links table from + ispypsa.translator.network (existing plus expansion links). + config: `ispypsa.config.ModelConfig` object. + generators: templated generator table (future name-mapping hook). + batteries: templated battery table (future name-mapping hook). + + Returns: dict with the PyPSA friendly custom_constraints_lhs, + custom_constraints_rhs and custom_constraints_generators tables. + """ + _raise_on_duplicate_input_rows( + ispypsa_tables["custom_constraints_rhs"], + ["constraint_id", "timeslice", "date_from"], + "RHS", + ) + _raise_on_duplicate_input_rows( + ispypsa_tables["custom_constraints_lhs"], + ["constraint_id", "term_type", "variable_name", "date_from"], + "LHS", + ) + period_starts = _investment_period_start_dates( + config.temporal.capacity_expansion.investment_periods, + config.temporal.year_type, + ) + rhs = _resolve_values_active_at_period_starts( + ispypsa_tables["custom_constraints_rhs"], + ["constraint_id", "timeslice"], + period_starts, + ) + rhs = _add_constraint_type(rhs, ispypsa_tables["custom_constraints"]) + lhs = _resolve_values_active_at_period_starts( + ispypsa_tables["custom_constraints_lhs"], + ["constraint_id", "term_type", "variable_name"], + period_starts, + ) + lhs = _add_component_and_attribute(lhs) + lhs = _expand_link_flow_terms(lhs, links) + rhs = _drop_rhs_without_lhs_terms(rhs, lhs) + + relaxation_generators = _create_constraint_relaxation_generators( + ispypsa_tables, set(rhs["constraint_id"]), config + ) + relaxation_generator_lhs = _relaxation_generator_lhs_terms( + relaxation_generators, + config.temporal.capacity_expansion.investment_periods, + ) + expansion_limit_lhs, expansion_limit_rhs = _create_expansion_limit_constraints( + links, relaxation_generators, ispypsa_tables["network_expansion_options"] + ) + + lhs = _concat_non_empty( + [lhs, relaxation_generator_lhs, expansion_limit_lhs], _INTERNAL_LHS_COLUMNS + ) + rhs = _concat_non_empty([rhs, expansion_limit_rhs], _INTERNAL_RHS_COLUMNS) + lhs, rhs = _finalise_lhs_and_rhs(lhs, rhs) + return { + "custom_constraints_lhs": lhs, + "custom_constraints_rhs": rhs, + "custom_constraints_generators": _finalise_generators(relaxation_generators), + } + + +def _investment_period_start_dates( + investment_periods: list[int], year_type: str +) -> dict[int, pd.Timestamp]: + """The datetime each investment period starts at. + + I/O Example: + [2030], "fy" -> {2030: 2029-07-01} # FY ending nomenclature + [2030], "calendar" -> {2030: 2030-01-01} + """ + if year_type == "fy": + return {p: pd.Timestamp(year=p - 1, month=7, day=1) for p in investment_periods} + return {p: pd.Timestamp(year=p, month=1, day=1) for p in investment_periods} + + +def _resolve_values_active_at_period_starts( + table: pd.DataFrame, + group_columns: list[str], + period_starts: dict[int, pd.Timestamp], +) -> pd.DataFrame: + """Resolves date_from-varying rows into one row per investment period. + + For each period, each group keeps the row active at the period's start: + the latest date_from on or before it, with no-date_from rows acting as + the baseline. A group whose earliest date_from is after a period's start + contributes no row for that period. + + I/O Example: + table (group_columns=["constraint_id", "timeslice"]): + constraint_id timeslice rhs date_from + SWQLD1 qld_peak_demand 3000 + SWQLD1 qld_peak_demand 2500 2032-12-01T00:00:00 + + period_starts={2030: 2029-07-01, 2035: 2034-07-01} returns: + constraint_id timeslice rhs investment_period + SWQLD1 qld_peak_demand 3000 2030 + SWQLD1 qld_peak_demand 2500 2035 # 2032 value held from period start + """ + table = table.copy() + table["date_from"] = pd.to_datetime(table["date_from"]).fillna(pd.Timestamp.min) + resolved = [] + for period, start in period_starts.items(): + active = table[table["date_from"] <= start] + active = active.sort_values("date_from") + active = active.groupby(group_columns, dropna=False).tail(1).copy() + active["investment_period"] = period + resolved.append(active) + return pd.concat(resolved, ignore_index=True).drop(columns="date_from") + + +def _add_constraint_type( + rhs: pd.DataFrame, custom_constraints: pd.DataFrame +) -> pd.DataFrame: + """Adds each constraint's sense as constraint_type ("=" becomes "==", + matching the vocabulary pypsa_build applies constraints with). + + I/O Example: + rhs: constraint_id=SWQLD1, rhs=3000 + custom_constraints: constraint_id=SWQLD1, direction="<=" + -> constraint_id=SWQLD1, rhs=3000, constraint_type="<=" + """ + rhs = rhs.merge(custom_constraints, on="constraint_id", how="left") + rhs["constraint_type"] = rhs["direction"].map(_DIRECTION_TO_CONSTRAINT_TYPE) + _raise_on_missing_constraint_type(rhs) + return rhs.drop(columns="direction") + + +def _raise_on_missing_constraint_type(rhs: pd.DataFrame) -> None: + """Raise if any RHS row's constraint has no (or an unmapped) sense — the + constraint can't be applied without one.""" + missing = rhs.loc[rhs["constraint_type"].isna(), "constraint_id"] + if not missing.empty: + raise ValueError( + f"Custom constraints with RHS values but no direction in the " + f"custom_constraints table: {sorted(set(missing))}" + ) + + +def _add_component_and_attribute(lhs: pd.DataFrame) -> pd.DataFrame: + """Maps each term_type to the PyPSA component and attribute its variable + belongs to. + + I/O Example: + term_type=generator_output -> component=Generator, attribute=p + term_type=link_flow -> component=Link, attribute=p + term_type=storage_output -> component=Storage, attribute=p + """ + lhs = lhs.copy() + lhs["component"] = lhs["term_type"].map( + _CUSTOM_CONSTRAINT_TERM_TYPE_TO_COMPONENT_TYPE + ) + lhs["attribute"] = lhs["term_type"].map( + _CUSTOM_CONSTRAINT_TERM_TYPE_TO_ATTRIBUTE_TYPE + ) + _raise_on_unmapped_term_types(lhs) + return lhs.drop(columns="term_type") + + +def _raise_on_unmapped_term_types(lhs: pd.DataFrame) -> None: + """Raise if any term_type has no component mapping — silently dropping a + term would weaken its constraint.""" + unmapped = lhs.loc[lhs["component"].isna(), "term_type"] + if not unmapped.empty: + raise ValueError( + f"Custom constraint LHS term_types with no component mapping: " + f"{sorted(set(unmapped))}" + ) + + +def _expand_link_flow_terms(lhs: pd.DataFrame, links: pd.DataFrame) -> pd.DataFrame: + """Replaces each link term's path_id with the model's link names — the + existing link plus each expansion link — one term per link. Terms for + paths not in the model are dropped and logged. + + I/O Example: + lhs: + constraint_id variable_name component coefficient + SWQLD1 NSW-QLD Link 0.84 + SWQLD1 KINGASF1 Generator 0.14 + + links (isp_name -> name): NSW-QLD -> NSW-QLD_existing, NSW-QLD_exp_2030 + + returns: + constraint_id variable_name component coefficient + SWQLD1 KINGASF1 Generator 0.14 + SWQLD1 NSW-QLD_existing Link 0.84 + SWQLD1 NSW-QLD_exp_2030 Link 0.84 + """ + link_terms = lhs[lhs["component"] == "Link"] + other_terms = lhs[lhs["component"] != "Link"] + _log_link_terms_not_in_model(link_terms, links) + expanded = link_terms.merge( + links.loc[:, ["isp_name", "name"]], left_on="variable_name", right_on="isp_name" + ) + expanded = expanded.drop(columns=["variable_name", "isp_name"]) + expanded = expanded.rename(columns={"name": "variable_name"}) + return pd.concat([other_terms, expanded], ignore_index=True) + + +def _log_link_terms_not_in_model(link_terms: pd.DataFrame, links: pd.DataFrame) -> None: + missing = set(link_terms["variable_name"]) - set(links["isp_name"]) + if missing: + logger.info( + f"Custom constraint link_flow terms dropped (paths not in model): " + f"{sorted(missing)}" + ) + + +def _drop_rhs_without_lhs_terms(rhs: pd.DataFrame, lhs: pd.DataFrame) -> pd.DataFrame: + """Drops (and logs) RHS rows for constraints left with no LHS terms — a + constraint with an empty LHS can't be applied.""" + without_lhs = set(rhs["constraint_id"]) - set(lhs["constraint_id"]) + if without_lhs: + logger.info( + f"Custom constraints dropped (no LHS terms in model): {sorted(without_lhs)}" + ) + return rhs[~rhs["constraint_id"].isin(without_lhs)] + + +def _create_constraint_relaxation_generators( + ispypsa_tables: dict[str, pd.DataFrame], + constraint_ids: set[str], + config: ModelConfig, +) -> pd.DataFrame: + """Builds one extendable dummy generator per relaxable constraint and + investment period, with the selected expansion option's annualised cost. + + The generators' p_nom enters the parent constraint's LHS with coefficient + -1.0 (see _relaxation_generator_lhs_terms), so building them relaxes the + constraint at the option's cost; total relaxation is capped at the + option's allowed_expansion by the expansion-limit constraints. + + I/O Example: + network_expansion_options: + expansion_id expansion_type allowed_expansion + SWQLD1 constraint_relaxation 500 + + network_transmission_path_expansion_costs: + expansion_id year cost + SWQLD1 2030 100000 + + constraint_ids={"SWQLD1"}, investment_periods=[2030] returns (abridged): + name isp_name bus p_nom build_year + SWQLD1_exp_2030 SWQLD1 bus_for_custom_constraint_gens 0.0 2030 + """ + if not config.network.rez_transmission_expansion: + return pd.DataFrame(columns=_GENERATOR_COLUMNS + ["allowed_expansion"]) + options = ispypsa_tables["network_expansion_options"] + relaxations = options[options["expansion_type"] == "constraint_relaxation"] + relaxations = _drop_relaxations_without_constraints(relaxations, constraint_ids) + costs = _prepare_expansion_costs( + ispypsa_tables["network_transmission_path_expansion_costs"], + config.temporal.capacity_expansion.investment_periods, + config.temporal.year_type, + config.wacc, + config.network.annuitisation_lifetime, + ) + generators = costs.merge( + relaxations.loc[:, ["expansion_id", "allowed_expansion"]], on="expansion_id" + ) + return _format_relaxation_generators(generators) + + +def _drop_relaxations_without_constraints( + relaxations: pd.DataFrame, constraint_ids: set[str] +) -> pd.DataFrame: + """Drops (and logs) relaxation options whose constraint is not in the + model — there is nothing for them to relax.""" + missing = set(relaxations["expansion_id"]) - constraint_ids + if missing: + logger.info( + f"Constraint relaxation expansion options dropped (their constraints " + f"are not in the model): {sorted(missing)}" + ) + return relaxations[~relaxations["expansion_id"].isin(missing)] + + +def _format_relaxation_generators(generators: pd.DataFrame) -> pd.DataFrame: + """Adds the PyPSA generator attributes shared by all relaxation generators. + + I/O Example: + expansion_id=SWQLD1, year=2030, capital_cost=8140, allowed_expansion=500 + -> name=SWQLD1_exp_2030, isp_name=SWQLD1, p_nom=0.0, + p_nom_extendable=True, build_year=2030, lifetime=inf + """ + generators = generators.rename(columns={"expansion_id": "isp_name"}) + generators["name"] = ( + generators["isp_name"] + "_exp_" + generators["year"].astype(str) + ) + generators["bus"] = "bus_for_custom_constraint_gens" + generators["p_nom"] = 0.0 + generators["p_nom_extendable"] = True + generators["build_year"] = generators["year"] + generators["lifetime"] = np.inf + return generators.loc[:, _GENERATOR_COLUMNS + ["allowed_expansion"]] + + +def _relaxation_generator_lhs_terms( + relaxation_generators: pd.DataFrame, investment_periods: list[int] +) -> pd.DataFrame: + """LHS terms subtracting each relaxation generator's capacity from its + parent constraint. + + Terms are per investment period and only include generators already built + by that period — capacity built in a later period can't relax an earlier + period's constraint. + + I/O Example: + relaxation_generators: + name isp_name build_year + SWQLD1_exp_2030 SWQLD1 2030 + SWQLD1_exp_2040 SWQLD1 2040 + + investment_periods=[2030, 2040] returns: + constraint_id investment_period variable_name component attribute coefficient + SWQLD1 2030 SWQLD1_exp_2030 Generator p_nom -1.0 + SWQLD1 2040 SWQLD1_exp_2030 Generator p_nom -1.0 + SWQLD1 2040 SWQLD1_exp_2040 Generator p_nom -1.0 + """ + terms = [] + for period in investment_periods: + built = relaxation_generators[ + relaxation_generators["build_year"] <= period + ].copy() + built["investment_period"] = period + terms.append(built) + terms = pd.concat(terms, ignore_index=True) + terms = terms.rename(columns={"isp_name": "constraint_id", "name": "variable_name"}) + terms["component"] = "Generator" + terms["attribute"] = "p_nom" + terms["coefficient"] = -1.0 + columns = [c for c in _LHS_COLUMNS if c != "constraint_name"] + ["constraint_id"] + return terms.loc[:, columns] + + +def _create_expansion_limit_constraints( + links: pd.DataFrame, + relaxation_generators: pd.DataFrame, + expansion_options: pd.DataFrame, +) -> tuple[pd.DataFrame, pd.DataFrame]: + """Caps the total capacity built across each expandable element's + per-period components at the selected option's capacity. + + For physical paths the cap is the option's forward capacity (reverse + capacity scales with it through each expansion link's p_min_pu); for + constraint relaxations it is the option's allowed_expansion. The + constraints have no investment_period or timeslice — they apply to the + p_nom variables globally. Names get an "_expansion_limit" suffix so a + relaxation cap doesn't collide with the constraint it relaxes. + + I/O Example: + links (extendable): CQ-NQ_exp_2030, CQ-NQ_exp_2040 (isp_name CQ-NQ) + expansion_options: CQ-NQ forward 1000 / reverse 900 + + returns lhs: + constraint_id variable_name component attribute coefficient investment_period + CQ-NQ_expansion_limit CQ-NQ_exp_2030 Link p_nom 1.0 NaN + CQ-NQ_expansion_limit CQ-NQ_exp_2040 Link p_nom 1.0 NaN + + and rhs: + constraint_id rhs constraint_type investment_period timeslice + CQ-NQ_expansion_limit 1000 <= NaN NaN + """ + lhs = pd.concat( + [ + _expansion_limit_lhs(links[links["p_nom_extendable"]], "Link"), + _expansion_limit_lhs(relaxation_generators, "Generator"), + ], + ignore_index=True, + ) + rhs = _expansion_limit_rhs(expansion_options, set(lhs["constraint_id"])) + lhs["constraint_id"] = lhs["constraint_id"] + "_expansion_limit" + rhs["constraint_id"] = rhs["constraint_id"] + "_expansion_limit" + return lhs, rhs + + +def _expansion_limit_lhs(components: pd.DataFrame, component_type: str) -> pd.DataFrame: + """One LHS term per expandable component, summing p_nom across the + investment periods of its parent element. + + I/O Example: + components: name=CQ-NQ_exp_2030, isp_name=CQ-NQ; component_type="Link" + -> constraint_id=CQ-NQ, variable_name=CQ-NQ_exp_2030, component=Link, + attribute=p_nom, coefficient=1.0, investment_period=NaN + """ + lhs = components.loc[:, ["isp_name", "name"]].copy() + lhs = lhs.rename(columns={"isp_name": "constraint_id", "name": "variable_name"}) + lhs["component"] = component_type + lhs["attribute"] = "p_nom" + lhs["coefficient"] = 1.0 + lhs["investment_period"] = np.nan + return lhs + + +def _expansion_limit_rhs( + expansion_options: pd.DataFrame, expandable_ids: set[str] +) -> pd.DataFrame: + """One RHS row per expandable element with components in the model: the + selected option's forward capacity (physical paths) or allowed_expansion + (constraint relaxations). + + I/O Example: + expansion_options: + expansion_id expansion_type allowed_expansion + CQ-NQ forward 1000 + CQ-NQ reverse 900 + SWQLD1 constraint_relaxation 500 + + expandable_ids={"CQ-NQ", "SWQLD1"} returns: + constraint_id rhs constraint_type investment_period timeslice + CQ-NQ 1000 <= NaN NaN + SWQLD1 500 <= NaN NaN + """ + caps = expansion_options[ + expansion_options["expansion_type"].isin(["forward", "constraint_relaxation"]) + ] + caps = caps[caps["expansion_id"].isin(expandable_ids)] + rhs = caps.rename(columns={"expansion_id": "constraint_id"}).copy() + rhs["rhs"] = rhs["allowed_expansion"] + rhs["constraint_type"] = "<=" + rhs["investment_period"] = np.nan + rhs["timeslice"] = np.nan + columns = [c for c in _RHS_COLUMNS if c != "constraint_name"] + ["constraint_id"] + return rhs.loc[:, columns] + + +def _finalise_lhs_and_rhs( + lhs: pd.DataFrame, rhs: pd.DataFrame +) -> tuple[pd.DataFrame, pd.DataFrame]: + """Suffixes expansion-limit names, validates the pairing, and sets the + final PyPSA friendly column orders.""" + lhs = lhs.rename(columns={"constraint_id": "constraint_name"}) + rhs = rhs.rename(columns={"constraint_id": "constraint_name"}) + _raise_on_duplicate_rhs_rows(rhs) + _raise_on_unpaired_constraints(lhs, rhs) + return ( + lhs.loc[:, _LHS_COLUMNS].reset_index(drop=True), + rhs.loc[:, _RHS_COLUMNS].reset_index(drop=True), + ) + + +def _raise_on_duplicate_rhs_rows(rhs: pd.DataFrame) -> None: + """Raise on duplicate (constraint, period, timeslice) RHS rows — pypsa_build + would create two constraints with the same name.""" + keys = ["constraint_name", "investment_period", "timeslice"] + duplicates = rhs[rhs.duplicated(subset=keys, keep=False)] + if not duplicates.empty: + raise ValueError( + f"Duplicate custom constraint RHS rows for: " + f"{sorted(set(duplicates['constraint_name']))}" + ) + + +def _raise_on_unpaired_constraints(lhs: pd.DataFrame, rhs: pd.DataFrame) -> None: + """Raise if any constraint appears on only one side — a one-sided + constraint can't be applied.""" + lhs_names = set(lhs["constraint_name"]) + rhs_names = set(rhs["constraint_name"]) + if lhs_names != rhs_names: + raise ValueError( + f"Custom constraints with LHS terms but no RHS: " + f"{sorted(lhs_names - rhs_names)}; with RHS but no LHS terms: " + f"{sorted(rhs_names - lhs_names)}" + ) + + +def _finalise_generators(relaxation_generators: pd.DataFrame) -> pd.DataFrame: + """Drops the allowed_expansion working column carried for the + expansion-limit RHS, leaving the PyPSA generator columns.""" + return relaxation_generators.loc[:, _GENERATOR_COLUMNS].reset_index(drop=True) diff --git a/src/ispypsa/translator/mappings.py b/src/ispypsa/translator/mappings.py index ed6e25b2..615203b5 100644 --- a/src/ispypsa/translator/mappings.py +++ b/src/ispypsa/translator/mappings.py @@ -152,6 +152,10 @@ "generator_capacity": "Generator", "generator_output": "Generator", "load_consumption": "Load", + # "load" is the new-format vocabulary for "load_consumption" (PLEXOS Node + # Load Coefficient terms). Load variables aren't implemented in + # pypsa_build, which skips these terms with a log line. + "load": "Load", "storage_output": "Storage", } @@ -160,6 +164,7 @@ "generator_capacity": "p_nom", "generator_output": "p", "load_consumption": "p", + "load": "p", "storage_output": "p", } diff --git a/tests/test_translator/test_constraints.py b/tests/test_translator/test_constraints.py new file mode 100644 index 00000000..0aa7d64b --- /dev/null +++ b/tests/test_translator/test_constraints.py @@ -0,0 +1,327 @@ +import pandas as pd +import pytest + +from ispypsa.translator.constraints import ( + _translate_custom_constraints_from_network_tables, +) +from ispypsa.translator.helpers import _annuitised_investment_costs + +# Annuitised $1/MW at the sample_model_config's wacc (0.06) and annuitisation +# lifetime (25). +_ANNUITY_PER_DOLLAR = _annuitised_investment_costs(1.0, 0.06, 25) + + +def _constraint_tables(csv_str_to_df) -> dict[str, pd.DataFrame]: + """One PLEXOS-derived constraint (SWQLD1) with a link, a generator and a + storage term, timeslice-varying RHS, and a relaxation expansion option. + The sample_model_config's investment periods are 2026 and 2028.""" + tables = {} + tables["custom_constraints"] = csv_str_to_df(""" + constraint_id, direction + SWQLD1, <= + """) + tables["custom_constraints_lhs"] = csv_str_to_df(""" + constraint_id, term_type, variable_name, coefficient, date_from + SWQLD1, link_flow, NSW-QLD, 0.84, + SWQLD1, generator_output, KINGASF1, 0.14, + SWQLD1, storage_output, Q8 Battery - 2h, 0.43, + """) + tables["custom_constraints_rhs"] = csv_str_to_df(""" + constraint_id, timeslice, rhs, date_from + SWQLD1, qld_peak_demand, 3000, + SWQLD1, qld_winter_reference, 3500, + """) + tables["network_expansion_options"] = csv_str_to_df(""" + expansion_id, expansion_type, allowed_expansion, expansion_option + NSW-QLD, forward, 1000, NSW-QLD Option 1 + NSW-QLD, reverse, 900, NSW-QLD Option 1 + SWQLD1, constraint_relaxation, 400, SWQLD1 Option 2 + """) + tables["network_transmission_path_expansion_costs"] = csv_str_to_df(""" + expansion_id, year, cost + NSW-QLD, 2026, 500000 + SWQLD1, 2026, 100000 + """) + return tables + + +def _links(csv_str_to_df) -> pd.DataFrame: + return csv_str_to_df(""" + isp_name, name, p_nom_extendable + NSW-QLD, NSW-QLD_existing, False + NSW-QLD, NSW-QLD_exp_2026, True + """) + + +def test_translate_custom_constraints_rhs(csv_str_to_df, sample_model_config): + ispypsa_tables = _constraint_tables(csv_str_to_df) + + result = _translate_custom_constraints_from_network_tables( + ispypsa_tables, _links(csv_str_to_df), sample_model_config + ) + + expected_rhs = csv_str_to_df(""" + constraint_name, investment_period, timeslice, rhs, constraint_type + SWQLD1, 2026, qld_peak_demand, 3000, <= + SWQLD1, 2026, qld_winter_reference, 3500, <= + SWQLD1, 2028, qld_peak_demand, 3000, <= + SWQLD1, 2028, qld_winter_reference, 3500, <= + NSW-QLD_expansion_limit, , , 1000, <= + SWQLD1_expansion_limit, , , 400, <= + """) + sort_cols = ["constraint_name", "investment_period", "timeslice"] + pd.testing.assert_frame_equal( + result["custom_constraints_rhs"].sort_values(sort_cols).reset_index(drop=True), + expected_rhs.sort_values(sort_cols).reset_index(drop=True), + check_dtype=False, + ) + + +def test_translate_custom_constraints_lhs(csv_str_to_df, sample_model_config): + ispypsa_tables = _constraint_tables(csv_str_to_df) + + result = _translate_custom_constraints_from_network_tables( + ispypsa_tables, _links(csv_str_to_df), sample_model_config + ) + + expected_lhs = csv_str_to_df(""" + constraint_name, investment_period, variable_name, component, attribute, coefficient + SWQLD1, 2026, NSW-QLD_existing, Link, p, 0.84 + SWQLD1, 2026, NSW-QLD_exp_2026, Link, p, 0.84 + SWQLD1, 2028, NSW-QLD_existing, Link, p, 0.84 + SWQLD1, 2028, NSW-QLD_exp_2026, Link, p, 0.84 + SWQLD1, 2026, KINGASF1, Generator, p, 0.14 + SWQLD1, 2028, KINGASF1, Generator, p, 0.14 + SWQLD1, 2026, Q8 Battery - 2h, Storage, p, 0.43 + SWQLD1, 2028, Q8 Battery - 2h, Storage, p, 0.43 + SWQLD1, 2026, SWQLD1_exp_2026, Generator, p_nom, -1.0 + SWQLD1, 2028, SWQLD1_exp_2026, Generator, p_nom, -1.0 + NSW-QLD_expansion_limit, , NSW-QLD_exp_2026, Link, p_nom, 1.0 + SWQLD1_expansion_limit, , SWQLD1_exp_2026, Generator, p_nom, 1.0 + """) + sort_cols = ["constraint_name", "investment_period", "variable_name", "attribute"] + pd.testing.assert_frame_equal( + result["custom_constraints_lhs"].sort_values(sort_cols).reset_index(drop=True), + expected_lhs.sort_values(sort_cols).reset_index(drop=True), + check_dtype=False, + ) + + +def test_translate_custom_constraints_relaxation_generators( + csv_str_to_df, sample_model_config +): + ispypsa_tables = _constraint_tables(csv_str_to_df) + + result = _translate_custom_constraints_from_network_tables( + ispypsa_tables, _links(csv_str_to_df), sample_model_config + ) + + expected_generators = csv_str_to_df(f""" + name, isp_name, bus, p_nom, p_nom_extendable, build_year, lifetime, capital_cost + SWQLD1_exp_2026, SWQLD1, bus_for_custom_constraint_gens, 0.0, True, 2026, inf, {100000 * _ANNUITY_PER_DOLLAR} + """) + pd.testing.assert_frame_equal( + result["custom_constraints_generators"], + expected_generators, + check_dtype=False, + rtol=1e-5, + ) + + +def test_translate_custom_constraints_rez_expansion_disabled( + csv_str_to_df, sample_model_config +): + ispypsa_tables = _constraint_tables(csv_str_to_df) + sample_model_config.network.rez_transmission_expansion = False + + result = _translate_custom_constraints_from_network_tables( + ispypsa_tables, _links(csv_str_to_df), sample_model_config + ) + + expected_generators = csv_str_to_df(""" + name, isp_name, bus, p_nom, p_nom_extendable, build_year, lifetime, capital_cost + """) + pd.testing.assert_frame_equal( + result["custom_constraints_generators"], expected_generators, check_dtype=False + ) + lhs = result["custom_constraints_lhs"] + assert "SWQLD1_exp_2026" not in set(lhs["variable_name"]) + + +def test_date_from_resolved_at_period_starts(csv_str_to_df, sample_model_config): + """A value dated mid-FY2027 (i.e. before FY2028 starts on 2027-07-01) does + not apply in the 2026 period but does in 2028.""" + ispypsa_tables = _constraint_tables(csv_str_to_df) + ispypsa_tables["custom_constraints_rhs"] = csv_str_to_df(""" + constraint_id, timeslice, rhs, date_from + SWQLD1, qld_peak_demand, 3000, + SWQLD1, qld_peak_demand, 2500, 2026-12-01T00:00:00 + """) + + result = _translate_custom_constraints_from_network_tables( + ispypsa_tables, _links(csv_str_to_df), sample_model_config + ) + + rhs = result["custom_constraints_rhs"] + rhs = rhs[rhs["constraint_name"] == "SWQLD1"] + expected = csv_str_to_df(""" + constraint_name, investment_period, timeslice, rhs, constraint_type + SWQLD1, 2026, qld_peak_demand, 3000, <= + SWQLD1, 2028, qld_peak_demand, 2500, <= + """) + pd.testing.assert_frame_equal( + rhs.sort_values("investment_period").reset_index(drop=True), + expected, + check_dtype=False, + ) + + +def test_date_from_after_all_periods_contributes_nothing( + csv_str_to_df, sample_model_config +): + ispypsa_tables = _constraint_tables(csv_str_to_df) + ispypsa_tables["custom_constraints_lhs"] = csv_str_to_df(""" + constraint_id, term_type, variable_name, coefficient, date_from + SWQLD1, generator_output, KINGASF1, 0.14, + SWQLD1, generator_output, LATEGEN, 0.5, 2040-01-01T00:00:00 + """) + + result = _translate_custom_constraints_from_network_tables( + ispypsa_tables, _links(csv_str_to_df), sample_model_config + ) + + assert "LATEGEN" not in set(result["custom_constraints_lhs"]["variable_name"]) + + +def test_equality_direction_becomes_double_equals(csv_str_to_df, sample_model_config): + ispypsa_tables = _constraint_tables(csv_str_to_df) + ispypsa_tables["custom_constraints"] = csv_str_to_df(""" + constraint_id, direction + SWQLD1, = + """) + + result = _translate_custom_constraints_from_network_tables( + ispypsa_tables, _links(csv_str_to_df), sample_model_config + ) + + rhs = result["custom_constraints_rhs"] + assert set(rhs.loc[rhs["constraint_name"] == "SWQLD1", "constraint_type"]) == {"=="} + + +def test_link_terms_not_in_model_dropped_and_logged( + csv_str_to_df, sample_model_config, caplog +): + ispypsa_tables = _constraint_tables(csv_str_to_df) + ispypsa_tables["custom_constraints_lhs"] = csv_str_to_df(""" + constraint_id, term_type, variable_name, coefficient, date_from + SWQLD1, link_flow, TAS-SEV, 0.5, + SWQLD1, generator_output, KINGASF1, 0.14, + """) + + with caplog.at_level("INFO"): + result = _translate_custom_constraints_from_network_tables( + ispypsa_tables, _links(csv_str_to_df), sample_model_config + ) + + assert ( + "Custom constraint link_flow terms dropped (paths not in model): ['TAS-SEV']" + ) in caplog.text + assert "TAS-SEV" not in set(result["custom_constraints_lhs"]["variable_name"]) + + +def test_constraint_with_no_lhs_terms_dropped_and_logged( + csv_str_to_df, sample_model_config, caplog +): + ispypsa_tables = _constraint_tables(csv_str_to_df) + ispypsa_tables["custom_constraints"] = csv_str_to_df(""" + constraint_id, direction + SWQLD1, <= + NQ1, <= + """) + ispypsa_tables["custom_constraints_rhs"] = csv_str_to_df(""" + constraint_id, timeslice, rhs, date_from + SWQLD1, qld_peak_demand, 3000, + NQ1, qld_peak_demand, 2650, + """) + + with caplog.at_level("INFO"): + result = _translate_custom_constraints_from_network_tables( + ispypsa_tables, _links(csv_str_to_df), sample_model_config + ) + + assert ( + "Custom constraints dropped (no LHS terms in model): ['NQ1']" + ) in caplog.text + assert "NQ1" not in set(result["custom_constraints_rhs"]["constraint_name"]) + + +def test_raises_on_rhs_without_direction(csv_str_to_df, sample_model_config): + ispypsa_tables = _constraint_tables(csv_str_to_df) + ispypsa_tables["custom_constraints"] = csv_str_to_df(""" + constraint_id, direction + """) + + with pytest.raises(ValueError, match=r"no direction.*SWQLD1"): + _translate_custom_constraints_from_network_tables( + ispypsa_tables, _links(csv_str_to_df), sample_model_config + ) + + +def test_raises_on_duplicate_rhs_rows(csv_str_to_df, sample_model_config): + ispypsa_tables = _constraint_tables(csv_str_to_df) + ispypsa_tables["custom_constraints_rhs"] = csv_str_to_df(""" + constraint_id, timeslice, rhs, date_from + SWQLD1, qld_peak_demand, 3000, + SWQLD1, qld_peak_demand, 2500, + """) + + with pytest.raises(ValueError, match=r"Duplicate custom constraint RHS.*SWQLD1"): + _translate_custom_constraints_from_network_tables( + ispypsa_tables, _links(csv_str_to_df), sample_model_config + ) + + +def test_empty_custom_constraint_tables(csv_str_to_df, sample_model_config): + """At coarser granularities the custom-constraint tables are header-only; + the expansion-limit constraints for links are still produced.""" + ispypsa_tables = _constraint_tables(csv_str_to_df) + ispypsa_tables["custom_constraints"] = pd.DataFrame( + columns=["constraint_id", "direction"] + ) + ispypsa_tables["custom_constraints_lhs"] = pd.DataFrame( + columns=[ + "constraint_id", + "term_type", + "variable_name", + "coefficient", + "date_from", + ] + ) + ispypsa_tables["custom_constraints_rhs"] = pd.DataFrame( + columns=["constraint_id", "timeslice", "rhs", "date_from"] + ) + ispypsa_tables["network_expansion_options"] = csv_str_to_df(""" + expansion_id, expansion_type, allowed_expansion, expansion_option + NSW-QLD, forward, 1000, Option 1 + NSW-QLD, reverse, 900, Option 1 + """) + + result = _translate_custom_constraints_from_network_tables( + ispypsa_tables, _links(csv_str_to_df), sample_model_config + ) + + expected_lhs = csv_str_to_df(""" + constraint_name, investment_period, variable_name, component, attribute, coefficient + NSW-QLD_expansion_limit, , NSW-QLD_exp_2026, Link, p_nom, 1.0 + """) + pd.testing.assert_frame_equal( + result["custom_constraints_lhs"], expected_lhs, check_dtype=False + ) + + expected_rhs = csv_str_to_df(""" + constraint_name, investment_period, timeslice, rhs, constraint_type + NSW-QLD_expansion_limit, , , 1000, <= + """) + pd.testing.assert_frame_equal( + result["custom_constraints_rhs"], expected_rhs, check_dtype=False + ) From e35fd9a9aa1cc2fe75d556b629ba8c024ae286f0 Mon Sep 17 00:00:00 2001 From: nick-gorman Date: Thu, 11 Jun 2026 12:28:35 +1000 Subject: [PATCH 5/6] Apply timeslice limits and temporal-scope constraints in pypsa_build Links expand their per-unit timeslice limits into per-snapshot p_max_pu/p_min_pu series at network-build time, using the timeslice_snapshots mapping; snapshots outside any window keep the static (winter) value. Doing the expansion here rather than shipping per-link series from the translator keeps the pypsa-friendly directory small and reuses the same mapping table that scopes the custom constraints. New-format custom constraints build one linopy constraint per RHS row, restricting time-indexed variables to the row's investment period and timeslice via label-based selection on (period, timestamp) pairs. LHS terms for components not in the model (generators and batteries, until their translation lands) are skipped with a log line each; constraint instances with no snapshots in scope are skipped silently because the translator already warns about timeslices that never apply. build.py dispatches between the old and new constraint paths on the presence of the investment_period column. Co-Authored-By: Claude Fable 5 --- src/ispypsa/pypsa_build/build.py | 45 ++-- src/ispypsa/pypsa_build/custom_constraints.py | 185 ++++++++++++++++ src/ispypsa/pypsa_build/links.py | 81 ++++++- .../test_add_links_with_timeslice_limits.py | 72 +++++++ ..._custom_constraints_with_temporal_scope.py | 197 ++++++++++++++++++ 5 files changed, 566 insertions(+), 14 deletions(-) create mode 100644 tests/test_model/test_add_links_with_timeslice_limits.py create mode 100644 tests/test_model/test_custom_constraints_with_temporal_scope.py diff --git a/src/ispypsa/pypsa_build/build.py b/src/ispypsa/pypsa_build/build.py index 68b0d0f6..15a3208a 100644 --- a/src/ispypsa/pypsa_build/build.py +++ b/src/ispypsa/pypsa_build/build.py @@ -7,7 +7,10 @@ _add_buses_to_network, ) from ispypsa.pypsa_build.carriers import _add_carriers_to_network -from ispypsa.pypsa_build.custom_constraints import _add_custom_constraints +from ispypsa.pypsa_build.custom_constraints import ( + _add_custom_constraints, + _add_custom_constraints_with_temporal_scope, +) from ispypsa.pypsa_build.generators import ( _add_custom_constraint_generators_to_network, _add_generators_to_network, @@ -70,13 +73,20 @@ def build_pypsa_network( ) if "links" in pypsa_friendly_tables.keys(): - _add_links_to_network(network, pypsa_friendly_tables["links"]) + _add_links_to_network( + network, + pypsa_friendly_tables["links"], + pypsa_friendly_tables.get("link_timeslice_limits"), + pypsa_friendly_tables.get("timeslice_snapshots"), + ) - _add_generators_to_network( - network, - pypsa_friendly_tables["generators"], - path_to_pypsa_friendly_timeseries_data, - ) + # Absent under the new table format until generator translation lands. + if "generators" in pypsa_friendly_tables.keys(): + _add_generators_to_network( + network, + pypsa_friendly_tables["generators"], + path_to_pypsa_friendly_timeseries_data, + ) if "batteries" in pypsa_friendly_tables.keys(): _add_batteries_to_network(network, pypsa_friendly_tables["batteries"]) @@ -91,11 +101,22 @@ def build_pypsa_network( # The underlying linopy model needs to get built so we can add custom constraints. network.optimize.create_model(multi_investment_periods=True) + # FEATURE_FLAG_CLEANUP[use_new_table_format]: the temporal-scope path + # becomes the only path once the old format is deleted; the column check + # distinguishes the formats until then. if "custom_constraints_rhs" in pypsa_friendly_tables: - _add_custom_constraints( - network, - pypsa_friendly_tables["custom_constraints_rhs"], - pypsa_friendly_tables["custom_constraints_lhs"], - ) + if "investment_period" in pypsa_friendly_tables["custom_constraints_rhs"]: + _add_custom_constraints_with_temporal_scope( + network, + pypsa_friendly_tables["custom_constraints_rhs"], + pypsa_friendly_tables["custom_constraints_lhs"], + pypsa_friendly_tables["timeslice_snapshots"], + ) + else: + _add_custom_constraints( + network, + pypsa_friendly_tables["custom_constraints_rhs"], + pypsa_friendly_tables["custom_constraints_lhs"], + ) return network diff --git a/src/ispypsa/pypsa_build/custom_constraints.py b/src/ispypsa/pypsa_build/custom_constraints.py index bb345d26..b40a2880 100644 --- a/src/ispypsa/pypsa_build/custom_constraints.py +++ b/src/ispypsa/pypsa_build/custom_constraints.py @@ -5,6 +5,8 @@ import pandas as pd import pypsa +logger = logging.getLogger(__name__) + def _get_variables( model: linopy.Model, component_name: str, component_type: str, attribute_type: str @@ -115,3 +117,186 @@ def _add_custom_constraints( raise ValueError( f"{row['constraint_type']} is not a valid constraint type." ) + + +def _add_custom_constraints_with_temporal_scope( + network: pypsa.Network, + custom_constraints_rhs: pd.DataFrame, + custom_constraints_lhs: pd.DataFrame, + timeslice_snapshots: pd.DataFrame, +): + """Adds the new-format custom constraints to the `pypsa.Network`. + + Each RHS row creates one linopy constraint scoped to the row's + investment_period and timeslice: time-indexed LHS variables are + restricted to the snapshots inside that scope. NaN investment_period or + timeslice means unrestricted in that dimension (e.g. the expansion-limit + constraints, which only involve p_nom variables, carry NaN in both). + + LHS terms referencing components not in the model (e.g. generators while + generator translation for the new format is pending) are skipped with a + log line; constraint instances with no snapshots in scope or no terms in + the model are skipped silently — the translator logs the timeslices that + never apply. + + Args: + network: The `pypsa.Network` object + custom_constraints_rhs: `pd.DataFrame` with columns constraint_name, + investment_period, timeslice, rhs, and constraint_type. + custom_constraints_lhs: `pd.DataFrame` with columns constraint_name, + investment_period, variable_name, component, attribute, and + coefficient. A NaN investment_period term applies to every + instance of its constraint. + timeslice_snapshots: `pd.DataFrame` mapping timeslice_ids to the + snapshots they are active at (columns timeslice_id, + investment_periods, snapshots). + + Returns: None + """ + timeslice_labels = _timeslice_constraint_labels(timeslice_snapshots) + for row in custom_constraints_rhs.itertuples(): + snapshot_subset = _constraint_snapshot_subset( + row, timeslice_labels, network.snapshots + ) + if snapshot_subset is not None and len(snapshot_subset) == 0: + continue + terms = _select_lhs_terms(custom_constraints_lhs, row) + expression = _build_constraint_expression(network.model, terms, snapshot_subset) + if expression is None: + continue + _add_constraint_for_rhs_row(network.model, expression, row) + + +def _timeslice_constraint_labels( + timeslice_snapshots: pd.DataFrame, +) -> dict[str, list[tuple]]: + """The (investment_period, snapshot) labels each timeslice is active at. + + I/O Example: + timeslice_id=qld_peak_demand, investment_periods=2025, + snapshots=2025-01-13 12:00 + -> {"qld_peak_demand": [(2025, Timestamp("2025-01-13 12:00"))]} + """ + mapping = timeslice_snapshots.copy() + mapping["snapshots"] = pd.to_datetime(mapping["snapshots"]) + return { + timeslice_id: list(zip(rows["investment_periods"], rows["snapshots"])) + for timeslice_id, rows in mapping.groupby("timeslice_id") + } + + +def _constraint_snapshot_subset( + rhs_row, timeslice_labels: dict[str, list[tuple]], snapshots: pd.MultiIndex +) -> list[tuple] | None: + """The snapshot labels an RHS row's constraint is restricted to, or None + when the constraint is unrestricted (NaN timeslice and investment_period). + + I/O Example: + timeslice=qld_peak_demand, investment_period=2025, + timeslice_labels={"qld_peak_demand": [(2025, t1), (2030, t2)]} + -> [(2025, t1)] + + timeslice=NaN, investment_period=2025 -> all 2025 snapshots + timeslice=NaN, investment_period=NaN -> None + """ + if not pd.isna(rhs_row.timeslice): + labels = timeslice_labels.get(rhs_row.timeslice, []) + if not pd.isna(rhs_row.investment_period): + period = int(rhs_row.investment_period) + labels = [label for label in labels if label[0] == period] + return labels + if not pd.isna(rhs_row.investment_period): + period = int(rhs_row.investment_period) + return list(snapshots[snapshots.get_level_values(0) == period]) + return None + + +def _select_lhs_terms(custom_constraints_lhs: pd.DataFrame, rhs_row) -> pd.DataFrame: + """The LHS terms belonging to an RHS row's constraint instance: terms for + the same constraint whose investment_period matches the row's or is NaN + (applies to every instance). + + I/O Example: + lhs: + constraint_name investment_period variable_name + SWQLD1 2025 KINGASF1 + SWQLD1 2030 KINGASF1 + SWQLD1 , SWQLD1_exp_2025 + + rhs_row (SWQLD1, investment_period=2025) selects rows 1 and 3. + """ + terms = custom_constraints_lhs[ + custom_constraints_lhs["constraint_name"] == rhs_row.constraint_name + ] + if pd.isna(rhs_row.investment_period): + return terms + return terms[ + terms["investment_period"].isna() + | (terms["investment_period"] == rhs_row.investment_period) + ] + + +def _build_constraint_expression( + model: linopy.Model, terms: pd.DataFrame, snapshot_subset: list[tuple] | None +): + """Builds the linear expression sum(coefficient * variable) for a + constraint instance, restricting time-indexed variables to the snapshot + subset. Returns None when no term's variable is in the model.""" + expression_terms = [] + for term in terms.itertuples(): + variable = _get_variable_with_snapshot_subset( + model, term.variable_name, term.component, term.attribute, snapshot_subset + ) + if variable is not None: + expression_terms.append((term.coefficient, variable)) + if not expression_terms: + return None + return model.linexpr(*expression_terms) + + +def _get_variable_with_snapshot_subset( + model: linopy.Model, + component_name: str, + component_type: str, + attribute_type: str, + snapshot_subset: list[tuple] | None, +): + """Retrieves a variable like _get_variables, restricting time-indexed + ('p') variables to the snapshot subset. Components not in the model are + skipped with a log line rather than raising — new-format constraints + legitimately reference generators and batteries before those components + are translated.""" + try: + variable = _get_variables(model, component_name, component_type, attribute_type) + except KeyError: + logger.info( + f"{component_type} {component_name} not in model, " + f"custom constraint term skipped." + ) + return None + if variable is not None and attribute_type == "p" and snapshot_subset is not None: + variable = variable.loc[snapshot_subset] + return variable + + +def _add_constraint_for_rhs_row(model: linopy.Model, expression, rhs_row) -> None: + """Adds one constraint, named uniquely for the RHS row's temporal scope. + + I/O Example: + (SWQLD1, 2025, qld_peak_demand) -> "SWQLD1_2025_qld_peak_demand" + (CQ-NQ_expansion_limit, NaN, NaN) -> "CQ-NQ_expansion_limit" + """ + name_parts = [rhs_row.constraint_name] + if not pd.isna(rhs_row.investment_period): + name_parts.append(str(int(rhs_row.investment_period))) + if not pd.isna(rhs_row.timeslice): + name_parts.append(rhs_row.timeslice) + name = "_".join(name_parts) + if rhs_row.constraint_type == "<=": + model.add_constraints(expression <= rhs_row.rhs, name=name) + elif rhs_row.constraint_type == ">=": + model.add_constraints(expression >= rhs_row.rhs, name=name) + elif rhs_row.constraint_type == "==": + model.add_constraints(expression == rhs_row.rhs, name=name) + else: + raise ValueError(f"{rhs_row.constraint_type} is not a valid constraint type.") diff --git a/src/ispypsa/pypsa_build/links.py b/src/ispypsa/pypsa_build/links.py index 36652bde..811a1324 100644 --- a/src/ispypsa/pypsa_build/links.py +++ b/src/ispypsa/pypsa_build/links.py @@ -2,15 +2,92 @@ import pypsa -def _add_links_to_network(network: pypsa.Network, links: pd.DataFrame) -> None: +def _add_links_to_network( + network: pypsa.Network, + links: pd.DataFrame, + link_timeslice_limits: pd.DataFrame | None = None, + timeslice_snapshots: pd.DataFrame | None = None, +) -> None: """Adds the Links defined in a pypsa-friendly input table called `"links"` to the `pypsa.Network` object. + When the new-format per-timeslice limit tables are given, links with + timeslice-varying limits get per-snapshot p_max_pu / p_min_pu series + instead of their static values (see _build_link_pu_overrides). + Args: network: The `pypsa.Network` object links: `pd.DataFrame` with `PyPSA` style `Link` attributes. + link_timeslice_limits: `pd.DataFrame` with per-timeslice per-unit + limits (columns name, attribute, timeslice, value), or None when + all link limits are static. + timeslice_snapshots: `pd.DataFrame` mapping timeslice_ids to the + snapshots they are active at (columns timeslice_id, + investment_periods, snapshots). Required when + link_timeslice_limits is given. Returns: None """ + pu_overrides = _build_link_pu_overrides( + link_timeslice_limits, timeslice_snapshots, links, network.snapshots + ) links["class_name"] = "Link" - links.apply(lambda row: network.add(**row.to_dict()), axis=1) + for _, row in links.iterrows(): + network.add(**(row.to_dict() | pu_overrides.get(row["name"], {}))) + + +def _build_link_pu_overrides( + link_timeslice_limits: pd.DataFrame | None, + timeslice_snapshots: pd.DataFrame | None, + links: pd.DataFrame, + snapshots: pd.MultiIndex, +) -> dict[str, dict[str, pd.Series]]: + """Expands each link's per-timeslice limits into per-snapshot series. + + Each series starts at the link's static value (1.0 for p_max_pu, the + links table's p_min_pu for p_min_pu — both equivalent to the + winter_reference limit) and gets each timeslice's value at the snapshots + that timeslice is active. Timeslices with no snapshots simply leave the + static value in place — the translator has already logged them. + + I/O Example: + link_timeslice_limits: + name attribute timeslice value + CQ-NQ_existing p_max_pu qld_peak_demand 0.857 + + timeslice_snapshots: qld_peak_demand active at (2025, 2025-01-13 12:00) + snapshots: (2025, 2025-01-13 12:00), (2025, 2025-01-15 12:00) + + returns: + {"CQ-NQ_existing": {"p_max_pu": series [0.857, 1.0]}} + """ + if link_timeslice_limits is None or link_timeslice_limits.empty: + return {} + timeslice_labels = _timeslice_snapshot_labels(timeslice_snapshots) + static_p_min_pu = links.set_index("name")["p_min_pu"] + overrides = {} + for (name, attribute), rows in link_timeslice_limits.groupby(["name", "attribute"]): + static_value = 1.0 if attribute == "p_max_pu" else static_p_min_pu[name] + series = pd.Series(static_value, index=snapshots, dtype=float) + for row in rows.itertuples(): + series.loc[timeslice_labels.get(row.timeslice, [])] = row.value + overrides.setdefault(name, {})[attribute] = series + return overrides + + +def _timeslice_snapshot_labels( + timeslice_snapshots: pd.DataFrame, +) -> dict[str, list[tuple]]: + """The (investment_period, snapshot) labels each timeslice is active at. + + I/O Example: + timeslice_id=qld_peak_demand, investment_periods=2025, + snapshots=2025-01-13 12:00 + -> {"qld_peak_demand": [(2025, Timestamp("2025-01-13 12:00"))]} + """ + mapping = timeslice_snapshots.copy() + mapping["snapshots"] = pd.to_datetime(mapping["snapshots"]) + return { + timeslice_id: list(zip(rows["investment_periods"], rows["snapshots"])) + for timeslice_id, rows in mapping.groupby("timeslice_id") + } diff --git a/tests/test_model/test_add_links_with_timeslice_limits.py b/tests/test_model/test_add_links_with_timeslice_limits.py new file mode 100644 index 00000000..624945d5 --- /dev/null +++ b/tests/test_model/test_add_links_with_timeslice_limits.py @@ -0,0 +1,72 @@ +import pandas as pd +import pypsa +import pytest + +from ispypsa.pypsa_build.links import _add_links_to_network + + +def _network() -> pypsa.Network: + snapshots = pd.date_range("2025-01-01", periods=4, freq="h") + index = pd.MultiIndex.from_arrays([[2025] * 4, list(snapshots)]) + network = pypsa.Network(snapshots=index, investment_periods=[2025]) + network.add("Bus", "bus1") + network.add("Bus", "bus2") + return network + + +def _links(csv_str_to_df) -> pd.DataFrame: + return csv_str_to_df(""" + name, bus0, bus1, carrier, p_nom, p_min_pu, p_nom_extendable + CQ-NQ_existing, bus1, bus2, AC, 1400, -1.364286, False + """) + + +def test_links_get_per_snapshot_pu_series(csv_str_to_df): + network = _network() + link_timeslice_limits = csv_str_to_df(""" + name, attribute, timeslice, value + CQ-NQ_existing, p_max_pu, qld_peak_demand, 0.857143 + CQ-NQ_existing, p_min_pu, qld_peak_demand, -1.028571 + """) + timeslice_snapshots = csv_str_to_df(""" + timeslice_id, investment_periods, snapshots + qld_peak_demand, 2025, 2025-01-01 01:00:00 + qld_peak_demand, 2025, 2025-01-01 02:00:00 + """) + + _add_links_to_network( + network, _links(csv_str_to_df), link_timeslice_limits, timeslice_snapshots + ) + + p_max_pu = network.links_t.p_max_pu["CQ-NQ_existing"] + p_min_pu = network.links_t.p_min_pu["CQ-NQ_existing"] + # The static (winter) limit holds outside the tagged snapshots. + expected_p_max_pu = [1.0, 0.857143, 0.857143, 1.0] + expected_p_min_pu = [-1.364286, -1.028571, -1.028571, -1.364286] + assert p_max_pu.tolist() == pytest.approx(expected_p_max_pu) + assert p_min_pu.tolist() == pytest.approx(expected_p_min_pu) + + +def test_links_without_timeslice_limits_stay_static(csv_str_to_df): + network = _network() + link_timeslice_limits = csv_str_to_df(""" + name, attribute, timeslice, value + """) + timeslice_snapshots = csv_str_to_df(""" + timeslice_id, investment_periods, snapshots + """) + + _add_links_to_network( + network, _links(csv_str_to_df), link_timeslice_limits, timeslice_snapshots + ) + + assert "CQ-NQ_existing" not in network.links_t.p_max_pu.columns + assert network.links.loc["CQ-NQ_existing", "p_min_pu"] == pytest.approx(-1.364286) + + +def test_links_old_format_call_without_limit_tables(csv_str_to_df): + network = _network() + + _add_links_to_network(network, _links(csv_str_to_df)) + + assert network.links.loc["CQ-NQ_existing", "p_nom"] == 1400 diff --git a/tests/test_model/test_custom_constraints_with_temporal_scope.py b/tests/test_model/test_custom_constraints_with_temporal_scope.py new file mode 100644 index 00000000..1ed3856f --- /dev/null +++ b/tests/test_model/test_custom_constraints_with_temporal_scope.py @@ -0,0 +1,197 @@ +import pandas as pd +import pypsa +import pytest + +from ispypsa.pypsa_build.custom_constraints import ( + _add_custom_constraints_with_temporal_scope, +) + + +def _two_period_network() -> pypsa.Network: + """Two investment periods (2025, 2030) with four hourly snapshots each, + one bus, a cheap and an expensive generator, and a constant 100 MW load.""" + snapshots_2025 = pd.date_range("2025-01-01", periods=4, freq="h") + snapshots_2030 = pd.date_range("2030-01-01", periods=4, freq="h") + index = pd.MultiIndex.from_arrays( + [[2025] * 4 + [2030] * 4, list(snapshots_2025) + list(snapshots_2030)] + ) + network = pypsa.Network(snapshots=index, investment_periods=[2025, 2030]) + network.investment_period_weightings = pd.DataFrame( + {"years": [5, 5], "objective": [1.0, 1.0]}, + index=pd.Index([2025, 2030], name="period"), + ) + network.add("Bus", "bus1") + network.add("Generator", "cheap_gen", bus="bus1", p_nom=200, marginal_cost=10) + network.add("Generator", "dear_gen", bus="bus1", p_nom=200, marginal_cost=100) + network.add("Load", "load1", bus="bus1") + network.loads_t.p_set = pd.DataFrame({"load1": [100] * 8}, index=network.snapshots) + return network + + +def _timeslice_snapshots(csv_str_to_df) -> pd.DataFrame: + """peak_2025 tags the middle two snapshots of the 2025 period.""" + mapping = csv_str_to_df(""" + timeslice_id, investment_periods, snapshots + peak_2025, 2025, 2025-01-01 01:00:00 + peak_2025, 2025, 2025-01-01 02:00:00 + """) + return mapping + + +def test_timeslice_restricted_constraint_binds_only_at_tagged_snapshots( + csv_str_to_df, +): + network = _two_period_network() + rhs = csv_str_to_df(""" + constraint_name, investment_period, timeslice, rhs, constraint_type + cheap_gen_limit, 2025, peak_2025, 40, <= + """) + lhs = csv_str_to_df(""" + constraint_name, investment_period, variable_name, component, attribute, coefficient + cheap_gen_limit, 2025, cheap_gen, Generator, p, 1.0 + """) + + network.optimize.create_model(multi_investment_periods=True) + _add_custom_constraints_with_temporal_scope( + network, rhs, lhs, _timeslice_snapshots(csv_str_to_df) + ) + network.optimize.solve_model() + + cheap_gen_output = network.generators_t.p["cheap_gen"] + # Constrained to 40 at the two tagged 2025 snapshots; free (meets the + # whole 100 MW load) everywhere else, including all of 2030. + expected = pd.Series( + [100.0, 40.0, 40.0, 100.0, 100.0, 100.0, 100.0, 100.0], + index=network.snapshots, + name="cheap_gen", + ) + pd.testing.assert_series_equal(cheap_gen_output, expected, check_names=False) + + +def test_rhs_varies_by_investment_period(csv_str_to_df): + network = _two_period_network() + rhs = csv_str_to_df(""" + constraint_name, investment_period, timeslice, rhs, constraint_type + cheap_gen_limit, 2025, , 40, <= + cheap_gen_limit, 2030, , 60, <= + """) + lhs = csv_str_to_df(""" + constraint_name, investment_period, variable_name, component, attribute, coefficient + cheap_gen_limit, 2025, cheap_gen, Generator, p, 1.0 + cheap_gen_limit, 2030, cheap_gen, Generator, p, 1.0 + """) + + network.optimize.create_model(multi_investment_periods=True) + _add_custom_constraints_with_temporal_scope( + network, rhs, lhs, _timeslice_snapshots(csv_str_to_df) + ) + network.optimize.solve_model() + + cheap_gen_output = network.generators_t.p["cheap_gen"] + assert (cheap_gen_output.loc[(2025,)] == 40.0).all() + assert (cheap_gen_output.loc[(2030,)] == 60.0).all() + + +def test_unrestricted_p_nom_constraint(csv_str_to_df): + """NaN investment_period and timeslice (the expansion-limit pattern) + creates one global constraint over p_nom variables.""" + network = _two_period_network() + network.add( + "Generator", + "exp_gen", + bus="bus1", + p_nom_extendable=True, + capital_cost=1, + marginal_cost=1, + build_year=2025, + lifetime=100, + ) + rhs = csv_str_to_df(""" + constraint_name, investment_period, timeslice, rhs, constraint_type + exp_limit, , , 30, <= + """) + lhs = csv_str_to_df(""" + constraint_name, investment_period, variable_name, component, attribute, coefficient + exp_limit, , exp_gen, Generator, p_nom, 1.0 + """) + + network.optimize.create_model(multi_investment_periods=True) + _add_custom_constraints_with_temporal_scope( + network, rhs, lhs, _timeslice_snapshots(csv_str_to_df) + ) + network.optimize.solve_model() + + # exp_gen is the cheapest source so it builds to the constraint's cap. + assert network.generators.loc["exp_gen", "p_nom_opt"] == pytest.approx(30.0) + + +def test_terms_for_components_not_in_model_skipped_and_logged(csv_str_to_df, caplog): + """Generator and battery terms reference IASR IDs until generator + translation lands — they are skipped per-miss with a log line and the + rest of the constraint still applies.""" + network = _two_period_network() + rhs = csv_str_to_df(""" + constraint_name, investment_period, timeslice, rhs, constraint_type + cheap_gen_limit, 2025, peak_2025, 40, <= + """) + lhs = csv_str_to_df(""" + constraint_name, investment_period, variable_name, component, attribute, coefficient + cheap_gen_limit, 2025, cheap_gen, Generator, p, 1.0 + cheap_gen_limit, 2025, KINGASF1, Generator, p, 0.5 + """) + + network.optimize.create_model(multi_investment_periods=True) + with caplog.at_level("INFO"): + _add_custom_constraints_with_temporal_scope( + network, rhs, lhs, _timeslice_snapshots(csv_str_to_df) + ) + network.optimize.solve_model() + + assert ( + "Generator KINGASF1 not in model, custom constraint term skipped." + ) in caplog.text + # The cheap_gen term still binds at the tagged snapshots. + assert network.generators_t.p["cheap_gen"].loc[(2025,)].iloc[1] == pytest.approx( + 40.0 + ) + + +def test_constraint_with_no_terms_in_model_skipped(csv_str_to_df): + network = _two_period_network() + rhs = csv_str_to_df(""" + constraint_name, investment_period, timeslice, rhs, constraint_type + ghost_limit, 2025, peak_2025, 40, <= + """) + lhs = csv_str_to_df(""" + constraint_name, investment_period, variable_name, component, attribute, coefficient + ghost_limit, 2025, KINGASF1, Generator, p, 1.0 + """) + + network.optimize.create_model(multi_investment_periods=True) + _add_custom_constraints_with_temporal_scope( + network, rhs, lhs, _timeslice_snapshots(csv_str_to_df) + ) + network.optimize.solve_model() + + # No constraint applied — the cheap generator meets the whole load. + assert (network.generators_t.p["cheap_gen"] == 100.0).all() + + +def test_constraint_for_timeslice_with_no_snapshots_skipped(csv_str_to_df): + network = _two_period_network() + rhs = csv_str_to_df(""" + constraint_name, investment_period, timeslice, rhs, constraint_type + cheap_gen_limit, 2025, tas_peak_demand, 40, <= + """) + lhs = csv_str_to_df(""" + constraint_name, investment_period, variable_name, component, attribute, coefficient + cheap_gen_limit, 2025, cheap_gen, Generator, p, 1.0 + """) + + network.optimize.create_model(multi_investment_periods=True) + _add_custom_constraints_with_temporal_scope( + network, rhs, lhs, _timeslice_snapshots(csv_str_to_df) + ) + network.optimize.solve_model() + + assert (network.generators_t.p["cheap_gen"] == 100.0).all() From 878a716aa3180a8c80cd6b4a823f4a5ec19d47a0 Mon Sep 17 00:00:00 2001 From: nick-gorman Date: Thu, 11 Jun 2026 12:29:12 +1000 Subject: [PATCH 6/6] Wire the new-format translator path through the orchestrator and CLI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit create_pypsa_friendly_inputs dispatches to the network-tables path under the use_new_table_format flag. Snapshots come straight from the temporal config — aggregation methods needing demand traces aren't available until generator translation lands — and the output set covers the network and custom-constraint functionality only: no generators, batteries, or timeseries yet, so the CLI skips the timeseries task and list_timeseries_files returns nothing under the flag. Co-Authored-By: Claude Fable 5 --- src/ispypsa/cli/dodo.py | 24 +-- .../translator/create_pypsa_friendly.py | 101 +++++++++++- ...pypsa_friendly_inputs_new_table_formats.py | 151 ++++++++++++++++++ ...create_pypsa_friendly_inputs_new_format.py | 142 ++++++++++++++++ 4 files changed, 408 insertions(+), 10 deletions(-) create mode 100644 tests/test_cli/test_create_pypsa_friendly_inputs_new_table_formats.py create mode 100644 tests/test_translator/test_create_pypsa_friendly_inputs_new_format.py diff --git a/src/ispypsa/cli/dodo.py b/src/ispypsa/cli/dodo.py index 55a39087..cac01b4a 100644 --- a/src/ispypsa/cli/dodo.py +++ b/src/ispypsa/cli/dodo.py @@ -15,6 +15,7 @@ read_csvs, write_csvs, ) +from ispypsa.feature_flags import FEATURE_FLAGS from ispypsa.iasr_table_caching import build_local_cache, list_cache_files from ispypsa.logging import configure_logging from ispypsa.plotting import ( @@ -483,15 +484,20 @@ def create_pypsa_inputs_for_capacity_expansion_model() -> None: ispypsa_tables = read_csvs(input_tables_dir) pypsa_tables = create_pypsa_friendly_inputs(config, ispypsa_tables) - # Create capacity expansion timeseries - pypsa_tables["snapshots"] = create_pypsa_friendly_timeseries_inputs( - config, - "capacity_expansion", - ispypsa_tables, - pypsa_tables["generators"], - parsed_trace_dir, - capacity_expansion_timeseries_location, - ) + # FEATURE_FLAG_CLEANUP[use_new_table_format]: revisit once generator + # translation lands for the new format. Until then the new-format path + # has no generators, so no timeseries inputs are created and snapshots + # come from create_pypsa_friendly_inputs itself. + if not FEATURE_FLAGS["use_new_table_format"]: + # Create capacity expansion timeseries + pypsa_tables["snapshots"] = create_pypsa_friendly_timeseries_inputs( + config, + "capacity_expansion", + ispypsa_tables, + pypsa_tables["generators"], + parsed_trace_dir, + capacity_expansion_timeseries_location, + ) write_csvs(pypsa_tables, pypsa_friendly_dir) diff --git a/src/ispypsa/translator/create_pypsa_friendly.py b/src/ispypsa/translator/create_pypsa_friendly.py index 285fe7b1..b3037eb2 100644 --- a/src/ispypsa/translator/create_pypsa_friendly.py +++ b/src/ispypsa/translator/create_pypsa_friendly.py @@ -8,6 +8,7 @@ from ispypsa.config import ( ModelConfig, ) +from ispypsa.feature_flags import FEATURE_FLAGS from ispypsa.translator.buses import ( _create_single_region_bus, _translate_isp_sub_regions_to_buses, @@ -15,6 +16,9 @@ _translate_rezs_to_buses, create_pypsa_friendly_bus_demand_timeseries, ) +from ispypsa.translator.constraints import ( + _translate_custom_constraints_from_network_tables, +) from ispypsa.translator.custom_constraints import ( _append_if_not_empty, _translate_custom_constraints, @@ -29,6 +33,10 @@ ) from ispypsa.translator.helpers import convert_to_numeric_if_possible from ispypsa.translator.links import _translate_flow_paths_to_links +from ispypsa.translator.network import ( + _translate_network_geography_to_buses, + _translate_network_to_links, +) from ispypsa.translator.renewable_energy_zones import ( _translate_renewable_energy_zone_build_limits_to_links, ) @@ -43,6 +51,10 @@ ) from ispypsa.translator.temporal_filters import _time_series_filter from ispypsa.translator.time_series_checker import _check_time_series +from ispypsa.translator.timeslices import ( + _create_timeslice_snapshot_mapping, + _log_referenced_timeslices_without_snapshots, +) _BASE_TRANSLATOR_OUTPUTS = [ "snapshots", @@ -56,6 +68,26 @@ "custom_constraints_generators", ] +# Outputs of the new-format translator path. Generators and batteries are +# absent because the new-format templater doesn't produce generator/battery +# tables yet; link_timeslice_limits and timeslice_snapshots carry the +# time-varying transmission limits and the snapshot-restriction of custom +# constraints. +# FEATURE_FLAG_CLEANUP[use_new_table_format]: rename to _TRANSLATOR_OUTPUTS +# and delete _BASE_TRANSLATOR_OUTPUTS above (reinstating generators and +# batteries once generator translation lands for the new format). +_NEW_FORMAT_TRANSLATOR_OUTPUTS = [ + "snapshots", + "investment_period_weights", + "buses", + "links", + "link_timeslice_limits", + "timeslice_snapshots", + "custom_constraints_lhs", + "custom_constraints_rhs", + "custom_constraints_generators", +] + def create_pypsa_friendly_inputs( config: ModelConfig, ispypsa_tables: dict[str, pd.DataFrame] @@ -90,6 +122,11 @@ def create_pypsa_friendly_inputs( Returns: dictionary of dataframes in the `PyPSA` friendly format. (add link to pypsa friendly format table docs) """ + # FEATURE_FLAG_CLEANUP[use_new_table_format]: inline the new-format + # function here and delete the legacy body below. + if FEATURE_FLAGS["use_new_table_format"]: + return _create_pypsa_friendly_inputs_from_network_tables(config, ispypsa_tables) + pypsa_inputs = {} pypsa_inputs["investment_period_weights"] = _create_investment_period_weightings( @@ -207,6 +244,57 @@ def create_pypsa_friendly_inputs( return pypsa_inputs +def _create_pypsa_friendly_inputs_from_network_tables( + config: ModelConfig, ispypsa_tables: dict[str, pd.DataFrame] +) -> dict[str, pd.DataFrame]: + """Creates the PyPSA friendly tables from the new-format network tables. + + Covers the network and custom-constraint functionality only: the + new-format templater doesn't produce generator/battery tables yet, so no + generators or batteries (and none of their timeseries) are created. + Snapshots are created here directly from the temporal config — snapshot + aggregation methods that need demand/generator traces (e.g. + named_representative_weeks) aren't available until generator translation + lands. + + Args: + config: `ISPyPSA` `ispypsa.config.ModelConfig` object. + ispypsa_tables: dictionary of dataframes providing the new-format + `ISPyPSA` input tables. + + Returns: dictionary of dataframes in the `PyPSA` friendly format, with + the tables listed in _NEW_FORMAT_TRANSLATOR_OUTPUTS. + """ + pypsa_inputs = {} + snapshots = create_pypsa_friendly_snapshots(config, "capacity_expansion") + pypsa_inputs["snapshots"] = _add_snapshot_weightings( + snapshots, config.temporal.capacity_expansion.resolution_min + ) + pypsa_inputs["investment_period_weights"] = _create_investment_period_weightings( + config.temporal.capacity_expansion.investment_periods, + config.temporal.range.end_year, + config.discount_rate, + ) + pypsa_inputs["buses"] = _translate_network_geography_to_buses( + ispypsa_tables["network_geography"], config.network.nodes.rezs + ) + links, link_timeslice_limits = _translate_network_to_links(ispypsa_tables, config) + pypsa_inputs["links"] = links + pypsa_inputs["link_timeslice_limits"] = link_timeslice_limits + pypsa_inputs["timeslice_snapshots"] = _create_timeslice_snapshot_mapping( + ispypsa_tables["timeslices"], snapshots, config + ) + pypsa_inputs.update( + _translate_custom_constraints_from_network_tables(ispypsa_tables, links, config) + ) + _log_referenced_timeslices_without_snapshots( + pypsa_inputs["timeslice_snapshots"], + pypsa_inputs["link_timeslice_limits"], + pypsa_inputs["custom_constraints_rhs"], + ) + return pypsa_inputs + + def create_pypsa_friendly_timeseries_inputs( config: ModelConfig, model_phase: Literal["capacity_expansion", "operational"], @@ -392,6 +480,13 @@ def list_timeseries_files( KeyError: If required ISPyPSA tables are missing ValueError: If required columns are missing from tables """ + # FEATURE_FLAG_CLEANUP[use_new_table_format]: revisit once generator + # translation lands for the new format. Until then the new-format path + # produces no timeseries files — there are no generators to build traces + # for, and demand traces need the generator-era timeseries machinery. + if FEATURE_FLAGS["use_new_table_format"]: + return [] + files = [] # Validate required tables exist @@ -516,7 +611,11 @@ def _filter_and_save_timeseries( def list_translator_output_files(output_path: Path | None = None) -> list[Path]: - files = _BASE_TRANSLATOR_OUTPUTS + # FEATURE_FLAG_CLEANUP[use_new_table_format]: drop the legacy list. + if FEATURE_FLAGS["use_new_table_format"]: + files = _NEW_FORMAT_TRANSLATOR_OUTPUTS + else: + files = _BASE_TRANSLATOR_OUTPUTS if output_path is not None: files = [output_path / Path(file + ".csv") for file in files] return files diff --git a/tests/test_cli/test_create_pypsa_friendly_inputs_new_table_formats.py b/tests/test_cli/test_create_pypsa_friendly_inputs_new_table_formats.py new file mode 100644 index 00000000..fc2448be --- /dev/null +++ b/tests/test_cli/test_create_pypsa_friendly_inputs_new_table_formats.py @@ -0,0 +1,151 @@ +"""Tests for the create_pypsa_friendly_inputs CLI task under the new table format. + +Mirrors test_create_pypsa_friendly_inputs.py (fresh run, up-to-date detection, +and triggers) for the new-format translator path: network and custom-constraint +tables only, no generators/batteries and no timeseries directory. + +FEATURE_FLAG_CLEANUP[use_new_table_format]: merge into +test_create_pypsa_friendly_inputs.py when the legacy format is removed. +""" + +import time + +import pandas as pd + +from ispypsa.translator.create_pypsa_friendly import _NEW_FORMAT_TRANSLATOR_OUTPUTS + +from .cli_test_helpers import ( + assert_task_ran, + assert_task_up_to_date, + get_file_timestamps, + mock_workbook_file, + modify_config_value, + run_cli_command, + run_extensive, + verify_output_files, +) +from .cli_test_helpers_new_table_formats import ( + mock_config_new_format, + prepare_test_cache_new_format, +) + + +def test_create_pypsa_friendly_inputs_task_new_format( + mock_config_new_format, + prepare_test_cache_new_format, + tmp_path, + run_cli_command, + monkeypatch, + run_extensive, +): + """Test fresh run, up-to-date detection, and triggers for the new format. + + Combines: + - Fresh run creates the new-format translator outputs (and no timeseries) + - Up-to-date detection + - Config file modification trigger (extensive only) + - Dependency modification trigger (extensive only) + - Missing single file trigger (extensive only) + + Runs: 2 (fresh + up-to-date), or 5 with extensive + """ + monkeypatch.setenv("ISPYPSA_USE_NEW_TABLE_FORMAT", "true") + monkeypatch.setenv("ISPYPSA_TEST_MOCK_CACHE", "true") + + # Test fresh run (doit also runs the upstream create_ispypsa_inputs task). + result = run_cli_command( + [f"config={mock_config_new_format}", "create_pypsa_friendly_inputs"] + ) + assert result.returncode == 0, ( + f"Command failed with return code {result.returncode}\n" + f"STDOUT:\n{result.stdout}\nSTDERR:\n{result.stderr}" + ) + + output_dir = tmp_path / "run_dir" / "test_run" / "pypsa_friendly" + assert output_dir.exists() + verify_output_files(output_dir, _NEW_FORMAT_TRANSLATOR_OUTPUTS) + + # No generators/batteries and no timeseries until generator translation + # lands for the new format. + assert not (output_dir / "generators.csv").exists() + assert not (output_dir / "batteries.csv").exists() + assert not (output_dir / "capacity_expansion_timeseries").exists() + + # Spot-check the wiring of the outputs (content is covered by the + # translator unit tests). 15 sub-regions + 47 REZs at 7.5. + buses = pd.read_csv(output_dir / "buses.csv") + assert len(buses) == 62 + + links = pd.read_csv(output_dir / "links.csv") + limits = pd.read_csv(output_dir / "link_timeslice_limits.csv") + assert set(limits["name"]) <= set(links["name"]) + + lhs = pd.read_csv(output_dir / "custom_constraints_lhs.csv") + rhs = pd.read_csv(output_dir / "custom_constraints_rhs.csv") + assert len(rhs) > 0 + assert set(lhs["constraint_name"]) == set(rhs["constraint_name"]) + + mapping = pd.read_csv(output_dir / "timeslice_snapshots.csv") + snapshots = pd.read_csv(output_dir / "snapshots.csv") + assert len(mapping) > 0 + assert set(mapping["snapshots"]) <= set(snapshots["snapshots"]) + + # Get timestamps from first run + first_run_timestamps = get_file_timestamps(output_dir) + time.sleep(0.1) + + # Test up-to-date detection - second run + result = run_cli_command( + [f"config={mock_config_new_format}", "create_pypsa_friendly_inputs"] + ) + assert result.returncode == 0 + assert_task_up_to_date(result.stdout, "create_pypsa_friendly_inputs") + + second_run_timestamps = get_file_timestamps(output_dir) + assert first_run_timestamps == second_run_timestamps + + if not run_extensive: + return + + # Test config file modification triggers rerun (extensive only) + time.sleep(0.1) + modify_config_value(mock_config_new_format, "discount_rate", 0.06) + result = run_cli_command( + [f"config={mock_config_new_format}", "create_pypsa_friendly_inputs"] + ) + assert result.returncode == 0 + assert_task_ran(result.stdout, "create_pypsa_friendly_inputs") + + # Test dependency modification trigger (extensive only): edit a templated + # input the translator consumes. + time.sleep(0.1) + limits_file = ( + tmp_path + / "run_dir" + / "test_run" + / "ispypsa_inputs" + / "network_transmission_path_limits.csv" + ) + df = pd.read_csv(limits_file) + df["capacity"] = df["capacity"] + 1.0 + df.to_csv(limits_file, index=False) + + result = run_cli_command( + [f"config={mock_config_new_format}", "create_pypsa_friendly_inputs"] + ) + assert result.returncode == 0 + assert_task_ran(result.stdout, "create_pypsa_friendly_inputs") + + config_change_timestamps = get_file_timestamps(output_dir) + + # Test missing single file trigger (extensive only) + target_file = output_dir / "buses.csv" + target_file.unlink() + + result = run_cli_command( + [f"config={mock_config_new_format}", "create_pypsa_friendly_inputs"] + ) + assert result.returncode == 0 + assert_task_ran(result.stdout, "create_pypsa_friendly_inputs") + assert target_file.exists() + assert target_file.stat().st_size > 0 diff --git a/tests/test_translator/test_create_pypsa_friendly_inputs_new_format.py b/tests/test_translator/test_create_pypsa_friendly_inputs_new_format.py new file mode 100644 index 00000000..4f1e8a8a --- /dev/null +++ b/tests/test_translator/test_create_pypsa_friendly_inputs_new_format.py @@ -0,0 +1,142 @@ +"""Integration test for the new-format translator orchestrator. + +Verifies wiring only — the expected output keys are present, each output has +the expected column set, and row counts flow through. Detailed content is +covered by the per-module tests (test_network.py, test_constraints.py, +test_timeslice_snapshots.py). +""" + +from unittest.mock import patch + +import pandas as pd +import pytest + +from ispypsa.translator import create_pypsa_friendly_inputs +from ispypsa.translator.create_pypsa_friendly import list_translator_output_files + + +@pytest.fixture +def new_format_ispypsa_tables(csv_str_to_df) -> dict[str, pd.DataFrame]: + tables = {} + tables["network_geography"] = csv_str_to_df(""" + geo_id, geo_type, region_id + NQ, subregion, QLD + CQ, subregion, QLD + Q1, rez, QLD + """) + tables["network_transmission_paths"] = csv_str_to_df(""" + path_id, geo_from, geo_to, carrier + CQ-NQ, CQ, NQ, AC + Q1-NQ, Q1, NQ, AC + """) + tables["network_transmission_path_limits"] = csv_str_to_df(""" + path_id, direction, timeslice, capacity + CQ-NQ, forward, qld_peak_demand, 1200 + CQ-NQ, forward, qld_winter_reference, 1400 + CQ-NQ, reverse, qld_winter_reference, 1910 + Q1-NQ, , , + """) + tables["network_expansion_options"] = csv_str_to_df(""" + expansion_id, expansion_type, allowed_expansion, expansion_option + CQ-NQ, forward, 1000, Option 1 + CQ-NQ, reverse, 900, Option 1 + SWQLD1, constraint_relaxation, 400, Option 2 + """) + tables["network_transmission_path_expansion_costs"] = csv_str_to_df(""" + expansion_id, year, cost + CQ-NQ, 2026, 1000000 + SWQLD1, 2026, 100000 + """) + # sample_model_config's reference_year_cycle is [2024]. + tables["timeslices"] = csv_str_to_df(""" + timeslice_id, reference_year, start_month_day, end_month_day + qld_peak_demand, 2024, 01-13, 01-15 + qld_winter_reference, 2024, 04-01, 10-01 + """) + tables["custom_constraints"] = csv_str_to_df(""" + constraint_id, direction + SWQLD1, <= + """) + tables["custom_constraints_lhs"] = csv_str_to_df(""" + constraint_id, term_type, variable_name, coefficient, date_from + SWQLD1, link_flow, CQ-NQ, 0.84, + SWQLD1, generator_output, KINGASF1, 0.14, + """) + tables["custom_constraints_rhs"] = csv_str_to_df(""" + constraint_id, timeslice, rhs, date_from + SWQLD1, qld_peak_demand, 3000, + """) + return tables + + +def test_create_pypsa_friendly_inputs_new_format( + new_format_ispypsa_tables, sample_model_config +): + with patch( + "ispypsa.translator.create_pypsa_friendly.FEATURE_FLAGS", + {"use_new_table_format": True}, + ): + result = create_pypsa_friendly_inputs( + sample_model_config, new_format_ispypsa_tables + ) + expected_outputs = list_translator_output_files() + + assert set(result.keys()) == set(expected_outputs) + + assert set(result["buses"]["name"]) == {"NQ", "CQ", "Q1"} + + # CQ-NQ existing + expansion (2026 cost row only), Q1-NQ existing. + assert sorted(result["links"]["name"]) == [ + "CQ-NQ_existing", + "CQ-NQ_exp_2026", + "Q1-NQ_existing", + ] + + # Two forward rows and one reverse row for CQ-NQ; the collapsed Q1-NQ row + # contributes nothing. + limits = result["link_timeslice_limits"] + assert set(limits.columns) == {"name", "attribute", "timeslice", "value"} + assert len(limits) == 3 + + # Snapshots span the config's investment periods at 30 min resolution and + # carry weighting columns. + snapshots = result["snapshots"] + assert {"investment_periods", "snapshots", "objective"} <= set(snapshots.columns) + assert set(snapshots["investment_periods"]) == {2026, 2028} + assert len(snapshots) > 0 + + mapping = result["timeslice_snapshots"] + assert set(mapping.columns) == {"timeslice_id", "investment_periods", "snapshots"} + # The patterns are re-sequenced into every model year, so both timeslices + # pick up snapshots. + assert set(mapping["timeslice_id"]) == {"qld_peak_demand", "qld_winter_reference"} + + # SWQLD1 per investment period (2026, 2028) plus the two expansion limits. + rhs = result["custom_constraints_rhs"] + assert set(rhs.columns) == { + "constraint_name", + "investment_period", + "timeslice", + "rhs", + "constraint_type", + } + assert len(rhs) == 4 + + lhs = result["custom_constraints_lhs"] + assert set(lhs.columns) == { + "constraint_name", + "investment_period", + "variable_name", + "component", + "attribute", + "coefficient", + } + # SWQLD1: (2 link names + 1 generator) x 2 periods + relax gen term x 2 + # periods + 2 expansion-limit terms. + assert len(lhs) == 10 + + generators = result["custom_constraints_generators"] + assert sorted(generators["name"]) == ["SWQLD1_exp_2026"] + + assert "generators" not in result + assert "batteries" not in result