diff --git a/src/ispypsa/cli/dodo.py b/src/ispypsa/cli/dodo.py index 55a39087..cac01b4a 100644 --- a/src/ispypsa/cli/dodo.py +++ b/src/ispypsa/cli/dodo.py @@ -15,6 +15,7 @@ read_csvs, write_csvs, ) +from ispypsa.feature_flags import FEATURE_FLAGS from ispypsa.iasr_table_caching import build_local_cache, list_cache_files from ispypsa.logging import configure_logging from ispypsa.plotting import ( @@ -483,15 +484,20 @@ def create_pypsa_inputs_for_capacity_expansion_model() -> None: ispypsa_tables = read_csvs(input_tables_dir) pypsa_tables = create_pypsa_friendly_inputs(config, ispypsa_tables) - # Create capacity expansion timeseries - pypsa_tables["snapshots"] = create_pypsa_friendly_timeseries_inputs( - config, - "capacity_expansion", - ispypsa_tables, - pypsa_tables["generators"], - parsed_trace_dir, - capacity_expansion_timeseries_location, - ) + # FEATURE_FLAG_CLEANUP[use_new_table_format]: revisit once generator + # translation lands for the new format. Until then the new-format path + # has no generators, so no timeseries inputs are created and snapshots + # come from create_pypsa_friendly_inputs itself. + if not FEATURE_FLAGS["use_new_table_format"]: + # Create capacity expansion timeseries + pypsa_tables["snapshots"] = create_pypsa_friendly_timeseries_inputs( + config, + "capacity_expansion", + ispypsa_tables, + pypsa_tables["generators"], + parsed_trace_dir, + capacity_expansion_timeseries_location, + ) write_csvs(pypsa_tables, pypsa_friendly_dir) diff --git a/src/ispypsa/pypsa_build/build.py b/src/ispypsa/pypsa_build/build.py index 68b0d0f6..15a3208a 100644 --- a/src/ispypsa/pypsa_build/build.py +++ b/src/ispypsa/pypsa_build/build.py @@ -7,7 +7,10 @@ _add_buses_to_network, ) from ispypsa.pypsa_build.carriers import _add_carriers_to_network -from ispypsa.pypsa_build.custom_constraints import _add_custom_constraints +from ispypsa.pypsa_build.custom_constraints import ( + _add_custom_constraints, + _add_custom_constraints_with_temporal_scope, +) from ispypsa.pypsa_build.generators import ( _add_custom_constraint_generators_to_network, _add_generators_to_network, @@ -70,13 +73,20 @@ def build_pypsa_network( ) if "links" in pypsa_friendly_tables.keys(): - _add_links_to_network(network, pypsa_friendly_tables["links"]) + _add_links_to_network( + network, + pypsa_friendly_tables["links"], + pypsa_friendly_tables.get("link_timeslice_limits"), + pypsa_friendly_tables.get("timeslice_snapshots"), + ) - _add_generators_to_network( - network, - pypsa_friendly_tables["generators"], - path_to_pypsa_friendly_timeseries_data, - ) + # Absent under the new table format until generator translation lands. + if "generators" in pypsa_friendly_tables.keys(): + _add_generators_to_network( + network, + pypsa_friendly_tables["generators"], + path_to_pypsa_friendly_timeseries_data, + ) if "batteries" in pypsa_friendly_tables.keys(): _add_batteries_to_network(network, pypsa_friendly_tables["batteries"]) @@ -91,11 +101,22 @@ def build_pypsa_network( # The underlying linopy model needs to get built so we can add custom constraints. network.optimize.create_model(multi_investment_periods=True) + # FEATURE_FLAG_CLEANUP[use_new_table_format]: the temporal-scope path + # becomes the only path once the old format is deleted; the column check + # distinguishes the formats until then. if "custom_constraints_rhs" in pypsa_friendly_tables: - _add_custom_constraints( - network, - pypsa_friendly_tables["custom_constraints_rhs"], - pypsa_friendly_tables["custom_constraints_lhs"], - ) + if "investment_period" in pypsa_friendly_tables["custom_constraints_rhs"]: + _add_custom_constraints_with_temporal_scope( + network, + pypsa_friendly_tables["custom_constraints_rhs"], + pypsa_friendly_tables["custom_constraints_lhs"], + pypsa_friendly_tables["timeslice_snapshots"], + ) + else: + _add_custom_constraints( + network, + pypsa_friendly_tables["custom_constraints_rhs"], + pypsa_friendly_tables["custom_constraints_lhs"], + ) return network diff --git a/src/ispypsa/pypsa_build/custom_constraints.py b/src/ispypsa/pypsa_build/custom_constraints.py index bb345d26..b40a2880 100644 --- a/src/ispypsa/pypsa_build/custom_constraints.py +++ b/src/ispypsa/pypsa_build/custom_constraints.py @@ -5,6 +5,8 @@ import pandas as pd import pypsa +logger = logging.getLogger(__name__) + def _get_variables( model: linopy.Model, component_name: str, component_type: str, attribute_type: str @@ -115,3 +117,186 @@ def _add_custom_constraints( raise ValueError( f"{row['constraint_type']} is not a valid constraint type." ) + + +def _add_custom_constraints_with_temporal_scope( + network: pypsa.Network, + custom_constraints_rhs: pd.DataFrame, + custom_constraints_lhs: pd.DataFrame, + timeslice_snapshots: pd.DataFrame, +): + """Adds the new-format custom constraints to the `pypsa.Network`. + + Each RHS row creates one linopy constraint scoped to the row's + investment_period and timeslice: time-indexed LHS variables are + restricted to the snapshots inside that scope. NaN investment_period or + timeslice means unrestricted in that dimension (e.g. the expansion-limit + constraints, which only involve p_nom variables, carry NaN in both). + + LHS terms referencing components not in the model (e.g. generators while + generator translation for the new format is pending) are skipped with a + log line; constraint instances with no snapshots in scope or no terms in + the model are skipped silently — the translator logs the timeslices that + never apply. + + Args: + network: The `pypsa.Network` object + custom_constraints_rhs: `pd.DataFrame` with columns constraint_name, + investment_period, timeslice, rhs, and constraint_type. + custom_constraints_lhs: `pd.DataFrame` with columns constraint_name, + investment_period, variable_name, component, attribute, and + coefficient. A NaN investment_period term applies to every + instance of its constraint. + timeslice_snapshots: `pd.DataFrame` mapping timeslice_ids to the + snapshots they are active at (columns timeslice_id, + investment_periods, snapshots). + + Returns: None + """ + timeslice_labels = _timeslice_constraint_labels(timeslice_snapshots) + for row in custom_constraints_rhs.itertuples(): + snapshot_subset = _constraint_snapshot_subset( + row, timeslice_labels, network.snapshots + ) + if snapshot_subset is not None and len(snapshot_subset) == 0: + continue + terms = _select_lhs_terms(custom_constraints_lhs, row) + expression = _build_constraint_expression(network.model, terms, snapshot_subset) + if expression is None: + continue + _add_constraint_for_rhs_row(network.model, expression, row) + + +def _timeslice_constraint_labels( + timeslice_snapshots: pd.DataFrame, +) -> dict[str, list[tuple]]: + """The (investment_period, snapshot) labels each timeslice is active at. + + I/O Example: + timeslice_id=qld_peak_demand, investment_periods=2025, + snapshots=2025-01-13 12:00 + -> {"qld_peak_demand": [(2025, Timestamp("2025-01-13 12:00"))]} + """ + mapping = timeslice_snapshots.copy() + mapping["snapshots"] = pd.to_datetime(mapping["snapshots"]) + return { + timeslice_id: list(zip(rows["investment_periods"], rows["snapshots"])) + for timeslice_id, rows in mapping.groupby("timeslice_id") + } + + +def _constraint_snapshot_subset( + rhs_row, timeslice_labels: dict[str, list[tuple]], snapshots: pd.MultiIndex +) -> list[tuple] | None: + """The snapshot labels an RHS row's constraint is restricted to, or None + when the constraint is unrestricted (NaN timeslice and investment_period). + + I/O Example: + timeslice=qld_peak_demand, investment_period=2025, + timeslice_labels={"qld_peak_demand": [(2025, t1), (2030, t2)]} + -> [(2025, t1)] + + timeslice=NaN, investment_period=2025 -> all 2025 snapshots + timeslice=NaN, investment_period=NaN -> None + """ + if not pd.isna(rhs_row.timeslice): + labels = timeslice_labels.get(rhs_row.timeslice, []) + if not pd.isna(rhs_row.investment_period): + period = int(rhs_row.investment_period) + labels = [label for label in labels if label[0] == period] + return labels + if not pd.isna(rhs_row.investment_period): + period = int(rhs_row.investment_period) + return list(snapshots[snapshots.get_level_values(0) == period]) + return None + + +def _select_lhs_terms(custom_constraints_lhs: pd.DataFrame, rhs_row) -> pd.DataFrame: + """The LHS terms belonging to an RHS row's constraint instance: terms for + the same constraint whose investment_period matches the row's or is NaN + (applies to every instance). + + I/O Example: + lhs: + constraint_name investment_period variable_name + SWQLD1 2025 KINGASF1 + SWQLD1 2030 KINGASF1 + SWQLD1 , SWQLD1_exp_2025 + + rhs_row (SWQLD1, investment_period=2025) selects rows 1 and 3. + """ + terms = custom_constraints_lhs[ + custom_constraints_lhs["constraint_name"] == rhs_row.constraint_name + ] + if pd.isna(rhs_row.investment_period): + return terms + return terms[ + terms["investment_period"].isna() + | (terms["investment_period"] == rhs_row.investment_period) + ] + + +def _build_constraint_expression( + model: linopy.Model, terms: pd.DataFrame, snapshot_subset: list[tuple] | None +): + """Builds the linear expression sum(coefficient * variable) for a + constraint instance, restricting time-indexed variables to the snapshot + subset. Returns None when no term's variable is in the model.""" + expression_terms = [] + for term in terms.itertuples(): + variable = _get_variable_with_snapshot_subset( + model, term.variable_name, term.component, term.attribute, snapshot_subset + ) + if variable is not None: + expression_terms.append((term.coefficient, variable)) + if not expression_terms: + return None + return model.linexpr(*expression_terms) + + +def _get_variable_with_snapshot_subset( + model: linopy.Model, + component_name: str, + component_type: str, + attribute_type: str, + snapshot_subset: list[tuple] | None, +): + """Retrieves a variable like _get_variables, restricting time-indexed + ('p') variables to the snapshot subset. Components not in the model are + skipped with a log line rather than raising — new-format constraints + legitimately reference generators and batteries before those components + are translated.""" + try: + variable = _get_variables(model, component_name, component_type, attribute_type) + except KeyError: + logger.info( + f"{component_type} {component_name} not in model, " + f"custom constraint term skipped." + ) + return None + if variable is not None and attribute_type == "p" and snapshot_subset is not None: + variable = variable.loc[snapshot_subset] + return variable + + +def _add_constraint_for_rhs_row(model: linopy.Model, expression, rhs_row) -> None: + """Adds one constraint, named uniquely for the RHS row's temporal scope. + + I/O Example: + (SWQLD1, 2025, qld_peak_demand) -> "SWQLD1_2025_qld_peak_demand" + (CQ-NQ_expansion_limit, NaN, NaN) -> "CQ-NQ_expansion_limit" + """ + name_parts = [rhs_row.constraint_name] + if not pd.isna(rhs_row.investment_period): + name_parts.append(str(int(rhs_row.investment_period))) + if not pd.isna(rhs_row.timeslice): + name_parts.append(rhs_row.timeslice) + name = "_".join(name_parts) + if rhs_row.constraint_type == "<=": + model.add_constraints(expression <= rhs_row.rhs, name=name) + elif rhs_row.constraint_type == ">=": + model.add_constraints(expression >= rhs_row.rhs, name=name) + elif rhs_row.constraint_type == "==": + model.add_constraints(expression == rhs_row.rhs, name=name) + else: + raise ValueError(f"{rhs_row.constraint_type} is not a valid constraint type.") diff --git a/src/ispypsa/pypsa_build/links.py b/src/ispypsa/pypsa_build/links.py index 36652bde..811a1324 100644 --- a/src/ispypsa/pypsa_build/links.py +++ b/src/ispypsa/pypsa_build/links.py @@ -2,15 +2,92 @@ import pypsa -def _add_links_to_network(network: pypsa.Network, links: pd.DataFrame) -> None: +def _add_links_to_network( + network: pypsa.Network, + links: pd.DataFrame, + link_timeslice_limits: pd.DataFrame | None = None, + timeslice_snapshots: pd.DataFrame | None = None, +) -> None: """Adds the Links defined in a pypsa-friendly input table called `"links"` to the `pypsa.Network` object. + When the new-format per-timeslice limit tables are given, links with + timeslice-varying limits get per-snapshot p_max_pu / p_min_pu series + instead of their static values (see _build_link_pu_overrides). + Args: network: The `pypsa.Network` object links: `pd.DataFrame` with `PyPSA` style `Link` attributes. + link_timeslice_limits: `pd.DataFrame` with per-timeslice per-unit + limits (columns name, attribute, timeslice, value), or None when + all link limits are static. + timeslice_snapshots: `pd.DataFrame` mapping timeslice_ids to the + snapshots they are active at (columns timeslice_id, + investment_periods, snapshots). Required when + link_timeslice_limits is given. Returns: None """ + pu_overrides = _build_link_pu_overrides( + link_timeslice_limits, timeslice_snapshots, links, network.snapshots + ) links["class_name"] = "Link" - links.apply(lambda row: network.add(**row.to_dict()), axis=1) + for _, row in links.iterrows(): + network.add(**(row.to_dict() | pu_overrides.get(row["name"], {}))) + + +def _build_link_pu_overrides( + link_timeslice_limits: pd.DataFrame | None, + timeslice_snapshots: pd.DataFrame | None, + links: pd.DataFrame, + snapshots: pd.MultiIndex, +) -> dict[str, dict[str, pd.Series]]: + """Expands each link's per-timeslice limits into per-snapshot series. + + Each series starts at the link's static value (1.0 for p_max_pu, the + links table's p_min_pu for p_min_pu — both equivalent to the + winter_reference limit) and gets each timeslice's value at the snapshots + that timeslice is active. Timeslices with no snapshots simply leave the + static value in place — the translator has already logged them. + + I/O Example: + link_timeslice_limits: + name attribute timeslice value + CQ-NQ_existing p_max_pu qld_peak_demand 0.857 + + timeslice_snapshots: qld_peak_demand active at (2025, 2025-01-13 12:00) + snapshots: (2025, 2025-01-13 12:00), (2025, 2025-01-15 12:00) + + returns: + {"CQ-NQ_existing": {"p_max_pu": series [0.857, 1.0]}} + """ + if link_timeslice_limits is None or link_timeslice_limits.empty: + return {} + timeslice_labels = _timeslice_snapshot_labels(timeslice_snapshots) + static_p_min_pu = links.set_index("name")["p_min_pu"] + overrides = {} + for (name, attribute), rows in link_timeslice_limits.groupby(["name", "attribute"]): + static_value = 1.0 if attribute == "p_max_pu" else static_p_min_pu[name] + series = pd.Series(static_value, index=snapshots, dtype=float) + for row in rows.itertuples(): + series.loc[timeslice_labels.get(row.timeslice, [])] = row.value + overrides.setdefault(name, {})[attribute] = series + return overrides + + +def _timeslice_snapshot_labels( + timeslice_snapshots: pd.DataFrame, +) -> dict[str, list[tuple]]: + """The (investment_period, snapshot) labels each timeslice is active at. + + I/O Example: + timeslice_id=qld_peak_demand, investment_periods=2025, + snapshots=2025-01-13 12:00 + -> {"qld_peak_demand": [(2025, Timestamp("2025-01-13 12:00"))]} + """ + mapping = timeslice_snapshots.copy() + mapping["snapshots"] = pd.to_datetime(mapping["snapshots"]) + return { + timeslice_id: list(zip(rows["investment_periods"], rows["snapshots"])) + for timeslice_id, rows in mapping.groupby("timeslice_id") + } diff --git a/src/ispypsa/templater/create_template.py b/src/ispypsa/templater/create_template.py index bf26486a..2861c85d 100644 --- a/src/ispypsa/templater/create_template.py +++ b/src/ispypsa/templater/create_template.py @@ -6,6 +6,7 @@ from ispypsa.feature_flags import FEATURE_FLAGS from ispypsa.templater.connection_and_build_costs import _template_connection_costs from ispypsa.templater.custom_constraints_from_plexos import ( + empty_custom_constraint_tables, template_custom_constraints_from_plexos, ) from ispypsa.templater.dynamic_generator_properties import ( @@ -44,6 +45,10 @@ _template_new_generators_static_properties, ) from ispypsa.templater.storage import _template_battery_properties +from ispypsa.templater.timeslices import ( + _template_timeslices, + load_timeslice_calendar, +) from ispypsa.templater.transmission import _template_network_transmission _BASE_TEMPLATE_OUTPUTS = [ @@ -74,8 +79,9 @@ ] # Outputs from the new-format templater branch. Granularity-invariant: the -# same five tables are emitted for sub_regions, nem_regions, and single_region -# (only their contents differ). +# same tables are emitted for sub_regions, nem_regions, and single_region +# (only their contents differ — the custom-constraint tables are header-only +# at coarser granularities). # FEATURE_FLAG_CLEANUP[use_new_table_format]: rename to _TEMPLATE_OUTPUTS and # delete _BASE_TEMPLATE_OUTPUTS above. _NEW_FORMAT_TEMPLATE_OUTPUTS = [ @@ -84,15 +90,8 @@ "network_transmission_path_limits", "network_expansion_options", "network_transmission_path_expansion_costs", -] - -# Custom constraints are templated only at sub_regions granularity (see the gate -# in create_ispypsa_inputs_template) — coarser granularities collapse the -# sub-region nodes, sub-regional flow paths and REZ-located units the constraints -# reference, leaving nothing to constrain. Listed as outputs only at that -# granularity so the create_ispypsa_inputs task tracks them where they are -# written and does not expect them where they never are. -_CUSTOM_CONSTRAINT_OUTPUTS = [ + "timeslices", + "costs_connection", "custom_constraints", "custom_constraints_lhs", "custom_constraints_rhs", @@ -219,6 +218,10 @@ def create_ispypsa_inputs_template( ) template["network_expansion_options"] = expansion_options template["network_transmission_path_expansion_costs"] = expansion_costs + template["timeslices"] = _template_timeslices( + load_timeslice_calendar(iasr_workbook_version), + manually_extracted_tables["reference_year_sequence"], + ) # todo: replace with actual generators_new_entrant once that templating # function is written — passing empty placeholder for now so costs_connection @@ -246,14 +249,17 @@ def create_ispypsa_inputs_template( # their LHS references sub-region nodes, sub-regional flow paths, and # REZ-located units that only exist as distinct entities at sub_regions # granularity. Once sub-regions are collapsed (nem_regions / - # single_region) they have no meaningful representation, so only emit - # them for sub_regions. + # single_region) they have no meaningful representation, so the tables + # are emitted header-only ("all columns, no rows") there, keeping the + # output set granularity-invariant. if regional_granularity == "sub_regions": template.update( template_custom_constraints_from_plexos( iasr_tables, iasr_workbook_version=iasr_workbook_version ) ) + else: + template.update(empty_custom_constraint_tables()) return template template = {} @@ -347,8 +353,6 @@ def list_templater_output_files(regional_granularity, output_path=None): # granularity-specific file removals. if FEATURE_FLAGS["use_new_table_format"]: files = _NEW_FORMAT_TEMPLATE_OUTPUTS.copy() - if regional_granularity == "sub_regions": - files += _CUSTOM_CONSTRAINT_OUTPUTS else: files = _BASE_TEMPLATE_OUTPUTS.copy() if regional_granularity in ["sub_regions", "single_region"]: diff --git a/src/ispypsa/templater/custom_constraints_from_plexos.py b/src/ispypsa/templater/custom_constraints_from_plexos.py index c2a4672e..80a84e54 100644 --- a/src/ispypsa/templater/custom_constraints_from_plexos.py +++ b/src/ispypsa/templater/custom_constraints_from_plexos.py @@ -234,6 +234,34 @@ _EXCLUDED_PARENT_CLASSES = ("Purchaser",) _AREA_SUFFIX_PATTERN = re.compile(r" Area\d+$") +# Output columns of the three custom-constraint template tables. +_CUSTOM_CONSTRAINTS_COLUMNS = ["constraint_id", "direction"] +_CUSTOM_CONSTRAINTS_LHS_COLUMNS = [ + "constraint_id", + "term_type", + "variable_name", + "coefficient", + "date_from", +] +_CUSTOM_CONSTRAINTS_RHS_COLUMNS = ["constraint_id", "timeslice", "rhs", "date_from"] + + +def empty_custom_constraint_tables() -> dict[str, pd.DataFrame]: + """Header-only versions of the three custom-constraint template tables. + + Emitted at nem_regions / single_region granularity, where the sub-region + nodes, sub-regional flow paths, and REZ-located units the constraints + reference are collapsed, leaving nothing to constrain. Emitting "all + columns, no rows" tables keeps the templater's output set + granularity-invariant, so downstream consumers never need to check for + missing tables. + """ + return { + "custom_constraints": pd.DataFrame(columns=_CUSTOM_CONSTRAINTS_COLUMNS), + "custom_constraints_lhs": pd.DataFrame(columns=_CUSTOM_CONSTRAINTS_LHS_COLUMNS), + "custom_constraints_rhs": pd.DataFrame(columns=_CUSTOM_CONSTRAINTS_RHS_COLUMNS), + } + def template_custom_constraints_from_plexos( iasr_tables: dict[str, pd.DataFrame], diff --git a/src/ispypsa/translator/constraints.py b/src/ispypsa/translator/constraints.py new file mode 100644 index 00000000..1c6cff04 --- /dev/null +++ b/src/ispypsa/translator/constraints.py @@ -0,0 +1,587 @@ +"""Translates the new-format custom-constraint tables into PyPSA friendly form. + +Consumes ``custom_constraints`` (constraint senses), ``custom_constraints_lhs`` +(terms) and ``custom_constraints_rhs`` (limit values), plus the unified +network expansion tables for constraint-relaxation generators and +expansion-limit constraints. + +The PyPSA friendly outputs carry two temporal columns the old format lacked: + +- ``investment_period``: time-varying inputs (``date_from``) are resolved by + holding, for each investment period, the value active at the period's + start. Rows with a NaN investment_period apply in every period. +- ``timeslice`` (RHS only): the constraint binds only at snapshots inside the + timeslice's windows (see ispypsa.translator.timeslices). NaN means the + limit applies at all snapshots. +""" + +import logging + +import numpy as np +import pandas as pd + +from ispypsa.config import ModelConfig +from ispypsa.translator.mappings import ( + _CUSTOM_CONSTRAINT_TERM_TYPE_TO_ATTRIBUTE_TYPE, + _CUSTOM_CONSTRAINT_TERM_TYPE_TO_COMPONENT_TYPE, +) +from ispypsa.translator.network import ( + _pivot_physical_expansion_options, + _prepare_expansion_costs, +) + +logger = logging.getLogger(__name__) + +_LHS_COLUMNS = [ + "constraint_name", + "investment_period", + "variable_name", + "component", + "attribute", + "coefficient", +] + +_RHS_COLUMNS = [ + "constraint_name", + "investment_period", + "timeslice", + "rhs", + "constraint_type", +] + +_GENERATOR_COLUMNS = [ + "name", + "isp_name", + "bus", + "p_nom", + "p_nom_extendable", + "build_year", + "lifetime", + "capital_cost", +] + +_DIRECTION_TO_CONSTRAINT_TYPE = {"<=": "<=", ">=": ">=", "=": "=="} + +# Working column orders before constraint_id is renamed to constraint_name. +_INTERNAL_LHS_COLUMNS = ["constraint_id"] + [ + c for c in _LHS_COLUMNS if c != "constraint_name" +] +_INTERNAL_RHS_COLUMNS = ["constraint_id"] + [ + c for c in _RHS_COLUMNS if c != "constraint_name" +] + + +def _raise_on_duplicate_input_rows( + table: pd.DataFrame, keys: list[str], label: str +) -> None: + """Raise on input rows sharing the same key — duplicates would otherwise + silently collapse to one arbitrary row during date_from resolution.""" + duplicates = table[table.duplicated(subset=keys, keep=False)] + if not duplicates.empty: + raise ValueError( + f"Duplicate custom constraint {label} rows for: " + f"{sorted(set(duplicates['constraint_id']))}" + ) + + +def _concat_non_empty(frames: list[pd.DataFrame], columns: list[str]) -> pd.DataFrame: + """Concatenates the non-empty frames (concatenating all-empty frames is + deprecated by pandas), returning a header-only frame when all are empty.""" + non_empty = [frame for frame in frames if not frame.empty] + if not non_empty: + return pd.DataFrame(columns=columns) + return pd.concat(non_empty, ignore_index=True) + + +def _translate_custom_constraints_from_network_tables( + ispypsa_tables: dict[str, pd.DataFrame], + links: pd.DataFrame, + config: ModelConfig, + generators: pd.DataFrame | None = None, + batteries: pd.DataFrame | None = None, +) -> dict[str, pd.DataFrame]: + """Translates the custom-constraint tables and builds the endogenous + expansion-limit constraints. + + Generator and battery terms pass through with their IASR IDs as + variable_names: pypsa_build skips (and logs) terms whose components are + not in the model. The ``generators`` and ``batteries`` arguments are + accepted so the templated generator/battery tables can be used to map + those IDs to model component names once generator templating lands; they + are currently unused. + + Args: + ispypsa_tables: dictionary of new-format `ISPyPSA` input tables; + consumes custom_constraints, custom_constraints_lhs, + custom_constraints_rhs, network_expansion_options and + network_transmission_path_expansion_costs. + links: PyPSA friendly links table from + ispypsa.translator.network (existing plus expansion links). + config: `ispypsa.config.ModelConfig` object. + generators: templated generator table (future name-mapping hook). + batteries: templated battery table (future name-mapping hook). + + Returns: dict with the PyPSA friendly custom_constraints_lhs, + custom_constraints_rhs and custom_constraints_generators tables. + """ + _raise_on_duplicate_input_rows( + ispypsa_tables["custom_constraints_rhs"], + ["constraint_id", "timeslice", "date_from"], + "RHS", + ) + _raise_on_duplicate_input_rows( + ispypsa_tables["custom_constraints_lhs"], + ["constraint_id", "term_type", "variable_name", "date_from"], + "LHS", + ) + period_starts = _investment_period_start_dates( + config.temporal.capacity_expansion.investment_periods, + config.temporal.year_type, + ) + rhs = _resolve_values_active_at_period_starts( + ispypsa_tables["custom_constraints_rhs"], + ["constraint_id", "timeslice"], + period_starts, + ) + rhs = _add_constraint_type(rhs, ispypsa_tables["custom_constraints"]) + lhs = _resolve_values_active_at_period_starts( + ispypsa_tables["custom_constraints_lhs"], + ["constraint_id", "term_type", "variable_name"], + period_starts, + ) + lhs = _add_component_and_attribute(lhs) + lhs = _expand_link_flow_terms(lhs, links) + rhs = _drop_rhs_without_lhs_terms(rhs, lhs) + + relaxation_generators = _create_constraint_relaxation_generators( + ispypsa_tables, set(rhs["constraint_id"]), config + ) + relaxation_generator_lhs = _relaxation_generator_lhs_terms( + relaxation_generators, + config.temporal.capacity_expansion.investment_periods, + ) + expansion_limit_lhs, expansion_limit_rhs = _create_expansion_limit_constraints( + links, relaxation_generators, ispypsa_tables["network_expansion_options"] + ) + + lhs = _concat_non_empty( + [lhs, relaxation_generator_lhs, expansion_limit_lhs], _INTERNAL_LHS_COLUMNS + ) + rhs = _concat_non_empty([rhs, expansion_limit_rhs], _INTERNAL_RHS_COLUMNS) + lhs, rhs = _finalise_lhs_and_rhs(lhs, rhs) + return { + "custom_constraints_lhs": lhs, + "custom_constraints_rhs": rhs, + "custom_constraints_generators": _finalise_generators(relaxation_generators), + } + + +def _investment_period_start_dates( + investment_periods: list[int], year_type: str +) -> dict[int, pd.Timestamp]: + """The datetime each investment period starts at. + + I/O Example: + [2030], "fy" -> {2030: 2029-07-01} # FY ending nomenclature + [2030], "calendar" -> {2030: 2030-01-01} + """ + if year_type == "fy": + return {p: pd.Timestamp(year=p - 1, month=7, day=1) for p in investment_periods} + return {p: pd.Timestamp(year=p, month=1, day=1) for p in investment_periods} + + +def _resolve_values_active_at_period_starts( + table: pd.DataFrame, + group_columns: list[str], + period_starts: dict[int, pd.Timestamp], +) -> pd.DataFrame: + """Resolves date_from-varying rows into one row per investment period. + + For each period, each group keeps the row active at the period's start: + the latest date_from on or before it, with no-date_from rows acting as + the baseline. A group whose earliest date_from is after a period's start + contributes no row for that period. + + I/O Example: + table (group_columns=["constraint_id", "timeslice"]): + constraint_id timeslice rhs date_from + SWQLD1 qld_peak_demand 3000 + SWQLD1 qld_peak_demand 2500 2032-12-01T00:00:00 + + period_starts={2030: 2029-07-01, 2035: 2034-07-01} returns: + constraint_id timeslice rhs investment_period + SWQLD1 qld_peak_demand 3000 2030 + SWQLD1 qld_peak_demand 2500 2035 # 2032 value held from period start + """ + table = table.copy() + table["date_from"] = pd.to_datetime(table["date_from"]).fillna(pd.Timestamp.min) + resolved = [] + for period, start in period_starts.items(): + active = table[table["date_from"] <= start] + active = active.sort_values("date_from") + active = active.groupby(group_columns, dropna=False).tail(1).copy() + active["investment_period"] = period + resolved.append(active) + return pd.concat(resolved, ignore_index=True).drop(columns="date_from") + + +def _add_constraint_type( + rhs: pd.DataFrame, custom_constraints: pd.DataFrame +) -> pd.DataFrame: + """Adds each constraint's sense as constraint_type ("=" becomes "==", + matching the vocabulary pypsa_build applies constraints with). + + I/O Example: + rhs: constraint_id=SWQLD1, rhs=3000 + custom_constraints: constraint_id=SWQLD1, direction="<=" + -> constraint_id=SWQLD1, rhs=3000, constraint_type="<=" + """ + rhs = rhs.merge(custom_constraints, on="constraint_id", how="left") + rhs["constraint_type"] = rhs["direction"].map(_DIRECTION_TO_CONSTRAINT_TYPE) + _raise_on_missing_constraint_type(rhs) + return rhs.drop(columns="direction") + + +def _raise_on_missing_constraint_type(rhs: pd.DataFrame) -> None: + """Raise if any RHS row's constraint has no (or an unmapped) sense — the + constraint can't be applied without one.""" + missing = rhs.loc[rhs["constraint_type"].isna(), "constraint_id"] + if not missing.empty: + raise ValueError( + f"Custom constraints with RHS values but no direction in the " + f"custom_constraints table: {sorted(set(missing))}" + ) + + +def _add_component_and_attribute(lhs: pd.DataFrame) -> pd.DataFrame: + """Maps each term_type to the PyPSA component and attribute its variable + belongs to. + + I/O Example: + term_type=generator_output -> component=Generator, attribute=p + term_type=link_flow -> component=Link, attribute=p + term_type=storage_output -> component=Storage, attribute=p + """ + lhs = lhs.copy() + lhs["component"] = lhs["term_type"].map( + _CUSTOM_CONSTRAINT_TERM_TYPE_TO_COMPONENT_TYPE + ) + lhs["attribute"] = lhs["term_type"].map( + _CUSTOM_CONSTRAINT_TERM_TYPE_TO_ATTRIBUTE_TYPE + ) + _raise_on_unmapped_term_types(lhs) + return lhs.drop(columns="term_type") + + +def _raise_on_unmapped_term_types(lhs: pd.DataFrame) -> None: + """Raise if any term_type has no component mapping — silently dropping a + term would weaken its constraint.""" + unmapped = lhs.loc[lhs["component"].isna(), "term_type"] + if not unmapped.empty: + raise ValueError( + f"Custom constraint LHS term_types with no component mapping: " + f"{sorted(set(unmapped))}" + ) + + +def _expand_link_flow_terms(lhs: pd.DataFrame, links: pd.DataFrame) -> pd.DataFrame: + """Replaces each link term's path_id with the model's link names — the + existing link plus each expansion link — one term per link. Terms for + paths not in the model are dropped and logged. + + I/O Example: + lhs: + constraint_id variable_name component coefficient + SWQLD1 NSW-QLD Link 0.84 + SWQLD1 KINGASF1 Generator 0.14 + + links (isp_name -> name): NSW-QLD -> NSW-QLD_existing, NSW-QLD_exp_2030 + + returns: + constraint_id variable_name component coefficient + SWQLD1 KINGASF1 Generator 0.14 + SWQLD1 NSW-QLD_existing Link 0.84 + SWQLD1 NSW-QLD_exp_2030 Link 0.84 + """ + link_terms = lhs[lhs["component"] == "Link"] + other_terms = lhs[lhs["component"] != "Link"] + _log_link_terms_not_in_model(link_terms, links) + expanded = link_terms.merge( + links.loc[:, ["isp_name", "name"]], left_on="variable_name", right_on="isp_name" + ) + expanded = expanded.drop(columns=["variable_name", "isp_name"]) + expanded = expanded.rename(columns={"name": "variable_name"}) + return pd.concat([other_terms, expanded], ignore_index=True) + + +def _log_link_terms_not_in_model(link_terms: pd.DataFrame, links: pd.DataFrame) -> None: + missing = set(link_terms["variable_name"]) - set(links["isp_name"]) + if missing: + logger.info( + f"Custom constraint link_flow terms dropped (paths not in model): " + f"{sorted(missing)}" + ) + + +def _drop_rhs_without_lhs_terms(rhs: pd.DataFrame, lhs: pd.DataFrame) -> pd.DataFrame: + """Drops (and logs) RHS rows for constraints left with no LHS terms — a + constraint with an empty LHS can't be applied.""" + without_lhs = set(rhs["constraint_id"]) - set(lhs["constraint_id"]) + if without_lhs: + logger.info( + f"Custom constraints dropped (no LHS terms in model): {sorted(without_lhs)}" + ) + return rhs[~rhs["constraint_id"].isin(without_lhs)] + + +def _create_constraint_relaxation_generators( + ispypsa_tables: dict[str, pd.DataFrame], + constraint_ids: set[str], + config: ModelConfig, +) -> pd.DataFrame: + """Builds one extendable dummy generator per relaxable constraint and + investment period, with the selected expansion option's annualised cost. + + The generators' p_nom enters the parent constraint's LHS with coefficient + -1.0 (see _relaxation_generator_lhs_terms), so building them relaxes the + constraint at the option's cost; total relaxation is capped at the + option's allowed_expansion by the expansion-limit constraints. + + I/O Example: + network_expansion_options: + expansion_id expansion_type allowed_expansion + SWQLD1 constraint_relaxation 500 + + network_transmission_path_expansion_costs: + expansion_id year cost + SWQLD1 2030 100000 + + constraint_ids={"SWQLD1"}, investment_periods=[2030] returns (abridged): + name isp_name bus p_nom build_year + SWQLD1_exp_2030 SWQLD1 bus_for_custom_constraint_gens 0.0 2030 + """ + if not config.network.rez_transmission_expansion: + return pd.DataFrame(columns=_GENERATOR_COLUMNS + ["allowed_expansion"]) + options = ispypsa_tables["network_expansion_options"] + relaxations = options[options["expansion_type"] == "constraint_relaxation"] + relaxations = _drop_relaxations_without_constraints(relaxations, constraint_ids) + costs = _prepare_expansion_costs( + ispypsa_tables["network_transmission_path_expansion_costs"], + config.temporal.capacity_expansion.investment_periods, + config.temporal.year_type, + config.wacc, + config.network.annuitisation_lifetime, + ) + generators = costs.merge( + relaxations.loc[:, ["expansion_id", "allowed_expansion"]], on="expansion_id" + ) + return _format_relaxation_generators(generators) + + +def _drop_relaxations_without_constraints( + relaxations: pd.DataFrame, constraint_ids: set[str] +) -> pd.DataFrame: + """Drops (and logs) relaxation options whose constraint is not in the + model — there is nothing for them to relax.""" + missing = set(relaxations["expansion_id"]) - constraint_ids + if missing: + logger.info( + f"Constraint relaxation expansion options dropped (their constraints " + f"are not in the model): {sorted(missing)}" + ) + return relaxations[~relaxations["expansion_id"].isin(missing)] + + +def _format_relaxation_generators(generators: pd.DataFrame) -> pd.DataFrame: + """Adds the PyPSA generator attributes shared by all relaxation generators. + + I/O Example: + expansion_id=SWQLD1, year=2030, capital_cost=8140, allowed_expansion=500 + -> name=SWQLD1_exp_2030, isp_name=SWQLD1, p_nom=0.0, + p_nom_extendable=True, build_year=2030, lifetime=inf + """ + generators = generators.rename(columns={"expansion_id": "isp_name"}) + generators["name"] = ( + generators["isp_name"] + "_exp_" + generators["year"].astype(str) + ) + generators["bus"] = "bus_for_custom_constraint_gens" + generators["p_nom"] = 0.0 + generators["p_nom_extendable"] = True + generators["build_year"] = generators["year"] + generators["lifetime"] = np.inf + return generators.loc[:, _GENERATOR_COLUMNS + ["allowed_expansion"]] + + +def _relaxation_generator_lhs_terms( + relaxation_generators: pd.DataFrame, investment_periods: list[int] +) -> pd.DataFrame: + """LHS terms subtracting each relaxation generator's capacity from its + parent constraint. + + Terms are per investment period and only include generators already built + by that period — capacity built in a later period can't relax an earlier + period's constraint. + + I/O Example: + relaxation_generators: + name isp_name build_year + SWQLD1_exp_2030 SWQLD1 2030 + SWQLD1_exp_2040 SWQLD1 2040 + + investment_periods=[2030, 2040] returns: + constraint_id investment_period variable_name component attribute coefficient + SWQLD1 2030 SWQLD1_exp_2030 Generator p_nom -1.0 + SWQLD1 2040 SWQLD1_exp_2030 Generator p_nom -1.0 + SWQLD1 2040 SWQLD1_exp_2040 Generator p_nom -1.0 + """ + terms = [] + for period in investment_periods: + built = relaxation_generators[ + relaxation_generators["build_year"] <= period + ].copy() + built["investment_period"] = period + terms.append(built) + terms = pd.concat(terms, ignore_index=True) + terms = terms.rename(columns={"isp_name": "constraint_id", "name": "variable_name"}) + terms["component"] = "Generator" + terms["attribute"] = "p_nom" + terms["coefficient"] = -1.0 + columns = [c for c in _LHS_COLUMNS if c != "constraint_name"] + ["constraint_id"] + return terms.loc[:, columns] + + +def _create_expansion_limit_constraints( + links: pd.DataFrame, + relaxation_generators: pd.DataFrame, + expansion_options: pd.DataFrame, +) -> tuple[pd.DataFrame, pd.DataFrame]: + """Caps the total capacity built across each expandable element's + per-period components at the selected option's capacity. + + For physical paths the cap is the option's forward capacity (reverse + capacity scales with it through each expansion link's p_min_pu); for + constraint relaxations it is the option's allowed_expansion. The + constraints have no investment_period or timeslice — they apply to the + p_nom variables globally. Names get an "_expansion_limit" suffix so a + relaxation cap doesn't collide with the constraint it relaxes. + + I/O Example: + links (extendable): CQ-NQ_exp_2030, CQ-NQ_exp_2040 (isp_name CQ-NQ) + expansion_options: CQ-NQ forward 1000 / reverse 900 + + returns lhs: + constraint_id variable_name component attribute coefficient investment_period + CQ-NQ_expansion_limit CQ-NQ_exp_2030 Link p_nom 1.0 NaN + CQ-NQ_expansion_limit CQ-NQ_exp_2040 Link p_nom 1.0 NaN + + and rhs: + constraint_id rhs constraint_type investment_period timeslice + CQ-NQ_expansion_limit 1000 <= NaN NaN + """ + lhs = pd.concat( + [ + _expansion_limit_lhs(links[links["p_nom_extendable"]], "Link"), + _expansion_limit_lhs(relaxation_generators, "Generator"), + ], + ignore_index=True, + ) + rhs = _expansion_limit_rhs(expansion_options, set(lhs["constraint_id"])) + lhs["constraint_id"] = lhs["constraint_id"] + "_expansion_limit" + rhs["constraint_id"] = rhs["constraint_id"] + "_expansion_limit" + return lhs, rhs + + +def _expansion_limit_lhs(components: pd.DataFrame, component_type: str) -> pd.DataFrame: + """One LHS term per expandable component, summing p_nom across the + investment periods of its parent element. + + I/O Example: + components: name=CQ-NQ_exp_2030, isp_name=CQ-NQ; component_type="Link" + -> constraint_id=CQ-NQ, variable_name=CQ-NQ_exp_2030, component=Link, + attribute=p_nom, coefficient=1.0, investment_period=NaN + """ + lhs = components.loc[:, ["isp_name", "name"]].copy() + lhs = lhs.rename(columns={"isp_name": "constraint_id", "name": "variable_name"}) + lhs["component"] = component_type + lhs["attribute"] = "p_nom" + lhs["coefficient"] = 1.0 + lhs["investment_period"] = np.nan + return lhs + + +def _expansion_limit_rhs( + expansion_options: pd.DataFrame, expandable_ids: set[str] +) -> pd.DataFrame: + """One RHS row per expandable element with components in the model: the + selected option's forward capacity (physical paths) or allowed_expansion + (constraint relaxations). + + I/O Example: + expansion_options: + expansion_id expansion_type allowed_expansion + CQ-NQ forward 1000 + CQ-NQ reverse 900 + SWQLD1 constraint_relaxation 500 + + expandable_ids={"CQ-NQ", "SWQLD1"} returns: + constraint_id rhs constraint_type investment_period timeslice + CQ-NQ 1000 <= NaN NaN + SWQLD1 500 <= NaN NaN + """ + caps = expansion_options[ + expansion_options["expansion_type"].isin(["forward", "constraint_relaxation"]) + ] + caps = caps[caps["expansion_id"].isin(expandable_ids)] + rhs = caps.rename(columns={"expansion_id": "constraint_id"}).copy() + rhs["rhs"] = rhs["allowed_expansion"] + rhs["constraint_type"] = "<=" + rhs["investment_period"] = np.nan + rhs["timeslice"] = np.nan + columns = [c for c in _RHS_COLUMNS if c != "constraint_name"] + ["constraint_id"] + return rhs.loc[:, columns] + + +def _finalise_lhs_and_rhs( + lhs: pd.DataFrame, rhs: pd.DataFrame +) -> tuple[pd.DataFrame, pd.DataFrame]: + """Suffixes expansion-limit names, validates the pairing, and sets the + final PyPSA friendly column orders.""" + lhs = lhs.rename(columns={"constraint_id": "constraint_name"}) + rhs = rhs.rename(columns={"constraint_id": "constraint_name"}) + _raise_on_duplicate_rhs_rows(rhs) + _raise_on_unpaired_constraints(lhs, rhs) + return ( + lhs.loc[:, _LHS_COLUMNS].reset_index(drop=True), + rhs.loc[:, _RHS_COLUMNS].reset_index(drop=True), + ) + + +def _raise_on_duplicate_rhs_rows(rhs: pd.DataFrame) -> None: + """Raise on duplicate (constraint, period, timeslice) RHS rows — pypsa_build + would create two constraints with the same name.""" + keys = ["constraint_name", "investment_period", "timeslice"] + duplicates = rhs[rhs.duplicated(subset=keys, keep=False)] + if not duplicates.empty: + raise ValueError( + f"Duplicate custom constraint RHS rows for: " + f"{sorted(set(duplicates['constraint_name']))}" + ) + + +def _raise_on_unpaired_constraints(lhs: pd.DataFrame, rhs: pd.DataFrame) -> None: + """Raise if any constraint appears on only one side — a one-sided + constraint can't be applied.""" + lhs_names = set(lhs["constraint_name"]) + rhs_names = set(rhs["constraint_name"]) + if lhs_names != rhs_names: + raise ValueError( + f"Custom constraints with LHS terms but no RHS: " + f"{sorted(lhs_names - rhs_names)}; with RHS but no LHS terms: " + f"{sorted(rhs_names - lhs_names)}" + ) + + +def _finalise_generators(relaxation_generators: pd.DataFrame) -> pd.DataFrame: + """Drops the allowed_expansion working column carried for the + expansion-limit RHS, leaving the PyPSA generator columns.""" + return relaxation_generators.loc[:, _GENERATOR_COLUMNS].reset_index(drop=True) diff --git a/src/ispypsa/translator/create_pypsa_friendly.py b/src/ispypsa/translator/create_pypsa_friendly.py index 285fe7b1..b3037eb2 100644 --- a/src/ispypsa/translator/create_pypsa_friendly.py +++ b/src/ispypsa/translator/create_pypsa_friendly.py @@ -8,6 +8,7 @@ from ispypsa.config import ( ModelConfig, ) +from ispypsa.feature_flags import FEATURE_FLAGS from ispypsa.translator.buses import ( _create_single_region_bus, _translate_isp_sub_regions_to_buses, @@ -15,6 +16,9 @@ _translate_rezs_to_buses, create_pypsa_friendly_bus_demand_timeseries, ) +from ispypsa.translator.constraints import ( + _translate_custom_constraints_from_network_tables, +) from ispypsa.translator.custom_constraints import ( _append_if_not_empty, _translate_custom_constraints, @@ -29,6 +33,10 @@ ) from ispypsa.translator.helpers import convert_to_numeric_if_possible from ispypsa.translator.links import _translate_flow_paths_to_links +from ispypsa.translator.network import ( + _translate_network_geography_to_buses, + _translate_network_to_links, +) from ispypsa.translator.renewable_energy_zones import ( _translate_renewable_energy_zone_build_limits_to_links, ) @@ -43,6 +51,10 @@ ) from ispypsa.translator.temporal_filters import _time_series_filter from ispypsa.translator.time_series_checker import _check_time_series +from ispypsa.translator.timeslices import ( + _create_timeslice_snapshot_mapping, + _log_referenced_timeslices_without_snapshots, +) _BASE_TRANSLATOR_OUTPUTS = [ "snapshots", @@ -56,6 +68,26 @@ "custom_constraints_generators", ] +# Outputs of the new-format translator path. Generators and batteries are +# absent because the new-format templater doesn't produce generator/battery +# tables yet; link_timeslice_limits and timeslice_snapshots carry the +# time-varying transmission limits and the snapshot-restriction of custom +# constraints. +# FEATURE_FLAG_CLEANUP[use_new_table_format]: rename to _TRANSLATOR_OUTPUTS +# and delete _BASE_TRANSLATOR_OUTPUTS above (reinstating generators and +# batteries once generator translation lands for the new format). +_NEW_FORMAT_TRANSLATOR_OUTPUTS = [ + "snapshots", + "investment_period_weights", + "buses", + "links", + "link_timeslice_limits", + "timeslice_snapshots", + "custom_constraints_lhs", + "custom_constraints_rhs", + "custom_constraints_generators", +] + def create_pypsa_friendly_inputs( config: ModelConfig, ispypsa_tables: dict[str, pd.DataFrame] @@ -90,6 +122,11 @@ def create_pypsa_friendly_inputs( Returns: dictionary of dataframes in the `PyPSA` friendly format. (add link to pypsa friendly format table docs) """ + # FEATURE_FLAG_CLEANUP[use_new_table_format]: inline the new-format + # function here and delete the legacy body below. + if FEATURE_FLAGS["use_new_table_format"]: + return _create_pypsa_friendly_inputs_from_network_tables(config, ispypsa_tables) + pypsa_inputs = {} pypsa_inputs["investment_period_weights"] = _create_investment_period_weightings( @@ -207,6 +244,57 @@ def create_pypsa_friendly_inputs( return pypsa_inputs +def _create_pypsa_friendly_inputs_from_network_tables( + config: ModelConfig, ispypsa_tables: dict[str, pd.DataFrame] +) -> dict[str, pd.DataFrame]: + """Creates the PyPSA friendly tables from the new-format network tables. + + Covers the network and custom-constraint functionality only: the + new-format templater doesn't produce generator/battery tables yet, so no + generators or batteries (and none of their timeseries) are created. + Snapshots are created here directly from the temporal config — snapshot + aggregation methods that need demand/generator traces (e.g. + named_representative_weeks) aren't available until generator translation + lands. + + Args: + config: `ISPyPSA` `ispypsa.config.ModelConfig` object. + ispypsa_tables: dictionary of dataframes providing the new-format + `ISPyPSA` input tables. + + Returns: dictionary of dataframes in the `PyPSA` friendly format, with + the tables listed in _NEW_FORMAT_TRANSLATOR_OUTPUTS. + """ + pypsa_inputs = {} + snapshots = create_pypsa_friendly_snapshots(config, "capacity_expansion") + pypsa_inputs["snapshots"] = _add_snapshot_weightings( + snapshots, config.temporal.capacity_expansion.resolution_min + ) + pypsa_inputs["investment_period_weights"] = _create_investment_period_weightings( + config.temporal.capacity_expansion.investment_periods, + config.temporal.range.end_year, + config.discount_rate, + ) + pypsa_inputs["buses"] = _translate_network_geography_to_buses( + ispypsa_tables["network_geography"], config.network.nodes.rezs + ) + links, link_timeslice_limits = _translate_network_to_links(ispypsa_tables, config) + pypsa_inputs["links"] = links + pypsa_inputs["link_timeslice_limits"] = link_timeslice_limits + pypsa_inputs["timeslice_snapshots"] = _create_timeslice_snapshot_mapping( + ispypsa_tables["timeslices"], snapshots, config + ) + pypsa_inputs.update( + _translate_custom_constraints_from_network_tables(ispypsa_tables, links, config) + ) + _log_referenced_timeslices_without_snapshots( + pypsa_inputs["timeslice_snapshots"], + pypsa_inputs["link_timeslice_limits"], + pypsa_inputs["custom_constraints_rhs"], + ) + return pypsa_inputs + + def create_pypsa_friendly_timeseries_inputs( config: ModelConfig, model_phase: Literal["capacity_expansion", "operational"], @@ -392,6 +480,13 @@ def list_timeseries_files( KeyError: If required ISPyPSA tables are missing ValueError: If required columns are missing from tables """ + # FEATURE_FLAG_CLEANUP[use_new_table_format]: revisit once generator + # translation lands for the new format. Until then the new-format path + # produces no timeseries files — there are no generators to build traces + # for, and demand traces need the generator-era timeseries machinery. + if FEATURE_FLAGS["use_new_table_format"]: + return [] + files = [] # Validate required tables exist @@ -516,7 +611,11 @@ def _filter_and_save_timeseries( def list_translator_output_files(output_path: Path | None = None) -> list[Path]: - files = _BASE_TRANSLATOR_OUTPUTS + # FEATURE_FLAG_CLEANUP[use_new_table_format]: drop the legacy list. + if FEATURE_FLAGS["use_new_table_format"]: + files = _NEW_FORMAT_TRANSLATOR_OUTPUTS + else: + files = _BASE_TRANSLATOR_OUTPUTS if output_path is not None: files = [output_path / Path(file + ".csv") for file in files] return files diff --git a/src/ispypsa/translator/mappings.py b/src/ispypsa/translator/mappings.py index ed6e25b2..615203b5 100644 --- a/src/ispypsa/translator/mappings.py +++ b/src/ispypsa/translator/mappings.py @@ -152,6 +152,10 @@ "generator_capacity": "Generator", "generator_output": "Generator", "load_consumption": "Load", + # "load" is the new-format vocabulary for "load_consumption" (PLEXOS Node + # Load Coefficient terms). Load variables aren't implemented in + # pypsa_build, which skips these terms with a log line. + "load": "Load", "storage_output": "Storage", } @@ -160,6 +164,7 @@ "generator_capacity": "p_nom", "generator_output": "p", "load_consumption": "p", + "load": "p", "storage_output": "p", } diff --git a/src/ispypsa/translator/network.py b/src/ispypsa/translator/network.py new file mode 100644 index 00000000..e3d6d543 --- /dev/null +++ b/src/ispypsa/translator/network.py @@ -0,0 +1,424 @@ +"""Translates the new-format network tables into PyPSA friendly buses and links. + +Consumes ``network_geography``, ``network_transmission_paths``, +``network_transmission_path_limits``, ``network_expansion_options`` and +``network_transmission_path_expansion_costs``, handling flow paths and REZ +connections through one unified pipeline (both are just paths). +""" + +import numpy as np +import pandas as pd + +from ispypsa.config import ModelConfig +from ispypsa.translator.helpers import _annuitised_investment_costs + +_LINK_COLUMNS = [ + "isp_name", + "name", + "carrier", + "bus0", + "bus1", + "p_nom", + "p_min_pu", + "build_year", + "lifetime", + "capital_cost", + "p_nom_extendable", + "isp_type", +] + +_LINK_TIMESLICE_LIMIT_COLUMNS = ["name", "attribute", "timeslice", "value"] + + +def _translate_network_geography_to_buses( + network_geography: pd.DataFrame, rezs: str +) -> pd.DataFrame: + """Creates one PyPSA bus per geography in the model. + + REZ buses are only created when REZs are modelled as discrete nodes; + with ``rezs="attached_to_parent_node"`` REZ-located components connect + straight to the parent geography's bus. + + I/O Example: + network_geography: + geo_id geo_type region_id subregion_id + NQ subregion QLD NQ + Q1 rez QLD NQ + + rezs = "discrete_nodes" returns: + name + NQ + Q1 + + rezs = "attached_to_parent_node" returns: + name + NQ + """ + buses = network_geography + if rezs != "discrete_nodes": + buses = buses[buses["geo_type"] != "rez"] + buses = buses.loc[:, ["geo_id"]].rename(columns={"geo_id": "name"}) + return buses.reset_index(drop=True) + + +def _translate_network_to_links( + ispypsa_tables: dict[str, pd.DataFrame], + config: ModelConfig, +) -> tuple[pd.DataFrame, pd.DataFrame]: + """Translates the network tables into PyPSA links and per-timeslice limits. + + Existing links carry the winter_reference limit as p_nom; limits for the + other demand conditions are returned per unit of p_nom in the + link_timeslice_limits table. Expansion links are built per investment + period from the unified expansion options/costs tables (their total + capacity is capped by the expansion-limit custom constraints built in + ispypsa.translator.constraints). + + Returns: + Tuple of (links, link_timeslice_limits) in PyPSA friendly format. + """ + rez_ids = _rez_geo_ids(ispypsa_tables["network_geography"]) + paths = _drop_rez_paths_if_not_modelled( + ispypsa_tables["network_transmission_paths"], rez_ids, config.network.nodes.rezs + ) + limits = ispypsa_tables["network_transmission_path_limits"] + static_limits = _extract_static_limits(limits) + existing_links = _build_existing_links( + paths, + static_limits, + rez_ids, + config.network.rez_to_sub_region_transmission_default_limit, + config.temporal.range.start_year, + ) + link_timeslice_limits = _translate_timeslice_limits_to_pu(limits, existing_links) + options = _pivot_physical_expansion_options( + ispypsa_tables["network_expansion_options"] + ) + options = _filter_options_to_enabled_expansion( + options, + existing_links, + config.network.transmission_expansion, + config.network.rez_transmission_expansion, + ) + costs = _prepare_expansion_costs( + ispypsa_tables["network_transmission_path_expansion_costs"], + config.temporal.capacity_expansion.investment_periods, + config.temporal.year_type, + config.wacc, + config.network.annuitisation_lifetime, + ) + expansion_links = _build_expansion_links(existing_links, options, costs) + links = pd.concat([existing_links, expansion_links], ignore_index=True) + return links.loc[:, _LINK_COLUMNS], link_timeslice_limits + + +def _rez_geo_ids(network_geography: pd.DataFrame) -> set[str]: + """geo_ids of the REZ entries in the geography. + + I/O Example: + geo_id=NQ/geo_type=subregion, geo_id=Q1/geo_type=rez -> {"Q1"} + """ + return set(network_geography.loc[network_geography["geo_type"] == "rez", "geo_id"]) + + +def _drop_rez_paths_if_not_modelled( + paths: pd.DataFrame, rez_ids: set[str], rezs: str +) -> pd.DataFrame: + """Drops REZ-to-parent paths when REZs are not modelled as discrete nodes. + + I/O Example: + paths (path_ids): CQ-NQ, Q1-NQ (Q1 is a REZ) + + rezs = "attached_to_parent_node" returns paths: CQ-NQ + rezs = "discrete_nodes" returns paths unchanged. + """ + if rezs == "discrete_nodes": + return paths + return paths[~paths["geo_from"].isin(rez_ids)] + + +def _extract_static_limits(limits: pd.DataFrame) -> pd.DataFrame: + """Reduces the per-timeslice limits to one static capacity per path and + direction: the winter_reference limit. + + Winter is used as the link's p_nom because it is the demand condition left + over when no summer timeslice is active — limits for the other conditions + are applied per unit of it. Collapsed all-NaN rows (paths with no limit + data) contribute nothing; paths absent from the result get the default + limit downstream. + + I/O Example: + limits: + path_id direction timeslice capacity + CQ-NQ forward qld_peak_demand 1200 + CQ-NQ forward qld_winter_reference 1400 + CQ-NQ reverse qld_winter_reference 1910 + N1-CNSW , , # collapsed row + + returns: + path_id direction capacity + CQ-NQ forward 1400 + CQ-NQ reverse 1910 + """ + winter = limits[limits["timeslice"].str.endswith("_winter_reference", na=False)] + return winter.loc[:, ["path_id", "direction", "capacity"]] + + +def _build_existing_links( + paths: pd.DataFrame, + static_limits: pd.DataFrame, + rez_ids: set[str], + default_limit: float, + start_year: int, +) -> pd.DataFrame: + """Builds one existing (non-extendable) PyPSA link per transmission path. + + I/O Example: + paths: + path_id geo_from geo_to carrier + CQ-NQ CQ NQ AC + N1-CNSW N1 CNSW AC # REZ path, no limit data + + static_limits: + path_id direction capacity + CQ-NQ forward 1400 + CQ-NQ reverse 1910 + + returns (abridged): + isp_name name bus0 bus1 p_nom p_min_pu isp_type + CQ-NQ CQ-NQ_existing CQ NQ 1400 -1.364 flow_path + N1-CNSW N1-CNSW_existing N1 CNSW 100000 -1.0 rez # default limit + """ + links = paths.rename( + columns={"path_id": "isp_name", "geo_from": "bus0", "geo_to": "bus1"} + ) + links = _add_static_capacities(links, static_limits, default_limit) + links["name"] = links["isp_name"] + "_existing" + links["isp_type"] = np.where(links["bus0"].isin(rez_ids), "rez", "flow_path") + links["build_year"] = start_year - 1 + links["lifetime"] = np.inf + links["capital_cost"] = np.nan + links["p_nom_extendable"] = False + return links + + +def _add_static_capacities( + links: pd.DataFrame, static_limits: pd.DataFrame, default_limit: float +) -> pd.DataFrame: + """Merges the static forward limit as p_nom and the reverse limit as p_min_pu. + + Paths with no static limit in a direction get the default: ``default_limit`` + as p_nom (the path is constraint-modelled with no explicit physical limit) + and a symmetric -1.0 as p_min_pu. Zero-capacity paths (new parallel + corridors) also get a symmetric p_min_pu, avoiding a 0/0 division. + + I/O Example: + links (isp_name): CQ-NQ, N1-CNSW + static_limits: CQ-NQ forward 1400, CQ-NQ reverse 1910 + + returns: + isp_name p_nom p_min_pu + CQ-NQ 1400 -1.364 # -1910/1400 + N1-CNSW 100000 -1.0 # defaults + """ + directions = static_limits.pivot( + index="path_id", columns="direction", values="capacity" + ).reindex(columns=["forward", "reverse"]) + links = links.merge( + directions, left_on="isp_name", right_index=True, how="left" + ).rename(columns={"forward": "p_nom"}) + links["p_min_pu"] = np.where( + links["reverse"].notna() & (links["p_nom"] > 0), + -1.0 * links["reverse"] / links["p_nom"], + -1.0, + ) + links["p_nom"] = links["p_nom"].fillna(default_limit) + return links.drop(columns=["reverse"]) + + +def _translate_timeslice_limits_to_pu( + limits: pd.DataFrame, existing_links: pd.DataFrame +) -> pd.DataFrame: + """Expresses each per-timeslice limit per unit of its link's p_nom. + + Forward limits become p_max_pu values and reverse limits p_min_pu values. + pypsa_build expands them into per-snapshot series via the + timeslice_snapshots mapping; snapshots outside any of a link's tagged + timeslices keep the static (winter_reference) limit. Zero-p_nom links + (new parallel corridors) are skipped — all their limits are zero and the + per-unit form is undefined. + + I/O Example: + limits: + path_id direction timeslice capacity + CQ-NQ forward qld_peak_demand 1200 + CQ-NQ forward qld_winter_reference 1400 + CQ-NQ reverse qld_peak_demand 1440 + + existing_links (abridged): + isp_name name p_nom + CQ-NQ CQ-NQ_existing 1400 + + returns: + name attribute timeslice value + CQ-NQ_existing p_max_pu qld_peak_demand 0.857 + CQ-NQ_existing p_max_pu qld_winter_reference 1.0 + CQ-NQ_existing p_min_pu qld_peak_demand -1.029 + """ + rows = limits.dropna(subset=["timeslice", "capacity"]) + rows = rows.merge( + existing_links.loc[:, ["isp_name", "name", "p_nom"]], + left_on="path_id", + right_on="isp_name", + ) + rows = rows[rows["p_nom"] > 0] + rows["attribute"] = rows["direction"].map( + {"forward": "p_max_pu", "reverse": "p_min_pu"} + ) + sign = rows["direction"].map({"forward": 1.0, "reverse": -1.0}) + rows["value"] = sign * rows["capacity"] / rows["p_nom"] + return rows.loc[:, _LINK_TIMESLICE_LIMIT_COLUMNS].reset_index(drop=True) + + +def _pivot_physical_expansion_options(expansion_options: pd.DataFrame) -> pd.DataFrame: + """Pairs each physical path's forward and reverse expansion rows into one row. + + constraint_relaxation rows are not physical paths — they are translated + into custom-constraint relaxation generators by + ispypsa.translator.constraints. + + I/O Example: + expansion_options: + expansion_id expansion_type allowed_expansion expansion_option + CQ-NQ forward 1000 Option 1 + CQ-NQ reverse 900 Option 1 + SWQLD1 constraint_relaxation 500 Option 2 + + returns: + expansion_id forward_capacity reverse_capacity + CQ-NQ 1000 900 + """ + physical = expansion_options[ + expansion_options["expansion_type"].isin(["forward", "reverse"]) + ] + options = ( + physical.pivot( + index="expansion_id", columns="expansion_type", values="allowed_expansion" + ) + .reindex(columns=["forward", "reverse"]) + .rename(columns={"forward": "forward_capacity", "reverse": "reverse_capacity"}) + ) + options.columns.name = None + return options.reset_index() + + +def _filter_options_to_enabled_expansion( + options: pd.DataFrame, + existing_links: pd.DataFrame, + transmission_expansion: bool, + rez_transmission_expansion: bool, +) -> pd.DataFrame: + """Keeps the expansion options enabled by the network config flags. + + ``transmission_expansion`` gates paths between (sub)regions; + ``rez_transmission_expansion`` gates REZ connection paths. Options for + paths not in the model (e.g. REZ paths dropped under + attached_to_parent_node) drop out here too, as they match no link. + + I/O Example: + options (expansion_ids): CQ-NQ, Q1-NQ + existing_links: CQ-NQ isp_type=flow_path, Q1-NQ isp_type=rez + + transmission_expansion=True, rez_transmission_expansion=False + returns options: CQ-NQ + """ + rez_path_ids = set( + existing_links.loc[existing_links["isp_type"] == "rez", "isp_name"] + ) + flow_path_ids = set( + existing_links.loc[existing_links["isp_type"] == "flow_path", "isp_name"] + ) + enabled = set() + if transmission_expansion: + enabled |= flow_path_ids + if rez_transmission_expansion: + enabled |= rez_path_ids + return options[options["expansion_id"].isin(enabled)] + + +def _prepare_expansion_costs( + expansion_costs: pd.DataFrame, + investment_periods: list[int], + year_type: str, + wacc: float, + asset_lifetime: int, +) -> pd.DataFrame: + """Filters the long-format expansion costs to the investment periods and + annuitises them. + + The ``year`` column holds financial-year ending years as ints, matching + the investment period labels used with ``year_type="fy"``. + + I/O Example: + expansion_costs: + expansion_id year cost + CQ-NQ 2025 1000 + CQ-NQ 2026 1010 + + investment_periods=[2026], wacc=0.07, asset_lifetime=30 returns: + expansion_id year capital_cost + CQ-NQ 2026 81.4 + """ + if year_type != "fy": + raise NotImplementedError( + f"Network expansion costs are not implemented for year_type: {year_type}" + ) + costs = expansion_costs[expansion_costs["year"].isin(investment_periods)].copy() + costs["capital_cost"] = costs["cost"].apply( + lambda cost: _annuitised_investment_costs(cost, wacc, asset_lifetime) + ) + return costs.loc[:, ["expansion_id", "year", "capital_cost"]] + + +def _build_expansion_links( + existing_links: pd.DataFrame, + options: pd.DataFrame, + costs: pd.DataFrame, +) -> pd.DataFrame: + """Builds one extendable PyPSA link per expandable path and investment period. + + Each expansion link is unbounded on its own — the expansion-limit custom + constraints built in ispypsa.translator.constraints cap the p_nom built + across a path's expansion links at the selected option's forward capacity. + Asymmetric options are modelled with p_min_pu = -reverse/forward, so + however much forward capacity is built, reverse capacity scales in the + option's proportion. + + I/O Example: + options: + expansion_id forward_capacity reverse_capacity + CQ-NQ 1000 900 + + costs: + expansion_id year capital_cost + CQ-NQ 2026 81.4 + + existing_links (abridged): CQ-NQ bus0=CQ bus1=NQ carrier=AC isp_type=flow_path + + returns (abridged): + isp_name name p_nom p_nom_extendable p_min_pu build_year capital_cost + CQ-NQ CQ-NQ_exp_2026 0.0 True -0.9 2026 81.4 + """ + links = costs.merge(options, on="expansion_id") + links = links.merge( + existing_links.loc[:, ["isp_name", "bus0", "bus1", "carrier", "isp_type"]], + left_on="expansion_id", + right_on="isp_name", + ) + links["name"] = links["expansion_id"] + "_exp_" + links["year"].astype(str) + links["p_nom"] = 0.0 + links["p_nom_extendable"] = True + links["p_min_pu"] = -1.0 * links["reverse_capacity"] / links["forward_capacity"] + links["build_year"] = links["year"] + links["lifetime"] = np.inf + return links diff --git a/src/ispypsa/translator/timeslices.py b/src/ispypsa/translator/timeslices.py new file mode 100644 index 00000000..1e9a68ff --- /dev/null +++ b/src/ispypsa/translator/timeslices.py @@ -0,0 +1,240 @@ +"""Re-sequences the timeslice patterns and maps them onto model snapshots. + +The templated ``timeslices`` table gives one month-day window pattern per +reference (weather) year. This module assigns a reference year to each model +financial year using the configured reference_year_cycle — the identical +assignment the trace pipeline uses, so timeslices stay consistent with the +demand and VRE traces — expands the assigned patterns into absolute date +windows, and joins them onto the model's snapshots. The resulting mapping is +what pypsa_build uses to expand per-timeslice link limits into series and to +restrict custom constraints to the snapshots their RHS values apply to. +""" + +import calendar +import logging + +import pandas as pd +from isp_trace_parser import construct_reference_year_mapping + +from ispypsa.config import ModelConfig + +logger = logging.getLogger(__name__) + +_TIMESLICE_SNAPSHOT_COLUMNS = ["timeslice_id", "investment_periods", "snapshots"] + + +def _create_timeslice_snapshot_mapping( + timeslices: pd.DataFrame, snapshots: pd.DataFrame, config: ModelConfig +) -> pd.DataFrame: + """Maps each snapshot to the timeslices active on its date under the + configured reference_year_cycle. + + I/O Example: + timeslices: + timeslice_id reference_year start_month_day end_month_day + nsw_peak_demand 2018 01-13 01-14 + vic_peak_demand 2018 01-20 01-21 # no snapshots inside + + snapshots (config: start_year 2025, reference_year_cycle [2018]): + investment_periods snapshots + 2025 2025-01-13 12:00:00 + 2025 2025-01-15 12:00:00 + + returns: + timeslice_id investment_periods snapshots + nsw_peak_demand 2025 2025-01-13 12:00:00 + """ + if config.temporal.year_type != "fy": + raise NotImplementedError( + "Timeslice re-sequencing is only implemented for fy year_type; " + "AEMO's timeslice calendar is built on financial years." + ) + if timeslices.empty: + return pd.DataFrame(columns=_TIMESLICE_SNAPSHOT_COLUMNS) + reference_year_mapping = _map_model_years_to_reference_years(config) + _raise_on_reference_years_without_patterns(reference_year_mapping, timeslices) + windows = _expand_patterns_to_absolute_windows(timeslices, reference_year_mapping) + mapped = [ + _snapshots_in_window(snapshots, window) for window in windows.itertuples() + ] + return _concat_window_snapshots(mapped) + + +def _map_model_years_to_reference_years(config: ModelConfig) -> dict[int, int]: + """The model-year -> reference-year assignment, via the identical + construct_reference_year_mapping call the trace pipeline makes, plus the + year before the first model year (cycle-consistent: the cycle's last + reference year precedes its first) so windows starting in the prior + financial year — winter runs April to October — cover the first model + year's July-September snapshots. + + I/O Example: + start_year 2025, end_year 2027, reference_year_cycle [2011, 2018] + -> {2024: 2018, 2025: 2011, 2026: 2018, 2027: 2011} + """ + cycle = config.temporal.capacity_expansion.reference_year_cycle + mapping = construct_reference_year_mapping( + start_year=config.temporal.range.start_year, + end_year=config.temporal.range.end_year, + reference_years=cycle, + ) + return {config.temporal.range.start_year - 1: cycle[-1], **mapping} + + +def _raise_on_reference_years_without_patterns( + reference_year_mapping: dict[int, int], timeslices: pd.DataFrame +) -> None: + """Raise if the configured cycle uses reference years the timeslices + table has no patterns for — silently producing no windows would let + timeslice-tagged limits and constraints never bind.""" + missing = sorted( + set(reference_year_mapping.values()) - set(timeslices["reference_year"]) + ) + if missing: + raise ValueError( + f"Configured reference_year_cycle includes reference years with " + f"no timeslice window patterns: {missing}" + ) + + +def _expand_patterns_to_absolute_windows( + timeslices: pd.DataFrame, reference_year_mapping: dict[int, int] +) -> pd.DataFrame: + """Expands each model year's assigned pattern into absolute date windows. + + I/O Example: + timeslices: + timeslice_id reference_year start_month_day end_month_day + nsw_peak_demand 2018 11-18 11-20 + + reference_year_mapping {2026: 2018} -> + timeslice_id start_date end_date + nsw_peak_demand 2025-11-18 2025-11-20 + """ + expanded = [] + for model_year, reference_year in reference_year_mapping.items(): + pattern = timeslices[timeslices["reference_year"] == reference_year] + expanded.append(_place_pattern_in_financial_year(pattern, model_year)) + return pd.concat(expanded, ignore_index=True) + + +def _place_pattern_in_financial_year( + pattern: pd.DataFrame, model_year: int +) -> pd.DataFrame: + """Turns one pattern's month-day windows into absolute dates in the + financial year ending model_year: starts in July-December land in + model_year - 1, January-June in model_year, and each end is its first + occurrence after the start (ends may fall past 30 June, e.g. winter + April-October). + + I/O Example (model_year=2026): + timeslice_id start_month_day end_month_day + nsw_peak_demand 11-18 11-20 + nsw_summer_typical 11-20 03-20 # end wraps past new year + nsw_winter_reference 04-01 10-01 # end past 30 June + + returns: + timeslice_id start_date end_date + nsw_peak_demand 2025-11-18 2025-11-20 + nsw_summer_typical 2025-11-20 2026-03-20 + nsw_winter_reference 2026-04-01 2026-10-01 + """ + placed = pattern.copy() + placed["start_date"] = placed["start_month_day"].apply( + lambda month_day: _place_start_in_financial_year(month_day, model_year) + ) + placed["end_date"] = placed.apply( + lambda row: _resolve_end_date( + row["end_month_day"], row["start_month_day"], row["start_date"] + ), + axis=1, + ) + return placed[["timeslice_id", "start_date", "end_date"]] + + +def _place_start_in_financial_year(month_day: str, model_year: int) -> pd.Timestamp: + """'11-18' in FY2026 -> 2025-11-18; '04-01' in FY2026 -> 2026-04-01.""" + month = int(month_day.split("-")[0]) + year = model_year - 1 if month >= 7 else model_year + return _timestamp_with_leap_day_clamp(year, month_day) + + +def _resolve_end_date( + end_month_day: str, start_month_day: str, start: pd.Timestamp +) -> pd.Timestamp: + """An end later in the calendar year than the start lands in the start's + year; otherwise it wraps into the next. The comparison uses the original + month-day strings (which compare lexically in chronological order) + rather than the placed dates, so leap-day clamping a one-day 02-28 to + 02-29 window in a non-leap year collapses it to an empty window instead + of wrapping its end a year out. + + I/O Example: + ('03-20', '11-18', 2025-11-18) -> 2026-03-20 # wraps + ('10-01', '04-01', 2026-04-01) -> 2026-10-01 + ('02-29', '02-28', 2025-02-28) -> 2025-02-28 # clamped: empty window + """ + year = start.year if end_month_day > start_month_day else start.year + 1 + return _timestamp_with_leap_day_clamp(year, end_month_day) + + +def _timestamp_with_leap_day_clamp(year: int, month_day: str) -> pd.Timestamp: + """'02-29' is clamped to 28 February in non-leap years — patterns from + leap reference years land in non-leap model years under re-sequencing. + + I/O Example: + (2025, '02-29') -> 2025-02-28; (2024, '02-29') -> 2024-02-29 + """ + month, day = (int(part) for part in month_day.split("-")) + if (month, day) == (2, 29) and not calendar.isleap(year): + day = 28 + return pd.Timestamp(year=year, month=month, day=day) + + +def _snapshots_in_window(snapshots: pd.DataFrame, window) -> pd.DataFrame: + """Selects the snapshots inside one window, tagged with its timeslice. + + I/O Example: + snapshots 2025-01-13 12:00 and 2025-01-15 12:00, + window nsw_peak_demand [2025-01-13, 2025-01-14) + -> the 2025-01-13 12:00 snapshot tagged nsw_peak_demand + """ + in_window = (snapshots["snapshots"] >= window.start_date) & ( + snapshots["snapshots"] < window.end_date + ) + tagged = snapshots.loc[in_window, ["investment_periods", "snapshots"]].copy() + tagged["timeslice_id"] = window.timeslice_id + return tagged + + +def _concat_window_snapshots(mapped: list[pd.DataFrame]) -> pd.DataFrame: + """Combines the per-window snapshot selections into one mapping table.""" + if not mapped: + return pd.DataFrame(columns=_TIMESLICE_SNAPSHOT_COLUMNS) + mapping = pd.concat(mapped, ignore_index=True) + return mapping.loc[:, _TIMESLICE_SNAPSHOT_COLUMNS] + + +def _log_referenced_timeslices_without_snapshots( + timeslice_snapshots: pd.DataFrame, + link_timeslice_limits: pd.DataFrame, + custom_constraints_rhs: pd.DataFrame, +) -> None: + """Logs the timeslices referenced by a limit or constraint but mapped to + no snapshots — those limits and constraints will never apply. + + This is expected when snapshot aggregation (e.g. representative weeks) + selects no snapshots inside a timeslice's windows, and for calendar + timeslices that never activate (tas_peak_demand in the Draft 2026 ISP + calendar), but the user should know the affected inputs will not bind. + """ + referenced = set(link_timeslice_limits["timeslice"]) | set( + custom_constraints_rhs["timeslice"].dropna() + ) + without_snapshots = referenced - set(timeslice_snapshots["timeslice_id"]) + if without_snapshots: + logger.warning( + f"Timeslices referenced by transmission limits or custom constraints " + f"but with no snapshots in the model (these limits and constraints " + f"will never apply): {sorted(without_snapshots)}" + ) diff --git a/tests/test_cli/test_create_ispypsa_inputs_new_table_formats.py b/tests/test_cli/test_create_ispypsa_inputs_new_table_formats.py index 79a967b4..1de4d8da 100644 --- a/tests/test_cli/test_create_ispypsa_inputs_new_table_formats.py +++ b/tests/test_cli/test_create_ispypsa_inputs_new_table_formats.py @@ -71,13 +71,15 @@ "network_transmission_path_limits", "network_expansion_options", "network_transmission_path_expansion_costs", + "timeslices", "costs_connection", ] # Custom constraints are templated only at sub_regions (coarser granularities -# collapse the entities they reference). Detailed content lives in -# test_custom_constraints_from_plexos.py; here we check the CLI writes them at -# sub_regions and omits them otherwise. +# collapse the entities they reference) but written at every granularity — +# header-only at nem_regions / single_region. Detailed content lives in +# test_custom_constraints_from_plexos.py; here we check the CLI writes them +# populated at sub_regions and empty otherwise. _CUSTOM_CONSTRAINT_OUTPUTS = [ "custom_constraints", "custom_constraints_lhs", @@ -286,6 +288,15 @@ def test_create_ispypsa_inputs_task_new_format( "single_region": 1209, } +# Per-reference-year window patterns decoded from the shipped RefYear5000 +# calendar and reference_year_sequence (drift-detection; +# granularity-invariant). 14 distinct ids, not 15: TAS Hot Day never +# activates in the calendar, so tas_peak_demand has no windows. 15 reference +# years: 2011 to 2025 in year-ending convention. +_NUM_TIMESLICE_PATTERN_ROWS_75 = 678 +_NUM_TIMESLICE_IDS_75 = 14 +_NUM_REFERENCE_YEARS_75 = 15 + @pytest.mark.parametrize("granularity", ["sub_regions", "nem_regions", "single_region"]) def test_create_ispypsa_inputs_new_format( @@ -355,18 +366,25 @@ def test_create_ispypsa_inputs_new_format( assert set(costs["expansion_id"]) == set(options["expansion_id"]) assert len(costs) == _EXPECTED_EXPANSION_COST_ROWS_75[granularity] - # custom_constraints — written only at sub_regions. Detailed content is - # covered by test_custom_constraints_from_plexos.py; here we assert the CLI - # emits the three tables at sub_regions with no orphan LHS/RHS rows, and - # omits them entirely at coarser granularities. + # timeslices — per-reference-year patterns decoded from the shipped + # RefYear5000 calendar, identical at every granularity. + timeslices = pd.read_csv(output_dir / "timeslices.csv") + assert len(timeslices) == _NUM_TIMESLICE_PATTERN_ROWS_75 + assert timeslices["timeslice_id"].nunique() == _NUM_TIMESLICE_IDS_75 + assert timeslices["reference_year"].nunique() == _NUM_REFERENCE_YEARS_75 + + # custom_constraints — templated only at sub_regions but written at every + # granularity (header-only at coarser ones). Detailed content is covered + # by test_custom_constraints_from_plexos.py; here we assert the CLI emits + # the three tables populated at sub_regions with no orphan LHS/RHS rows, + # and empty otherwise. + verify_output_files(output_dir, _CUSTOM_CONSTRAINT_OUTPUTS) + constraints = pd.read_csv(output_dir / "custom_constraints.csv") + lhs = pd.read_csv(output_dir / "custom_constraints_lhs.csv") + rhs = pd.read_csv(output_dir / "custom_constraints_rhs.csv") if granularity == "sub_regions": - verify_output_files(output_dir, _CUSTOM_CONSTRAINT_OUTPUTS) - constraints = pd.read_csv(output_dir / "custom_constraints.csv") - lhs = pd.read_csv(output_dir / "custom_constraints_lhs.csv") - rhs = pd.read_csv(output_dir / "custom_constraints_rhs.csv") constraint_ids = set(constraints["constraint_id"]) assert set(lhs["constraint_id"]) <= constraint_ids assert set(rhs["constraint_id"]) <= constraint_ids else: - for name in _CUSTOM_CONSTRAINT_OUTPUTS: - assert not (output_dir / f"{name}.csv").exists() + assert constraints.empty and lhs.empty and rhs.empty diff --git a/tests/test_cli/test_create_pypsa_friendly_inputs_new_table_formats.py b/tests/test_cli/test_create_pypsa_friendly_inputs_new_table_formats.py new file mode 100644 index 00000000..fc2448be --- /dev/null +++ b/tests/test_cli/test_create_pypsa_friendly_inputs_new_table_formats.py @@ -0,0 +1,151 @@ +"""Tests for the create_pypsa_friendly_inputs CLI task under the new table format. + +Mirrors test_create_pypsa_friendly_inputs.py (fresh run, up-to-date detection, +and triggers) for the new-format translator path: network and custom-constraint +tables only, no generators/batteries and no timeseries directory. + +FEATURE_FLAG_CLEANUP[use_new_table_format]: merge into +test_create_pypsa_friendly_inputs.py when the legacy format is removed. +""" + +import time + +import pandas as pd + +from ispypsa.translator.create_pypsa_friendly import _NEW_FORMAT_TRANSLATOR_OUTPUTS + +from .cli_test_helpers import ( + assert_task_ran, + assert_task_up_to_date, + get_file_timestamps, + mock_workbook_file, + modify_config_value, + run_cli_command, + run_extensive, + verify_output_files, +) +from .cli_test_helpers_new_table_formats import ( + mock_config_new_format, + prepare_test_cache_new_format, +) + + +def test_create_pypsa_friendly_inputs_task_new_format( + mock_config_new_format, + prepare_test_cache_new_format, + tmp_path, + run_cli_command, + monkeypatch, + run_extensive, +): + """Test fresh run, up-to-date detection, and triggers for the new format. + + Combines: + - Fresh run creates the new-format translator outputs (and no timeseries) + - Up-to-date detection + - Config file modification trigger (extensive only) + - Dependency modification trigger (extensive only) + - Missing single file trigger (extensive only) + + Runs: 2 (fresh + up-to-date), or 5 with extensive + """ + monkeypatch.setenv("ISPYPSA_USE_NEW_TABLE_FORMAT", "true") + monkeypatch.setenv("ISPYPSA_TEST_MOCK_CACHE", "true") + + # Test fresh run (doit also runs the upstream create_ispypsa_inputs task). + result = run_cli_command( + [f"config={mock_config_new_format}", "create_pypsa_friendly_inputs"] + ) + assert result.returncode == 0, ( + f"Command failed with return code {result.returncode}\n" + f"STDOUT:\n{result.stdout}\nSTDERR:\n{result.stderr}" + ) + + output_dir = tmp_path / "run_dir" / "test_run" / "pypsa_friendly" + assert output_dir.exists() + verify_output_files(output_dir, _NEW_FORMAT_TRANSLATOR_OUTPUTS) + + # No generators/batteries and no timeseries until generator translation + # lands for the new format. + assert not (output_dir / "generators.csv").exists() + assert not (output_dir / "batteries.csv").exists() + assert not (output_dir / "capacity_expansion_timeseries").exists() + + # Spot-check the wiring of the outputs (content is covered by the + # translator unit tests). 15 sub-regions + 47 REZs at 7.5. + buses = pd.read_csv(output_dir / "buses.csv") + assert len(buses) == 62 + + links = pd.read_csv(output_dir / "links.csv") + limits = pd.read_csv(output_dir / "link_timeslice_limits.csv") + assert set(limits["name"]) <= set(links["name"]) + + lhs = pd.read_csv(output_dir / "custom_constraints_lhs.csv") + rhs = pd.read_csv(output_dir / "custom_constraints_rhs.csv") + assert len(rhs) > 0 + assert set(lhs["constraint_name"]) == set(rhs["constraint_name"]) + + mapping = pd.read_csv(output_dir / "timeslice_snapshots.csv") + snapshots = pd.read_csv(output_dir / "snapshots.csv") + assert len(mapping) > 0 + assert set(mapping["snapshots"]) <= set(snapshots["snapshots"]) + + # Get timestamps from first run + first_run_timestamps = get_file_timestamps(output_dir) + time.sleep(0.1) + + # Test up-to-date detection - second run + result = run_cli_command( + [f"config={mock_config_new_format}", "create_pypsa_friendly_inputs"] + ) + assert result.returncode == 0 + assert_task_up_to_date(result.stdout, "create_pypsa_friendly_inputs") + + second_run_timestamps = get_file_timestamps(output_dir) + assert first_run_timestamps == second_run_timestamps + + if not run_extensive: + return + + # Test config file modification triggers rerun (extensive only) + time.sleep(0.1) + modify_config_value(mock_config_new_format, "discount_rate", 0.06) + result = run_cli_command( + [f"config={mock_config_new_format}", "create_pypsa_friendly_inputs"] + ) + assert result.returncode == 0 + assert_task_ran(result.stdout, "create_pypsa_friendly_inputs") + + # Test dependency modification trigger (extensive only): edit a templated + # input the translator consumes. + time.sleep(0.1) + limits_file = ( + tmp_path + / "run_dir" + / "test_run" + / "ispypsa_inputs" + / "network_transmission_path_limits.csv" + ) + df = pd.read_csv(limits_file) + df["capacity"] = df["capacity"] + 1.0 + df.to_csv(limits_file, index=False) + + result = run_cli_command( + [f"config={mock_config_new_format}", "create_pypsa_friendly_inputs"] + ) + assert result.returncode == 0 + assert_task_ran(result.stdout, "create_pypsa_friendly_inputs") + + config_change_timestamps = get_file_timestamps(output_dir) + + # Test missing single file trigger (extensive only) + target_file = output_dir / "buses.csv" + target_file.unlink() + + result = run_cli_command( + [f"config={mock_config_new_format}", "create_pypsa_friendly_inputs"] + ) + assert result.returncode == 0 + assert_task_ran(result.stdout, "create_pypsa_friendly_inputs") + assert target_file.exists() + assert target_file.stat().st_size > 0 diff --git a/tests/test_model/test_add_links_with_timeslice_limits.py b/tests/test_model/test_add_links_with_timeslice_limits.py new file mode 100644 index 00000000..624945d5 --- /dev/null +++ b/tests/test_model/test_add_links_with_timeslice_limits.py @@ -0,0 +1,72 @@ +import pandas as pd +import pypsa +import pytest + +from ispypsa.pypsa_build.links import _add_links_to_network + + +def _network() -> pypsa.Network: + snapshots = pd.date_range("2025-01-01", periods=4, freq="h") + index = pd.MultiIndex.from_arrays([[2025] * 4, list(snapshots)]) + network = pypsa.Network(snapshots=index, investment_periods=[2025]) + network.add("Bus", "bus1") + network.add("Bus", "bus2") + return network + + +def _links(csv_str_to_df) -> pd.DataFrame: + return csv_str_to_df(""" + name, bus0, bus1, carrier, p_nom, p_min_pu, p_nom_extendable + CQ-NQ_existing, bus1, bus2, AC, 1400, -1.364286, False + """) + + +def test_links_get_per_snapshot_pu_series(csv_str_to_df): + network = _network() + link_timeslice_limits = csv_str_to_df(""" + name, attribute, timeslice, value + CQ-NQ_existing, p_max_pu, qld_peak_demand, 0.857143 + CQ-NQ_existing, p_min_pu, qld_peak_demand, -1.028571 + """) + timeslice_snapshots = csv_str_to_df(""" + timeslice_id, investment_periods, snapshots + qld_peak_demand, 2025, 2025-01-01 01:00:00 + qld_peak_demand, 2025, 2025-01-01 02:00:00 + """) + + _add_links_to_network( + network, _links(csv_str_to_df), link_timeslice_limits, timeslice_snapshots + ) + + p_max_pu = network.links_t.p_max_pu["CQ-NQ_existing"] + p_min_pu = network.links_t.p_min_pu["CQ-NQ_existing"] + # The static (winter) limit holds outside the tagged snapshots. + expected_p_max_pu = [1.0, 0.857143, 0.857143, 1.0] + expected_p_min_pu = [-1.364286, -1.028571, -1.028571, -1.364286] + assert p_max_pu.tolist() == pytest.approx(expected_p_max_pu) + assert p_min_pu.tolist() == pytest.approx(expected_p_min_pu) + + +def test_links_without_timeslice_limits_stay_static(csv_str_to_df): + network = _network() + link_timeslice_limits = csv_str_to_df(""" + name, attribute, timeslice, value + """) + timeslice_snapshots = csv_str_to_df(""" + timeslice_id, investment_periods, snapshots + """) + + _add_links_to_network( + network, _links(csv_str_to_df), link_timeslice_limits, timeslice_snapshots + ) + + assert "CQ-NQ_existing" not in network.links_t.p_max_pu.columns + assert network.links.loc["CQ-NQ_existing", "p_min_pu"] == pytest.approx(-1.364286) + + +def test_links_old_format_call_without_limit_tables(csv_str_to_df): + network = _network() + + _add_links_to_network(network, _links(csv_str_to_df)) + + assert network.links.loc["CQ-NQ_existing", "p_nom"] == 1400 diff --git a/tests/test_model/test_custom_constraints_with_temporal_scope.py b/tests/test_model/test_custom_constraints_with_temporal_scope.py new file mode 100644 index 00000000..1ed3856f --- /dev/null +++ b/tests/test_model/test_custom_constraints_with_temporal_scope.py @@ -0,0 +1,197 @@ +import pandas as pd +import pypsa +import pytest + +from ispypsa.pypsa_build.custom_constraints import ( + _add_custom_constraints_with_temporal_scope, +) + + +def _two_period_network() -> pypsa.Network: + """Two investment periods (2025, 2030) with four hourly snapshots each, + one bus, a cheap and an expensive generator, and a constant 100 MW load.""" + snapshots_2025 = pd.date_range("2025-01-01", periods=4, freq="h") + snapshots_2030 = pd.date_range("2030-01-01", periods=4, freq="h") + index = pd.MultiIndex.from_arrays( + [[2025] * 4 + [2030] * 4, list(snapshots_2025) + list(snapshots_2030)] + ) + network = pypsa.Network(snapshots=index, investment_periods=[2025, 2030]) + network.investment_period_weightings = pd.DataFrame( + {"years": [5, 5], "objective": [1.0, 1.0]}, + index=pd.Index([2025, 2030], name="period"), + ) + network.add("Bus", "bus1") + network.add("Generator", "cheap_gen", bus="bus1", p_nom=200, marginal_cost=10) + network.add("Generator", "dear_gen", bus="bus1", p_nom=200, marginal_cost=100) + network.add("Load", "load1", bus="bus1") + network.loads_t.p_set = pd.DataFrame({"load1": [100] * 8}, index=network.snapshots) + return network + + +def _timeslice_snapshots(csv_str_to_df) -> pd.DataFrame: + """peak_2025 tags the middle two snapshots of the 2025 period.""" + mapping = csv_str_to_df(""" + timeslice_id, investment_periods, snapshots + peak_2025, 2025, 2025-01-01 01:00:00 + peak_2025, 2025, 2025-01-01 02:00:00 + """) + return mapping + + +def test_timeslice_restricted_constraint_binds_only_at_tagged_snapshots( + csv_str_to_df, +): + network = _two_period_network() + rhs = csv_str_to_df(""" + constraint_name, investment_period, timeslice, rhs, constraint_type + cheap_gen_limit, 2025, peak_2025, 40, <= + """) + lhs = csv_str_to_df(""" + constraint_name, investment_period, variable_name, component, attribute, coefficient + cheap_gen_limit, 2025, cheap_gen, Generator, p, 1.0 + """) + + network.optimize.create_model(multi_investment_periods=True) + _add_custom_constraints_with_temporal_scope( + network, rhs, lhs, _timeslice_snapshots(csv_str_to_df) + ) + network.optimize.solve_model() + + cheap_gen_output = network.generators_t.p["cheap_gen"] + # Constrained to 40 at the two tagged 2025 snapshots; free (meets the + # whole 100 MW load) everywhere else, including all of 2030. + expected = pd.Series( + [100.0, 40.0, 40.0, 100.0, 100.0, 100.0, 100.0, 100.0], + index=network.snapshots, + name="cheap_gen", + ) + pd.testing.assert_series_equal(cheap_gen_output, expected, check_names=False) + + +def test_rhs_varies_by_investment_period(csv_str_to_df): + network = _two_period_network() + rhs = csv_str_to_df(""" + constraint_name, investment_period, timeslice, rhs, constraint_type + cheap_gen_limit, 2025, , 40, <= + cheap_gen_limit, 2030, , 60, <= + """) + lhs = csv_str_to_df(""" + constraint_name, investment_period, variable_name, component, attribute, coefficient + cheap_gen_limit, 2025, cheap_gen, Generator, p, 1.0 + cheap_gen_limit, 2030, cheap_gen, Generator, p, 1.0 + """) + + network.optimize.create_model(multi_investment_periods=True) + _add_custom_constraints_with_temporal_scope( + network, rhs, lhs, _timeslice_snapshots(csv_str_to_df) + ) + network.optimize.solve_model() + + cheap_gen_output = network.generators_t.p["cheap_gen"] + assert (cheap_gen_output.loc[(2025,)] == 40.0).all() + assert (cheap_gen_output.loc[(2030,)] == 60.0).all() + + +def test_unrestricted_p_nom_constraint(csv_str_to_df): + """NaN investment_period and timeslice (the expansion-limit pattern) + creates one global constraint over p_nom variables.""" + network = _two_period_network() + network.add( + "Generator", + "exp_gen", + bus="bus1", + p_nom_extendable=True, + capital_cost=1, + marginal_cost=1, + build_year=2025, + lifetime=100, + ) + rhs = csv_str_to_df(""" + constraint_name, investment_period, timeslice, rhs, constraint_type + exp_limit, , , 30, <= + """) + lhs = csv_str_to_df(""" + constraint_name, investment_period, variable_name, component, attribute, coefficient + exp_limit, , exp_gen, Generator, p_nom, 1.0 + """) + + network.optimize.create_model(multi_investment_periods=True) + _add_custom_constraints_with_temporal_scope( + network, rhs, lhs, _timeslice_snapshots(csv_str_to_df) + ) + network.optimize.solve_model() + + # exp_gen is the cheapest source so it builds to the constraint's cap. + assert network.generators.loc["exp_gen", "p_nom_opt"] == pytest.approx(30.0) + + +def test_terms_for_components_not_in_model_skipped_and_logged(csv_str_to_df, caplog): + """Generator and battery terms reference IASR IDs until generator + translation lands — they are skipped per-miss with a log line and the + rest of the constraint still applies.""" + network = _two_period_network() + rhs = csv_str_to_df(""" + constraint_name, investment_period, timeslice, rhs, constraint_type + cheap_gen_limit, 2025, peak_2025, 40, <= + """) + lhs = csv_str_to_df(""" + constraint_name, investment_period, variable_name, component, attribute, coefficient + cheap_gen_limit, 2025, cheap_gen, Generator, p, 1.0 + cheap_gen_limit, 2025, KINGASF1, Generator, p, 0.5 + """) + + network.optimize.create_model(multi_investment_periods=True) + with caplog.at_level("INFO"): + _add_custom_constraints_with_temporal_scope( + network, rhs, lhs, _timeslice_snapshots(csv_str_to_df) + ) + network.optimize.solve_model() + + assert ( + "Generator KINGASF1 not in model, custom constraint term skipped." + ) in caplog.text + # The cheap_gen term still binds at the tagged snapshots. + assert network.generators_t.p["cheap_gen"].loc[(2025,)].iloc[1] == pytest.approx( + 40.0 + ) + + +def test_constraint_with_no_terms_in_model_skipped(csv_str_to_df): + network = _two_period_network() + rhs = csv_str_to_df(""" + constraint_name, investment_period, timeslice, rhs, constraint_type + ghost_limit, 2025, peak_2025, 40, <= + """) + lhs = csv_str_to_df(""" + constraint_name, investment_period, variable_name, component, attribute, coefficient + ghost_limit, 2025, KINGASF1, Generator, p, 1.0 + """) + + network.optimize.create_model(multi_investment_periods=True) + _add_custom_constraints_with_temporal_scope( + network, rhs, lhs, _timeslice_snapshots(csv_str_to_df) + ) + network.optimize.solve_model() + + # No constraint applied — the cheap generator meets the whole load. + assert (network.generators_t.p["cheap_gen"] == 100.0).all() + + +def test_constraint_for_timeslice_with_no_snapshots_skipped(csv_str_to_df): + network = _two_period_network() + rhs = csv_str_to_df(""" + constraint_name, investment_period, timeslice, rhs, constraint_type + cheap_gen_limit, 2025, tas_peak_demand, 40, <= + """) + lhs = csv_str_to_df(""" + constraint_name, investment_period, variable_name, component, attribute, coefficient + cheap_gen_limit, 2025, cheap_gen, Generator, p, 1.0 + """) + + network.optimize.create_model(multi_investment_periods=True) + _add_custom_constraints_with_temporal_scope( + network, rhs, lhs, _timeslice_snapshots(csv_str_to_df) + ) + network.optimize.solve_model() + + assert (network.generators_t.p["cheap_gen"] == 100.0).all() diff --git a/tests/test_templater/test_create_ispypsa_inputs_template.py b/tests/test_templater/test_create_ispypsa_inputs_template.py index 9675f1ed..ecc04dcc 100644 --- a/tests/test_templater/test_create_ispypsa_inputs_template.py +++ b/tests/test_templater/test_create_ispypsa_inputs_template.py @@ -57,21 +57,33 @@ def _stub_custom_constraints_tables() -> dict[str, pd.DataFrame]: } -def test_list_templater_output_files_includes_custom_constraints_only_at_sub_regions(): - """Custom-constraint tables are declared as task outputs only at sub_regions. - - They are templated (and written) only at sub_regions, so they must appear in - the output-file list there — otherwise the create_ispypsa_inputs task leaves - files it writes untracked — and must be absent at nem_regions / - single_region, where declaring them would make the task expect files that are - never written. +def _stub_timeslice_calendar() -> pd.DataFrame: + """Minimal raw timeslice calendar: one closed window in planning year + FY2026. Patched in place of the packaged PLEXOS calendar; full decode + behaviour is covered by the per-module tests in ``test_timeslices.py``. """ - custom_constraint_files = { - "custom_constraints", - "custom_constraints_lhs", - "custom_constraints_rhs", - } + return pd.DataFrame( + { + "DATETIME": ["18/11/2025", "20/11/2025"], + "NAME": ["NSW Hot Day", "NSW Hot Day"], + "TIMESLICE": [-1, 0], + } + ) + + +def _stub_reference_year_sequence() -> pd.DataFrame: + """Assigns the stub calendar's single planning year a reference year.""" + return pd.DataFrame({"planning_year": [2026], "reference_year": [2015]}) + + +def test_list_templater_output_files_granularity_invariant(): + """The new-format templater declares the same output set at every granularity. + The custom-constraint tables are header-only at nem_regions / + single_region but still written, so the create_ispypsa_inputs task must + track them everywhere — a granularity-dependent list would leave written + files untracked or expect files that are never written. + """ with patch( "ispypsa.templater.create_template.FEATURE_FLAGS", {"use_new_table_format": True}, @@ -80,9 +92,14 @@ def test_list_templater_output_files_includes_custom_constraints_only_at_sub_reg nem_regions = set(list_templater_output_files("nem_regions")) single_region = set(list_templater_output_files("single_region")) - assert custom_constraint_files <= sub_regions - assert custom_constraint_files.isdisjoint(nem_regions) - assert custom_constraint_files.isdisjoint(single_region) + assert sub_regions == nem_regions == single_region + assert { + "custom_constraints", + "custom_constraints_lhs", + "custom_constraints_rhs", + "timeslices", + "costs_connection", + } <= sub_regions def test_create_ispypsa_inputs_template_sub_regions( @@ -239,6 +256,10 @@ def test_create_ispypsa_inputs_template_new_format(csv_str_to_df): "ispypsa.templater.create_template.template_custom_constraints_from_plexos", return_value=_stub_custom_constraints_tables(), ), + patch( + "ispypsa.templater.create_template.load_timeslice_calendar", + return_value=_stub_timeslice_calendar(), + ), ): result = create_ispypsa_inputs_template( scenario="Step Change", @@ -264,6 +285,7 @@ def test_create_ispypsa_inputs_template_new_format(csv_str_to_df): # wiring runs. Output stays empty: generators/storage are placeholder-empty. manually_extracted_tables={ "connection_capacity_non_vre": connection_capacity_non_vre, + "reference_year_sequence": _stub_reference_year_sequence(), }, iasr_workbook_version="ignored-by-patch", ) @@ -304,6 +326,16 @@ def test_create_ispypsa_inputs_template_new_format(csv_str_to_df): # 3 expansion_ids x 2 years assert len(expansion_costs) == 6 + timeslices = result["timeslices"] + assert set(timeslices.columns) == { + "timeslice_id", + "reference_year", + "start_month_day", + "end_month_day", + } + # The stub calendar's single on/off pair decodes into one window pattern. + assert len(timeslices) == 1 + assert "costs_connection" in result costs_connection = result["costs_connection"] assert set(costs_connection.columns) == { @@ -427,6 +459,10 @@ def test_create_ispypsa_inputs_template_new_format_nem_regions(csv_str_to_df): "ispypsa.templater.create_template.template_custom_constraints_from_plexos", return_value=_stub_custom_constraints_tables(), ) as mock_template_custom_constraints, + patch( + "ispypsa.templater.create_template.load_timeslice_calendar", + return_value=_stub_timeslice_calendar(), + ), ): result = create_ispypsa_inputs_template( scenario="Step Change", @@ -449,6 +485,7 @@ def test_create_ispypsa_inputs_template_new_format_nem_regions(csv_str_to_df): }, manually_extracted_tables={ "connection_capacity_non_vre": connection_capacity_non_vre, + "reference_year_sequence": _stub_reference_year_sequence(), }, iasr_workbook_version="ignored-by-patch", ) @@ -490,13 +527,28 @@ def test_create_ispypsa_inputs_template_new_format_nem_regions(csv_str_to_df): assert costs_connection.empty # Custom constraints from PLEXOS are sub-regional export limits with no - # meaningful representation once sub-regions are collapsed, so the templater - # skips them at nem_regions granularity. The mock turns a regression in that - # gate into a clean assertion failure instead of a PLEXOS-extract disk read. + # meaningful representation once sub-regions are collapsed, so at + # nem_regions the templater emits them header-only instead of templating + # them. The mock turns a regression in that gate into a clean assertion + # failure instead of a PLEXOS-extract disk read. mock_template_custom_constraints.assert_not_called() - assert "custom_constraints" not in result - assert "custom_constraints_lhs" not in result - assert "custom_constraints_rhs" not in result + assert set(result["custom_constraints"].columns) == {"constraint_id", "direction"} + assert result["custom_constraints"].empty + assert set(result["custom_constraints_lhs"].columns) == { + "constraint_id", + "term_type", + "variable_name", + "coefficient", + "date_from", + } + assert result["custom_constraints_lhs"].empty + assert set(result["custom_constraints_rhs"].columns) == { + "constraint_id", + "timeslice", + "rhs", + "date_from", + } + assert result["custom_constraints_rhs"].empty def test_create_ispypsa_inputs_template_new_format_single_region(csv_str_to_df): @@ -570,6 +622,10 @@ def test_create_ispypsa_inputs_template_new_format_single_region(csv_str_to_df): "ispypsa.templater.create_template.template_custom_constraints_from_plexos", return_value=_stub_custom_constraints_tables(), ) as mock_template_custom_constraints, + patch( + "ispypsa.templater.create_template.load_timeslice_calendar", + return_value=_stub_timeslice_calendar(), + ), ): result = create_ispypsa_inputs_template( scenario="Step Change", @@ -590,6 +646,7 @@ def test_create_ispypsa_inputs_template_new_format_single_region(csv_str_to_df): }, manually_extracted_tables={ "connection_capacity_non_vre": connection_capacity_non_vre, + "reference_year_sequence": _stub_reference_year_sequence(), }, iasr_workbook_version="ignored-by-patch", ) @@ -630,10 +687,15 @@ def test_create_ispypsa_inputs_template_new_format_single_region(csv_str_to_df): assert connection_costs.empty # Custom constraints from PLEXOS are sub-regional export limits with no - # meaningful representation at single_region, so the templater skips them. - # The mock turns a regression in that gate into a clean assertion failure - # instead of a PLEXOS-extract disk read. + # meaningful representation at single_region, so the templater emits them + # header-only instead of templating them. The mock turns a regression in + # that gate into a clean assertion failure instead of a PLEXOS-extract + # disk read. mock_template_custom_constraints.assert_not_called() - assert "custom_constraints" not in result - assert "custom_constraints_lhs" not in result - assert "custom_constraints_rhs" not in result + for table in [ + "custom_constraints", + "custom_constraints_lhs", + "custom_constraints_rhs", + ]: + assert result[table].empty + assert len(result[table].columns) > 0 diff --git a/tests/test_translator/test_constraints.py b/tests/test_translator/test_constraints.py new file mode 100644 index 00000000..0aa7d64b --- /dev/null +++ b/tests/test_translator/test_constraints.py @@ -0,0 +1,327 @@ +import pandas as pd +import pytest + +from ispypsa.translator.constraints import ( + _translate_custom_constraints_from_network_tables, +) +from ispypsa.translator.helpers import _annuitised_investment_costs + +# Annuitised $1/MW at the sample_model_config's wacc (0.06) and annuitisation +# lifetime (25). +_ANNUITY_PER_DOLLAR = _annuitised_investment_costs(1.0, 0.06, 25) + + +def _constraint_tables(csv_str_to_df) -> dict[str, pd.DataFrame]: + """One PLEXOS-derived constraint (SWQLD1) with a link, a generator and a + storage term, timeslice-varying RHS, and a relaxation expansion option. + The sample_model_config's investment periods are 2026 and 2028.""" + tables = {} + tables["custom_constraints"] = csv_str_to_df(""" + constraint_id, direction + SWQLD1, <= + """) + tables["custom_constraints_lhs"] = csv_str_to_df(""" + constraint_id, term_type, variable_name, coefficient, date_from + SWQLD1, link_flow, NSW-QLD, 0.84, + SWQLD1, generator_output, KINGASF1, 0.14, + SWQLD1, storage_output, Q8 Battery - 2h, 0.43, + """) + tables["custom_constraints_rhs"] = csv_str_to_df(""" + constraint_id, timeslice, rhs, date_from + SWQLD1, qld_peak_demand, 3000, + SWQLD1, qld_winter_reference, 3500, + """) + tables["network_expansion_options"] = csv_str_to_df(""" + expansion_id, expansion_type, allowed_expansion, expansion_option + NSW-QLD, forward, 1000, NSW-QLD Option 1 + NSW-QLD, reverse, 900, NSW-QLD Option 1 + SWQLD1, constraint_relaxation, 400, SWQLD1 Option 2 + """) + tables["network_transmission_path_expansion_costs"] = csv_str_to_df(""" + expansion_id, year, cost + NSW-QLD, 2026, 500000 + SWQLD1, 2026, 100000 + """) + return tables + + +def _links(csv_str_to_df) -> pd.DataFrame: + return csv_str_to_df(""" + isp_name, name, p_nom_extendable + NSW-QLD, NSW-QLD_existing, False + NSW-QLD, NSW-QLD_exp_2026, True + """) + + +def test_translate_custom_constraints_rhs(csv_str_to_df, sample_model_config): + ispypsa_tables = _constraint_tables(csv_str_to_df) + + result = _translate_custom_constraints_from_network_tables( + ispypsa_tables, _links(csv_str_to_df), sample_model_config + ) + + expected_rhs = csv_str_to_df(""" + constraint_name, investment_period, timeslice, rhs, constraint_type + SWQLD1, 2026, qld_peak_demand, 3000, <= + SWQLD1, 2026, qld_winter_reference, 3500, <= + SWQLD1, 2028, qld_peak_demand, 3000, <= + SWQLD1, 2028, qld_winter_reference, 3500, <= + NSW-QLD_expansion_limit, , , 1000, <= + SWQLD1_expansion_limit, , , 400, <= + """) + sort_cols = ["constraint_name", "investment_period", "timeslice"] + pd.testing.assert_frame_equal( + result["custom_constraints_rhs"].sort_values(sort_cols).reset_index(drop=True), + expected_rhs.sort_values(sort_cols).reset_index(drop=True), + check_dtype=False, + ) + + +def test_translate_custom_constraints_lhs(csv_str_to_df, sample_model_config): + ispypsa_tables = _constraint_tables(csv_str_to_df) + + result = _translate_custom_constraints_from_network_tables( + ispypsa_tables, _links(csv_str_to_df), sample_model_config + ) + + expected_lhs = csv_str_to_df(""" + constraint_name, investment_period, variable_name, component, attribute, coefficient + SWQLD1, 2026, NSW-QLD_existing, Link, p, 0.84 + SWQLD1, 2026, NSW-QLD_exp_2026, Link, p, 0.84 + SWQLD1, 2028, NSW-QLD_existing, Link, p, 0.84 + SWQLD1, 2028, NSW-QLD_exp_2026, Link, p, 0.84 + SWQLD1, 2026, KINGASF1, Generator, p, 0.14 + SWQLD1, 2028, KINGASF1, Generator, p, 0.14 + SWQLD1, 2026, Q8 Battery - 2h, Storage, p, 0.43 + SWQLD1, 2028, Q8 Battery - 2h, Storage, p, 0.43 + SWQLD1, 2026, SWQLD1_exp_2026, Generator, p_nom, -1.0 + SWQLD1, 2028, SWQLD1_exp_2026, Generator, p_nom, -1.0 + NSW-QLD_expansion_limit, , NSW-QLD_exp_2026, Link, p_nom, 1.0 + SWQLD1_expansion_limit, , SWQLD1_exp_2026, Generator, p_nom, 1.0 + """) + sort_cols = ["constraint_name", "investment_period", "variable_name", "attribute"] + pd.testing.assert_frame_equal( + result["custom_constraints_lhs"].sort_values(sort_cols).reset_index(drop=True), + expected_lhs.sort_values(sort_cols).reset_index(drop=True), + check_dtype=False, + ) + + +def test_translate_custom_constraints_relaxation_generators( + csv_str_to_df, sample_model_config +): + ispypsa_tables = _constraint_tables(csv_str_to_df) + + result = _translate_custom_constraints_from_network_tables( + ispypsa_tables, _links(csv_str_to_df), sample_model_config + ) + + expected_generators = csv_str_to_df(f""" + name, isp_name, bus, p_nom, p_nom_extendable, build_year, lifetime, capital_cost + SWQLD1_exp_2026, SWQLD1, bus_for_custom_constraint_gens, 0.0, True, 2026, inf, {100000 * _ANNUITY_PER_DOLLAR} + """) + pd.testing.assert_frame_equal( + result["custom_constraints_generators"], + expected_generators, + check_dtype=False, + rtol=1e-5, + ) + + +def test_translate_custom_constraints_rez_expansion_disabled( + csv_str_to_df, sample_model_config +): + ispypsa_tables = _constraint_tables(csv_str_to_df) + sample_model_config.network.rez_transmission_expansion = False + + result = _translate_custom_constraints_from_network_tables( + ispypsa_tables, _links(csv_str_to_df), sample_model_config + ) + + expected_generators = csv_str_to_df(""" + name, isp_name, bus, p_nom, p_nom_extendable, build_year, lifetime, capital_cost + """) + pd.testing.assert_frame_equal( + result["custom_constraints_generators"], expected_generators, check_dtype=False + ) + lhs = result["custom_constraints_lhs"] + assert "SWQLD1_exp_2026" not in set(lhs["variable_name"]) + + +def test_date_from_resolved_at_period_starts(csv_str_to_df, sample_model_config): + """A value dated mid-FY2027 (i.e. before FY2028 starts on 2027-07-01) does + not apply in the 2026 period but does in 2028.""" + ispypsa_tables = _constraint_tables(csv_str_to_df) + ispypsa_tables["custom_constraints_rhs"] = csv_str_to_df(""" + constraint_id, timeslice, rhs, date_from + SWQLD1, qld_peak_demand, 3000, + SWQLD1, qld_peak_demand, 2500, 2026-12-01T00:00:00 + """) + + result = _translate_custom_constraints_from_network_tables( + ispypsa_tables, _links(csv_str_to_df), sample_model_config + ) + + rhs = result["custom_constraints_rhs"] + rhs = rhs[rhs["constraint_name"] == "SWQLD1"] + expected = csv_str_to_df(""" + constraint_name, investment_period, timeslice, rhs, constraint_type + SWQLD1, 2026, qld_peak_demand, 3000, <= + SWQLD1, 2028, qld_peak_demand, 2500, <= + """) + pd.testing.assert_frame_equal( + rhs.sort_values("investment_period").reset_index(drop=True), + expected, + check_dtype=False, + ) + + +def test_date_from_after_all_periods_contributes_nothing( + csv_str_to_df, sample_model_config +): + ispypsa_tables = _constraint_tables(csv_str_to_df) + ispypsa_tables["custom_constraints_lhs"] = csv_str_to_df(""" + constraint_id, term_type, variable_name, coefficient, date_from + SWQLD1, generator_output, KINGASF1, 0.14, + SWQLD1, generator_output, LATEGEN, 0.5, 2040-01-01T00:00:00 + """) + + result = _translate_custom_constraints_from_network_tables( + ispypsa_tables, _links(csv_str_to_df), sample_model_config + ) + + assert "LATEGEN" not in set(result["custom_constraints_lhs"]["variable_name"]) + + +def test_equality_direction_becomes_double_equals(csv_str_to_df, sample_model_config): + ispypsa_tables = _constraint_tables(csv_str_to_df) + ispypsa_tables["custom_constraints"] = csv_str_to_df(""" + constraint_id, direction + SWQLD1, = + """) + + result = _translate_custom_constraints_from_network_tables( + ispypsa_tables, _links(csv_str_to_df), sample_model_config + ) + + rhs = result["custom_constraints_rhs"] + assert set(rhs.loc[rhs["constraint_name"] == "SWQLD1", "constraint_type"]) == {"=="} + + +def test_link_terms_not_in_model_dropped_and_logged( + csv_str_to_df, sample_model_config, caplog +): + ispypsa_tables = _constraint_tables(csv_str_to_df) + ispypsa_tables["custom_constraints_lhs"] = csv_str_to_df(""" + constraint_id, term_type, variable_name, coefficient, date_from + SWQLD1, link_flow, TAS-SEV, 0.5, + SWQLD1, generator_output, KINGASF1, 0.14, + """) + + with caplog.at_level("INFO"): + result = _translate_custom_constraints_from_network_tables( + ispypsa_tables, _links(csv_str_to_df), sample_model_config + ) + + assert ( + "Custom constraint link_flow terms dropped (paths not in model): ['TAS-SEV']" + ) in caplog.text + assert "TAS-SEV" not in set(result["custom_constraints_lhs"]["variable_name"]) + + +def test_constraint_with_no_lhs_terms_dropped_and_logged( + csv_str_to_df, sample_model_config, caplog +): + ispypsa_tables = _constraint_tables(csv_str_to_df) + ispypsa_tables["custom_constraints"] = csv_str_to_df(""" + constraint_id, direction + SWQLD1, <= + NQ1, <= + """) + ispypsa_tables["custom_constraints_rhs"] = csv_str_to_df(""" + constraint_id, timeslice, rhs, date_from + SWQLD1, qld_peak_demand, 3000, + NQ1, qld_peak_demand, 2650, + """) + + with caplog.at_level("INFO"): + result = _translate_custom_constraints_from_network_tables( + ispypsa_tables, _links(csv_str_to_df), sample_model_config + ) + + assert ( + "Custom constraints dropped (no LHS terms in model): ['NQ1']" + ) in caplog.text + assert "NQ1" not in set(result["custom_constraints_rhs"]["constraint_name"]) + + +def test_raises_on_rhs_without_direction(csv_str_to_df, sample_model_config): + ispypsa_tables = _constraint_tables(csv_str_to_df) + ispypsa_tables["custom_constraints"] = csv_str_to_df(""" + constraint_id, direction + """) + + with pytest.raises(ValueError, match=r"no direction.*SWQLD1"): + _translate_custom_constraints_from_network_tables( + ispypsa_tables, _links(csv_str_to_df), sample_model_config + ) + + +def test_raises_on_duplicate_rhs_rows(csv_str_to_df, sample_model_config): + ispypsa_tables = _constraint_tables(csv_str_to_df) + ispypsa_tables["custom_constraints_rhs"] = csv_str_to_df(""" + constraint_id, timeslice, rhs, date_from + SWQLD1, qld_peak_demand, 3000, + SWQLD1, qld_peak_demand, 2500, + """) + + with pytest.raises(ValueError, match=r"Duplicate custom constraint RHS.*SWQLD1"): + _translate_custom_constraints_from_network_tables( + ispypsa_tables, _links(csv_str_to_df), sample_model_config + ) + + +def test_empty_custom_constraint_tables(csv_str_to_df, sample_model_config): + """At coarser granularities the custom-constraint tables are header-only; + the expansion-limit constraints for links are still produced.""" + ispypsa_tables = _constraint_tables(csv_str_to_df) + ispypsa_tables["custom_constraints"] = pd.DataFrame( + columns=["constraint_id", "direction"] + ) + ispypsa_tables["custom_constraints_lhs"] = pd.DataFrame( + columns=[ + "constraint_id", + "term_type", + "variable_name", + "coefficient", + "date_from", + ] + ) + ispypsa_tables["custom_constraints_rhs"] = pd.DataFrame( + columns=["constraint_id", "timeslice", "rhs", "date_from"] + ) + ispypsa_tables["network_expansion_options"] = csv_str_to_df(""" + expansion_id, expansion_type, allowed_expansion, expansion_option + NSW-QLD, forward, 1000, Option 1 + NSW-QLD, reverse, 900, Option 1 + """) + + result = _translate_custom_constraints_from_network_tables( + ispypsa_tables, _links(csv_str_to_df), sample_model_config + ) + + expected_lhs = csv_str_to_df(""" + constraint_name, investment_period, variable_name, component, attribute, coefficient + NSW-QLD_expansion_limit, , NSW-QLD_exp_2026, Link, p_nom, 1.0 + """) + pd.testing.assert_frame_equal( + result["custom_constraints_lhs"], expected_lhs, check_dtype=False + ) + + expected_rhs = csv_str_to_df(""" + constraint_name, investment_period, timeslice, rhs, constraint_type + NSW-QLD_expansion_limit, , , 1000, <= + """) + pd.testing.assert_frame_equal( + result["custom_constraints_rhs"], expected_rhs, check_dtype=False + ) diff --git a/tests/test_translator/test_create_pypsa_friendly_inputs_new_format.py b/tests/test_translator/test_create_pypsa_friendly_inputs_new_format.py new file mode 100644 index 00000000..4f1e8a8a --- /dev/null +++ b/tests/test_translator/test_create_pypsa_friendly_inputs_new_format.py @@ -0,0 +1,142 @@ +"""Integration test for the new-format translator orchestrator. + +Verifies wiring only — the expected output keys are present, each output has +the expected column set, and row counts flow through. Detailed content is +covered by the per-module tests (test_network.py, test_constraints.py, +test_timeslice_snapshots.py). +""" + +from unittest.mock import patch + +import pandas as pd +import pytest + +from ispypsa.translator import create_pypsa_friendly_inputs +from ispypsa.translator.create_pypsa_friendly import list_translator_output_files + + +@pytest.fixture +def new_format_ispypsa_tables(csv_str_to_df) -> dict[str, pd.DataFrame]: + tables = {} + tables["network_geography"] = csv_str_to_df(""" + geo_id, geo_type, region_id + NQ, subregion, QLD + CQ, subregion, QLD + Q1, rez, QLD + """) + tables["network_transmission_paths"] = csv_str_to_df(""" + path_id, geo_from, geo_to, carrier + CQ-NQ, CQ, NQ, AC + Q1-NQ, Q1, NQ, AC + """) + tables["network_transmission_path_limits"] = csv_str_to_df(""" + path_id, direction, timeslice, capacity + CQ-NQ, forward, qld_peak_demand, 1200 + CQ-NQ, forward, qld_winter_reference, 1400 + CQ-NQ, reverse, qld_winter_reference, 1910 + Q1-NQ, , , + """) + tables["network_expansion_options"] = csv_str_to_df(""" + expansion_id, expansion_type, allowed_expansion, expansion_option + CQ-NQ, forward, 1000, Option 1 + CQ-NQ, reverse, 900, Option 1 + SWQLD1, constraint_relaxation, 400, Option 2 + """) + tables["network_transmission_path_expansion_costs"] = csv_str_to_df(""" + expansion_id, year, cost + CQ-NQ, 2026, 1000000 + SWQLD1, 2026, 100000 + """) + # sample_model_config's reference_year_cycle is [2024]. + tables["timeslices"] = csv_str_to_df(""" + timeslice_id, reference_year, start_month_day, end_month_day + qld_peak_demand, 2024, 01-13, 01-15 + qld_winter_reference, 2024, 04-01, 10-01 + """) + tables["custom_constraints"] = csv_str_to_df(""" + constraint_id, direction + SWQLD1, <= + """) + tables["custom_constraints_lhs"] = csv_str_to_df(""" + constraint_id, term_type, variable_name, coefficient, date_from + SWQLD1, link_flow, CQ-NQ, 0.84, + SWQLD1, generator_output, KINGASF1, 0.14, + """) + tables["custom_constraints_rhs"] = csv_str_to_df(""" + constraint_id, timeslice, rhs, date_from + SWQLD1, qld_peak_demand, 3000, + """) + return tables + + +def test_create_pypsa_friendly_inputs_new_format( + new_format_ispypsa_tables, sample_model_config +): + with patch( + "ispypsa.translator.create_pypsa_friendly.FEATURE_FLAGS", + {"use_new_table_format": True}, + ): + result = create_pypsa_friendly_inputs( + sample_model_config, new_format_ispypsa_tables + ) + expected_outputs = list_translator_output_files() + + assert set(result.keys()) == set(expected_outputs) + + assert set(result["buses"]["name"]) == {"NQ", "CQ", "Q1"} + + # CQ-NQ existing + expansion (2026 cost row only), Q1-NQ existing. + assert sorted(result["links"]["name"]) == [ + "CQ-NQ_existing", + "CQ-NQ_exp_2026", + "Q1-NQ_existing", + ] + + # Two forward rows and one reverse row for CQ-NQ; the collapsed Q1-NQ row + # contributes nothing. + limits = result["link_timeslice_limits"] + assert set(limits.columns) == {"name", "attribute", "timeslice", "value"} + assert len(limits) == 3 + + # Snapshots span the config's investment periods at 30 min resolution and + # carry weighting columns. + snapshots = result["snapshots"] + assert {"investment_periods", "snapshots", "objective"} <= set(snapshots.columns) + assert set(snapshots["investment_periods"]) == {2026, 2028} + assert len(snapshots) > 0 + + mapping = result["timeslice_snapshots"] + assert set(mapping.columns) == {"timeslice_id", "investment_periods", "snapshots"} + # The patterns are re-sequenced into every model year, so both timeslices + # pick up snapshots. + assert set(mapping["timeslice_id"]) == {"qld_peak_demand", "qld_winter_reference"} + + # SWQLD1 per investment period (2026, 2028) plus the two expansion limits. + rhs = result["custom_constraints_rhs"] + assert set(rhs.columns) == { + "constraint_name", + "investment_period", + "timeslice", + "rhs", + "constraint_type", + } + assert len(rhs) == 4 + + lhs = result["custom_constraints_lhs"] + assert set(lhs.columns) == { + "constraint_name", + "investment_period", + "variable_name", + "component", + "attribute", + "coefficient", + } + # SWQLD1: (2 link names + 1 generator) x 2 periods + relax gen term x 2 + # periods + 2 expansion-limit terms. + assert len(lhs) == 10 + + generators = result["custom_constraints_generators"] + assert sorted(generators["name"]) == ["SWQLD1_exp_2026"] + + assert "generators" not in result + assert "batteries" not in result diff --git a/tests/test_translator/test_network.py b/tests/test_translator/test_network.py new file mode 100644 index 00000000..8c1ae216 --- /dev/null +++ b/tests/test_translator/test_network.py @@ -0,0 +1,297 @@ +import pandas as pd +import pytest + +from ispypsa.translator.helpers import _annuitised_investment_costs +from ispypsa.translator.network import ( + _translate_network_geography_to_buses, + _translate_network_to_links, +) + +# Annuitised $1/MW at the sample_model_config's wacc (0.06) and annuitisation +# lifetime (25) — expansion cost expectations are multiples of this. +_ANNUITY_PER_DOLLAR = _annuitised_investment_costs(1.0, 0.06, 25) + + +def test_translate_network_geography_to_buses_discrete_nodes(csv_str_to_df): + network_geography = csv_str_to_df(""" + geo_id, geo_type, region_id + NQ, subregion, QLD + CNSW, subregion, NSW + Q1, rez, QLD + """) + + result = _translate_network_geography_to_buses(network_geography, "discrete_nodes") + + expected = csv_str_to_df(""" + name + NQ + CNSW + Q1 + """) + pd.testing.assert_frame_equal(result, expected) + + +def test_translate_network_geography_to_buses_attached_to_parent_node(csv_str_to_df): + network_geography = csv_str_to_df(""" + geo_id, geo_type, region_id + NQ, subregion, QLD + Q1, rez, QLD + """) + + result = _translate_network_geography_to_buses( + network_geography, "attached_to_parent_node" + ) + + expected = csv_str_to_df(""" + name + NQ + """) + pd.testing.assert_frame_equal(result, expected) + + +def _network_tables(csv_str_to_df) -> dict[str, pd.DataFrame]: + """New-format network tables with one flow path (CQ-NQ, asymmetric winter + limits, expandable), one REZ path with limits (Q1-NQ) and one collapsed + REZ path with no limit data (N1-CNSW).""" + tables = {} + tables["network_geography"] = csv_str_to_df(""" + geo_id, geo_type, region_id + NQ, subregion, QLD + CQ, subregion, QLD + CNSW, subregion, NSW + Q1, rez, QLD + N1, rez, NSW + """) + tables["network_transmission_paths"] = csv_str_to_df(""" + path_id, geo_from, geo_to, carrier + CQ-NQ, CQ, NQ, AC + Q1-NQ, Q1, NQ, AC + N1-CNSW, N1, CNSW, AC + """) + tables["network_transmission_path_limits"] = csv_str_to_df(""" + path_id, direction, timeslice, capacity + CQ-NQ, forward, qld_peak_demand, 1200 + CQ-NQ, forward, qld_summer_typical, 1300 + CQ-NQ, forward, qld_winter_reference, 1400 + CQ-NQ, reverse, qld_peak_demand, 1440 + CQ-NQ, reverse, qld_summer_typical, 1600 + CQ-NQ, reverse, qld_winter_reference, 1910 + Q1-NQ, forward, qld_winter_reference, 750 + Q1-NQ, reverse, qld_winter_reference, 750 + N1-CNSW, , , + """) + tables["network_expansion_options"] = csv_str_to_df(""" + expansion_id, expansion_type, allowed_expansion, expansion_option + CQ-NQ, forward, 1000, CQ-NQ Option 1 + CQ-NQ, reverse, 900, CQ-NQ Option 1 + Q1-NQ, forward, 500, Q1 Option 1 + Q1-NQ, reverse, 500, Q1 Option 1 + SWQLD1, constraint_relaxation, 400, SWQLD1 Option 2 + """) + # 2025 is outside the sample config's investment periods (2026, 2028) and + # is expected to be filtered out. + tables["network_transmission_path_expansion_costs"] = csv_str_to_df(""" + expansion_id, year, cost + CQ-NQ, 2025, 900000 + CQ-NQ, 2026, 1000000 + Q1-NQ, 2026, 500000 + SWQLD1, 2026, 400000 + """) + return tables + + +def test_translate_network_to_links(csv_str_to_df, sample_model_config): + ispypsa_tables = _network_tables(csv_str_to_df) + + links, link_timeslice_limits = _translate_network_to_links( + ispypsa_tables, sample_model_config + ) + + expected_links = csv_str_to_df(f""" + isp_name, name, carrier, bus0, bus1, p_nom, p_min_pu, build_year, lifetime, capital_cost, p_nom_extendable, isp_type + CQ-NQ, CQ-NQ_existing, AC, CQ, NQ, 1400, -1.364286, 2025, inf, , False, flow_path + Q1-NQ, Q1-NQ_existing, AC, Q1, NQ, 750, -1.0, 2025, inf, , False, rez + N1-CNSW, N1-CNSW_existing,AC, N1, CNSW, 1000000, -1.0, 2025, inf, , False, rez + CQ-NQ, CQ-NQ_exp_2026, AC, CQ, NQ, 0.0, -0.9, 2026, inf, {1000000 * _ANNUITY_PER_DOLLAR}, True, flow_path + Q1-NQ, Q1-NQ_exp_2026, AC, Q1, NQ, 0.0, -1.0, 2026, inf, {500000 * _ANNUITY_PER_DOLLAR}, True, rez + """) + pd.testing.assert_frame_equal( + links.sort_values("name").reset_index(drop=True), + expected_links.sort_values("name").reset_index(drop=True), + check_dtype=False, + rtol=1e-5, + ) + + expected_limits = csv_str_to_df(""" + name, attribute, timeslice, value + CQ-NQ_existing, p_max_pu, qld_peak_demand, 0.857143 + CQ-NQ_existing, p_max_pu, qld_summer_typical, 0.928571 + CQ-NQ_existing, p_max_pu, qld_winter_reference, 1.0 + CQ-NQ_existing, p_min_pu, qld_peak_demand, -1.028571 + CQ-NQ_existing, p_min_pu, qld_summer_typical, -1.142857 + CQ-NQ_existing, p_min_pu, qld_winter_reference, -1.364286 + Q1-NQ_existing, p_max_pu, qld_winter_reference, 1.0 + Q1-NQ_existing, p_min_pu, qld_winter_reference, -1.0 + """) + pd.testing.assert_frame_equal( + link_timeslice_limits.sort_values( + ["name", "attribute", "timeslice"] + ).reset_index(drop=True), + expected_limits.sort_values(["name", "attribute", "timeslice"]).reset_index( + drop=True + ), + rtol=1e-5, + ) + + +def test_translate_network_to_links_expansion_disabled( + csv_str_to_df, sample_model_config +): + ispypsa_tables = _network_tables(csv_str_to_df) + sample_model_config.network.transmission_expansion = False + + links, _ = _translate_network_to_links(ispypsa_tables, sample_model_config) + + # The flow path's expansion link is gone; the REZ path's remains. + assert sorted(links["name"]) == [ + "CQ-NQ_existing", + "N1-CNSW_existing", + "Q1-NQ_existing", + "Q1-NQ_exp_2026", + ] + + +def test_translate_network_to_links_rez_expansion_disabled( + csv_str_to_df, sample_model_config +): + ispypsa_tables = _network_tables(csv_str_to_df) + sample_model_config.network.rez_transmission_expansion = False + + links, _ = _translate_network_to_links(ispypsa_tables, sample_model_config) + + # The REZ path's expansion link is gone; the flow path's remains. + assert sorted(links["name"]) == [ + "CQ-NQ_existing", + "CQ-NQ_exp_2026", + "N1-CNSW_existing", + "Q1-NQ_existing", + ] + + +def test_translate_network_to_links_rezs_attached_to_parent_node( + csv_str_to_df, sample_model_config +): + ispypsa_tables = _network_tables(csv_str_to_df) + sample_model_config.network.nodes.rezs = "attached_to_parent_node" + + links, link_timeslice_limits = _translate_network_to_links( + ispypsa_tables, sample_model_config + ) + + # REZ paths (and their expansion links and timeslice limits) are dropped. + assert sorted(links["name"]) == ["CQ-NQ_existing", "CQ-NQ_exp_2026"] + assert set(link_timeslice_limits["name"]) == {"CQ-NQ_existing"} + + +def test_translate_network_to_links_zero_capacity_parallel_path( + csv_str_to_df, sample_model_config +): + """New parallel corridors arrive as zero-capacity paths: p_nom 0 with a + symmetric p_min_pu (no 0/0 division), and no per-unit timeslice rows.""" + ispypsa_tables = _network_tables(csv_str_to_df) + ispypsa_tables["network_transmission_paths"] = csv_str_to_df(""" + path_id, geo_from, geo_to, carrier + CNSW-SNW, CNSW, SNW, AC + """) + ispypsa_tables["network_transmission_path_limits"] = csv_str_to_df(""" + path_id, direction, timeslice, capacity + CNSW-SNW, forward, nsw_winter_reference, 0 + CNSW-SNW, reverse, nsw_winter_reference, 0 + """) + ispypsa_tables["network_expansion_options"] = pd.DataFrame( + columns=[ + "expansion_id", + "expansion_type", + "allowed_expansion", + "expansion_option", + ] + ) + + links, link_timeslice_limits = _translate_network_to_links( + ispypsa_tables, sample_model_config + ) + + expected_links = csv_str_to_df(""" + isp_name, name, carrier, bus0, bus1, p_nom, p_min_pu, build_year, lifetime, capital_cost, p_nom_extendable, isp_type + CNSW-SNW, CNSW-SNW_existing, AC, CNSW, SNW, 0, -1.0, 2025, inf, , False, flow_path + """) + pd.testing.assert_frame_equal(links, expected_links, check_dtype=False) + + expected_limits = csv_str_to_df(""" + name, attribute, timeslice, value + """) + pd.testing.assert_frame_equal( + link_timeslice_limits, expected_limits, check_dtype=False + ) + + +def test_translate_network_to_links_empty_expansion_tables( + csv_str_to_df, sample_model_config +): + ispypsa_tables = _network_tables(csv_str_to_df) + ispypsa_tables["network_expansion_options"] = pd.DataFrame( + columns=[ + "expansion_id", + "expansion_type", + "allowed_expansion", + "expansion_option", + ] + ) + ispypsa_tables["network_transmission_path_expansion_costs"] = pd.DataFrame( + columns=["expansion_id", "year", "cost"] + ) + + links, _ = _translate_network_to_links(ispypsa_tables, sample_model_config) + + assert sorted(links["name"]) == [ + "CQ-NQ_existing", + "N1-CNSW_existing", + "Q1-NQ_existing", + ] + + +def test_translate_network_to_links_empty_paths(csv_str_to_df, sample_model_config): + ispypsa_tables = _network_tables(csv_str_to_df) + ispypsa_tables["network_transmission_paths"] = pd.DataFrame( + columns=["path_id", "geo_from", "geo_to", "carrier"] + ) + ispypsa_tables["network_transmission_path_limits"] = pd.DataFrame( + columns=["path_id", "direction", "timeslice", "capacity"] + ) + ispypsa_tables["network_expansion_options"] = pd.DataFrame( + columns=[ + "expansion_id", + "expansion_type", + "allowed_expansion", + "expansion_option", + ] + ) + ispypsa_tables["network_transmission_path_expansion_costs"] = pd.DataFrame( + columns=["expansion_id", "year", "cost"] + ) + + links, link_timeslice_limits = _translate_network_to_links( + ispypsa_tables, sample_model_config + ) + + expected_links = csv_str_to_df(""" + isp_name, name, carrier, bus0, bus1, p_nom, p_min_pu, build_year, lifetime, capital_cost, p_nom_extendable, isp_type + """) + pd.testing.assert_frame_equal(links, expected_links, check_dtype=False) + + expected_limits = csv_str_to_df(""" + name, attribute, timeslice, value + """) + pd.testing.assert_frame_equal( + link_timeslice_limits, expected_limits, check_dtype=False + ) diff --git a/tests/test_translator/test_timeslice_snapshots.py b/tests/test_translator/test_timeslice_snapshots.py new file mode 100644 index 00000000..4561d4e2 --- /dev/null +++ b/tests/test_translator/test_timeslice_snapshots.py @@ -0,0 +1,310 @@ +import pandas as pd +import pytest + +from ispypsa.translator.timeslices import ( + _create_timeslice_snapshot_mapping, + _log_referenced_timeslices_without_snapshots, +) + +# sample_model_config: start_year 2026, end_year 2028, year_type fy, +# reference_year_cycle [2024]. + + +def _snapshots(csv_str_to_df, csv_str: str) -> pd.DataFrame: + snapshots = csv_str_to_df(csv_str) + snapshots["snapshots"] = pd.to_datetime(snapshots["snapshots"]) + return snapshots + + +def test_pattern_expanded_into_every_model_year(csv_str_to_df, sample_model_config): + timeslices = csv_str_to_df(""" + timeslice_id, reference_year, start_month_day, end_month_day + nsw_peak_demand, 2024, 01-13, 01-14 + """) + snapshots = _snapshots( + csv_str_to_df, + """ + investment_periods, snapshots + 2026, 2026-01-13 12:00:00 + 2026, 2026-01-14 12:00:00 + 2026, 2027-01-13 12:00:00 + 2028, 2028-01-13 12:00:00 + """, + ) + + result = _create_timeslice_snapshot_mapping( + timeslices, snapshots, sample_model_config + ) + + expected = _snapshots( + csv_str_to_df, + """ + timeslice_id, investment_periods, snapshots + nsw_peak_demand, 2026, 2026-01-13 12:00:00 + nsw_peak_demand, 2026, 2027-01-13 12:00:00 + nsw_peak_demand, 2028, 2028-01-13 12:00:00 + """, + ) + pd.testing.assert_frame_equal( + result.sort_values("snapshots").reset_index(drop=True), + expected.sort_values("snapshots").reset_index(drop=True), + ) + + +def test_cycle_assigns_different_patterns_to_different_model_years( + csv_str_to_df, sample_model_config +): + sample_model_config.temporal.capacity_expansion.reference_year_cycle = [2024, 2018] + timeslices = csv_str_to_df(""" + timeslice_id, reference_year, start_month_day, end_month_day + nsw_peak_demand, 2024, 01-13, 01-14 + nsw_peak_demand, 2018, 02-01, 02-03 + """) + # FY2026 -> 2024, FY2027 -> 2018, FY2028 -> 2024. Snapshots sit on both + # patterns' dates in FY2026 and FY2027; only the assigned pattern tags. + snapshots = _snapshots( + csv_str_to_df, + """ + investment_periods, snapshots + 2026, 2026-01-13 12:00:00 + 2026, 2026-02-02 12:00:00 + 2026, 2027-01-13 12:00:00 + 2026, 2027-02-02 12:00:00 + """, + ) + + result = _create_timeslice_snapshot_mapping( + timeslices, snapshots, sample_model_config + ) + + expected = _snapshots( + csv_str_to_df, + """ + timeslice_id, investment_periods, snapshots + nsw_peak_demand, 2026, 2026-01-13 12:00:00 + nsw_peak_demand, 2026, 2027-02-02 12:00:00 + """, + ) + pd.testing.assert_frame_equal( + result.sort_values("snapshots").reset_index(drop=True), + expected.sort_values("snapshots").reset_index(drop=True), + ) + + +def test_prior_year_winter_window_covers_first_model_year_july( + csv_str_to_df, sample_model_config +): + sample_model_config.temporal.capacity_expansion.reference_year_cycle = [2024, 2018] + # The year before the first model year takes the cycle's last reference + # year (2018), whose winter window [04-01, 10-01) spills into the first + # model year's July-September. 2018's winter ends 10-01 so the October + # snapshot is uncovered (2024's FY2026 winter only starts 2026-04-15). + timeslices = csv_str_to_df(""" + timeslice_id, reference_year, start_month_day, end_month_day + nsw_winter_reference, 2018, 04-01, 10-01 + nsw_winter_reference, 2024, 04-15, 10-15 + """) + snapshots = _snapshots( + csv_str_to_df, + """ + investment_periods, snapshots + 2026, 2025-07-15 12:00:00 + 2026, 2025-10-10 12:00:00 + """, + ) + + result = _create_timeslice_snapshot_mapping( + timeslices, snapshots, sample_model_config + ) + + expected = _snapshots( + csv_str_to_df, + """ + timeslice_id, investment_periods, snapshots + nsw_winter_reference, 2026, 2025-07-15 12:00:00 + """, + ) + pd.testing.assert_frame_equal(result.reset_index(drop=True), expected) + + +def test_end_month_day_wraps_past_new_year(csv_str_to_df, sample_model_config): + timeslices = csv_str_to_df(""" + timeslice_id, reference_year, start_month_day, end_month_day + nsw_summer_typical, 2024, 11-20, 03-20 + """) + snapshots = _snapshots( + csv_str_to_df, + """ + investment_periods, snapshots + 2026, 2025-11-20 12:00:00 + 2026, 2026-03-19 12:00:00 + 2026, 2026-03-21 12:00:00 + """, + ) + + result = _create_timeslice_snapshot_mapping( + timeslices, snapshots, sample_model_config + ) + + expected = _snapshots( + csv_str_to_df, + """ + timeslice_id, investment_periods, snapshots + nsw_summer_typical, 2026, 2025-11-20 12:00:00 + nsw_summer_typical, 2026, 2026-03-19 12:00:00 + """, + ) + pd.testing.assert_frame_equal(result.reset_index(drop=True), expected) + + +def test_leap_day_windows_clamped_in_non_leap_years(csv_str_to_df, sample_model_config): + # Reference year 2024's pattern carries leap-day boundaries. In leap + # FY2028 both windows apply as-is. In non-leap FY2026 the one-day + # [02-28, 02-29) window collapses to empty (the 28th is NOT tagged + # vic_peak_demand) while the [02-29, 03-02) window clamps its start to + # the 28th. + timeslices = csv_str_to_df(""" + timeslice_id, reference_year, start_month_day, end_month_day + vic_peak_demand, 2024, 02-28, 02-29 + nsw_peak_demand, 2024, 02-29, 03-02 + """) + snapshots = _snapshots( + csv_str_to_df, + """ + investment_periods, snapshots + 2026, 2026-02-28 12:00:00 + 2028, 2028-02-28 12:00:00 + 2028, 2028-02-29 12:00:00 + """, + ) + + result = _create_timeslice_snapshot_mapping( + timeslices, snapshots, sample_model_config + ) + + expected = _snapshots( + csv_str_to_df, + """ + timeslice_id, investment_periods, snapshots + nsw_peak_demand, 2026, 2026-02-28 12:00:00 + vic_peak_demand, 2028, 2028-02-28 12:00:00 + nsw_peak_demand, 2028, 2028-02-29 12:00:00 + """, + ) + pd.testing.assert_frame_equal( + result.sort_values(["snapshots", "timeslice_id"]).reset_index(drop=True), + expected.sort_values(["snapshots", "timeslice_id"]).reset_index(drop=True), + ) + + +def test_raises_on_reference_years_without_patterns(csv_str_to_df, sample_model_config): + timeslices = csv_str_to_df(""" + timeslice_id, reference_year, start_month_day, end_month_day + nsw_peak_demand, 2018, 01-13, 01-14 + """) + snapshots = _snapshots( + csv_str_to_df, + """ + investment_periods, snapshots + 2026, 2026-01-13 12:00:00 + """, + ) + + with pytest.raises(ValueError, match=r"no timeslice window patterns: \[2024\]"): + _create_timeslice_snapshot_mapping(timeslices, snapshots, sample_model_config) + + +def test_raises_for_calendar_year_type(csv_str_to_df, sample_model_config): + sample_model_config.temporal.year_type = "calendar" + timeslices = csv_str_to_df(""" + timeslice_id, reference_year, start_month_day, end_month_day + nsw_peak_demand, 2024, 01-13, 01-14 + """) + snapshots = _snapshots( + csv_str_to_df, + """ + investment_periods, snapshots + 2026, 2026-01-13 12:00:00 + """, + ) + + with pytest.raises(NotImplementedError, match="only implemented for fy"): + _create_timeslice_snapshot_mapping(timeslices, snapshots, sample_model_config) + + +def test_empty_timeslices_table(csv_str_to_df, sample_model_config): + timeslices = pd.DataFrame( + columns=["timeslice_id", "reference_year", "start_month_day", "end_month_day"] + ) + snapshots = _snapshots( + csv_str_to_df, + """ + investment_periods, snapshots + 2026, 2026-01-13 12:00:00 + """, + ) + + result = _create_timeslice_snapshot_mapping( + timeslices, snapshots, sample_model_config + ) + + expected = csv_str_to_df(""" + timeslice_id, investment_periods, snapshots + """) + pd.testing.assert_frame_equal(result, expected, check_dtype=False) + + +def test_logs_referenced_timeslices_without_snapshots(csv_str_to_df, caplog): + timeslice_snapshots = _snapshots( + csv_str_to_df, + """ + timeslice_id, investment_periods, snapshots + nsw_peak_demand, 2026, 2026-01-13 12:00:00 + """, + ) + link_timeslice_limits = csv_str_to_df(""" + name, attribute, timeslice, value + CQ-NQ_existing, p_max_pu, nsw_peak_demand, 0.8 + CQ-NQ_existing, p_max_pu, tas_peak_demand, 0.9 + """) + custom_constraints_rhs = csv_str_to_df(""" + constraint_name, investment_period, timeslice, rhs, constraint_type + SWQLD1, 2026, vic_peak_demand, 3000, <= + CQ-NQ_expansion_limit, , , 1000, <= + """) + + with caplog.at_level("WARNING"): + _log_referenced_timeslices_without_snapshots( + timeslice_snapshots, link_timeslice_limits, custom_constraints_rhs + ) + + assert ( + "Timeslices referenced by transmission limits or custom constraints " + "but with no snapshots in the model (these limits and constraints " + "will never apply): ['tas_peak_demand', 'vic_peak_demand']" + ) in caplog.text + + +def test_no_log_when_all_referenced_timeslices_have_snapshots(csv_str_to_df, caplog): + timeslice_snapshots = _snapshots( + csv_str_to_df, + """ + timeslice_id, investment_periods, snapshots + nsw_peak_demand, 2026, 2026-01-13 12:00:00 + """, + ) + link_timeslice_limits = csv_str_to_df(""" + name, attribute, timeslice, value + CQ-NQ_existing, p_max_pu, nsw_peak_demand, 0.8 + """) + custom_constraints_rhs = csv_str_to_df(""" + constraint_name, investment_period, timeslice, rhs, constraint_type + SWQLD1, 2026, nsw_peak_demand, 3000, <= + """) + + with caplog.at_level("WARNING"): + _log_referenced_timeslices_without_snapshots( + timeslice_snapshots, link_timeslice_limits, custom_constraints_rhs + ) + + assert caplog.text == ""