diff --git a/src/ispypsa/iasr_table_caching/local_cache.py b/src/ispypsa/iasr_table_caching/local_cache.py index a1904404..a3f59c94 100644 --- a/src/ispypsa/iasr_table_caching/local_cache.py +++ b/src/ispypsa/iasr_table_caching/local_cache.py @@ -53,6 +53,13 @@ def _build_required_tables(iasr_workbook_version: str) -> list[str]: "efficient_level_of_system_strength_cost", "existing_committed_anticipated_additional_generator_summary", "new_entrants_summary", + "fixed_opex_new_entrants", + "variable_opex_new_entrants", + "lead_time_and_project_life", + "heat_rates_new_entrants", + "gpg_min_stable_level_new_entrants", + "battery_properties", + "pumped_hydro_new_entrant_properties", ] + augmentation else: diff --git a/src/ispypsa/templater/connection_and_build_costs.py b/src/ispypsa/templater/connection_and_build_costs.py index 7f18b0a2..d1eee7c0 100644 --- a/src/ispypsa/templater/connection_and_build_costs.py +++ b/src/ispypsa/templater/connection_and_build_costs.py @@ -9,7 +9,7 @@ from ispypsa.templater.helpers import ( _financial_year_string_to_end_year_int, - _fuzzy_map_to_canonical, + _fuzzy_map_to_allowed_values, _looks_like_financial_year, _where_any_substring_appears, ) @@ -443,7 +443,7 @@ def _canonicalise_non_vre_technologies( # BOTN - Cethana row dropped (listed in ``_NON_VRE_EXCLUDED_TECHNOLOGIES``) """ # NOTE: an only-VRE canonical set (non-empty, but no non-VRE techs) still raises - # via _fuzzy_map_to_canonical when the forecast has non-VRE rows. + # via _fuzzy_map_to_allowed_values when the forecast has non-VRE rows. # Intentional (for now) — kinda related to https://github.com/Open-ISP/ISPyPSA/discussions/103 and the final role(s) of validator. if not canonical_technologies: return pd.DataFrame(columns=df.columns).astype(df.dtypes) @@ -452,7 +452,7 @@ def _canonicalise_non_vre_technologies( df["technology"], _NON_VRE_EXCLUDED_TECHNOLOGIES ) result = df.loc[~excluded].copy() - result["technology"] = _fuzzy_map_to_canonical( + result["technology"] = _fuzzy_map_to_allowed_values( result["technology"], canonical_technologies, task_desc="canonicalising non-VRE connection cost `technology` values", @@ -913,7 +913,7 @@ def _filter_table_by_isp_scenario( NSW 100 150 """ table = table.copy() - table[scenario_col_name] = _fuzzy_map_to_canonical( + table[scenario_col_name] = _fuzzy_map_to_allowed_values( table[scenario_col_name], _ISP_SCENARIOS_NEW, task_desc=f"filtering {table_desc} table by ISP scenario", diff --git a/src/ispypsa/templater/create_template.py b/src/ispypsa/templater/create_template.py index 67d6260c..fe02e483 100644 --- a/src/ispypsa/templater/create_template.py +++ b/src/ispypsa/templater/create_template.py @@ -232,13 +232,9 @@ def create_ispypsa_inputs_template( "connection_capacity_non_vre" ].copy() - # Identity columns only for now - not yet a templater output - generators_new_entrant = _template_generators_new_entrant( - iasr_tables["new_entrants_summary"] - ) - storage_new_entrant = _template_storage_new_entrant( - iasr_tables["new_entrants_summary"] - ) + # Not yet a templater output - fed into connection cost templating below. + generators_new_entrant = _template_generators_new_entrant(iasr_tables) + storage_new_entrant = _template_storage_new_entrant(iasr_tables) template["costs_connection"] = _template_connection_costs( iasr_tables, scenario, diff --git a/src/ispypsa/templater/helpers.py b/src/ispypsa/templater/helpers.py index da9456b5..7590483f 100644 --- a/src/ispypsa/templater/helpers.py +++ b/src/ispypsa/templater/helpers.py @@ -154,7 +154,7 @@ def _best_fuzzy_match(value: str, choices: Iterable[str], threshold: int) -> str return best_choice if best_score >= threshold else None -def _fuzzy_map_to_canonical( +def _fuzzy_map_to_allowed_values( name_series: pd.Series, choices: Iterable[str], task_desc: str, @@ -163,13 +163,13 @@ def _fuzzy_map_to_canonical( """Maps each value in name_series to the closest match in choices (many-to-one). Unlike _fuzzy_match_names, choices are not consumed — multiple input values can - map to the same canonical name. Successful fuzzy corrections are logged at INFO + map to the same allowed value. Successful fuzzy corrections are logged at INFO via _log_fuzzy_match. Any remaining values scoring below ``threshold`` after matching raises a ValueError. Args: name_series: Series of names to map. - choices: Canonical names to match against. + choices: Allowed values to match against. task_desc: Description included in log messages. threshold: Minimum fuzz.ratio score (0–100) to accept a replacement. Default 85. @@ -177,9 +177,8 @@ def _fuzzy_map_to_canonical( Series with values replaced by the closest match where score >= threshold. Raises: - ValueError: if any unmatched values remain - values that aren't able to be - canonicalised (but are expected to be) flag potential inconsistencies across - datasets. + ValueError: if any values in ``name_series`` do not have a match scoring + above ``threshold`` in ``choices``. I/O Examples: name_series: ["Step Change", "Step Chaneg", "Step Change"] @@ -201,15 +200,18 @@ def _fuzzy_map_to_canonical( """ canonical = set(choices) match_dict = { - v: _best_fuzzy_match(v, canonical, threshold) for v in name_series.unique() + name: _best_fuzzy_match(name, canonical, threshold) + for name in name_series.unique() } matched = name_series.map( - lambda v: match_dict[v] if match_dict[v] is not None else v + lambda name: match_dict[name] if match_dict[name] is not None else name ) _log_fuzzy_match(name_series, matched, task_desc) - unmatched = sorted(k for k, v in match_dict.items() if v is None) + unmatched = sorted(name for name, match in match_dict.items() if match is None) if unmatched: - msg = f"Could not fuzzy match to a canonical value whilst {task_desc}: {unmatched}" + msg = ( + f"Could not fuzzy match to an allowed value whilst {task_desc}: {unmatched}" + ) raise ValueError(msg) return matched diff --git a/src/ispypsa/templater/mappings.py b/src/ispypsa/templater/mappings.py index e6f18f8b..e9c10bf6 100644 --- a/src/ispypsa/templater/mappings.py +++ b/src/ispypsa/templater/mappings.py @@ -314,6 +314,113 @@ for opex mapping to rename columns in the table. """ +# - New-format (flag: use_new_table_format=True) per-technology property merge maps - + +# Property entries shared by new entrant generators and storage: each looked up by +# technology from the same IASR table for both. +_COMMON_NEW_ENTRANT_PROPERTY_MAP = { + "fom": dict( + table="fixed_opex_new_entrants", + technology_col="Technology Type", + # NOTE: literal double ")" — parsed directly from the v7.5 IASR workbook + value_col="Base value ($/kW/year))", + scale=1000.0, + ), + "lifetime_technical": dict( + table="lead_time_and_project_life", + technology_col="Technology", + value_col="Technical life (years)", + ), + "lifetime_economic": dict( + table="lead_time_and_project_life", + technology_col="Technology", + value_col="Economic life (years)", + ), + "minimum_stable_level": dict( + table="gpg_min_stable_level_new_entrants", + technology_col="Technology", + value_col="Min Stable Level (% of nameplate)", + ), +} + +_GENERATORS_NEW_ENTRANT_PROPERTY_MAP = { + **_COMMON_NEW_ENTRANT_PROPERTY_MAP, + "vom": dict( + table="variable_opex_new_entrants", + technology_col="Generator", + value_col="Base value", + ), + "heat_rate": dict( + table="heat_rates_new_entrants", + technology_col="Technology", + value_col="Heat rate (GJ/MWh)", + ), +} + +_STORAGE_BATTERY_PROPERTY_MAP = { + "storage_hours": dict( + table="battery_properties", + technology_col="Technology", + value_col="Energy capacity_Hours", + ), + "efficiency_charge": dict( + table="battery_properties", + technology_col="Technology", + value_col="Charge efficiency_%", + ), + "efficiency_discharge": dict( + table="battery_properties", + technology_col="Technology", + value_col="Discharge efficiency_%", + ), + "soc_max": dict( + table="battery_properties", + technology_col="Technology", + value_col="Allowable max state of charge_%", + ), + "soc_min": dict( + table="battery_properties", + technology_col="Technology", + value_col="Allowable min state of charge_%", + ), + "degradation_annual": dict( + table="battery_properties", + technology_col="Technology", + value_col="Annual degradation_%", + ), +} + +# PHES properties are keyed by "Power Station / Technology" — usually the technology, but +# 'BOTN - Cethana' carries its own row. +_STORAGE_PHES_PROPERTY_MAP = { + "storage_hours": dict( + table="pumped_hydro_new_entrant_properties", + technology_col="Power Station / Technology", + value_col="Storage capacity (hours)", + ), + "round_trip_efficiency": dict( + table="pumped_hydro_new_entrant_properties", + technology_col="Power Station / Technology", + value_col="Pumping efficiency (%)", + ), +} +""" +New entrant property columns (keys) mapped to the IASR table and columns that contain +property values and the technology for which the values apply. Consumed by +``ispypsa.templater.new_entrants`` via ``_merge_properties``. + + `table`: IASR table name holding the named property (key) + `technology_col`: column in the IASR table that contains the 'technology' string. + This is the column used to merge on (after mapping to canonical values). + `value_col`: column holding the value to merge in + `scale`: amount by which to multiply the value (default 1.0), used for unit + conversions. e.g. 1000 for $/kW → $/MW + +``_COMMON_...`` holds the entries shared by generators and storage; the generator and +battery maps spread it / sit alongside it, and the storage orchestrator merges it onto +the combined battery + PHES rows. +""" + _ECAA_STORAGE_STATIC_PROPERTY_TABLE_MAP = { "maximum_capacity_mw": dict( table=[f"maximum_capacity_{gen_type}" for gen_type in _ECAA_BATTERY_TYPES], diff --git a/src/ispypsa/templater/new_entrants.py b/src/ispypsa/templater/new_entrants.py index 5f06d2ef..315b0e86 100644 --- a/src/ispypsa/templater/new_entrants.py +++ b/src/ispypsa/templater/new_entrants.py @@ -1,17 +1,23 @@ """Templates the new entrant generator and storage identity tables. -Both tables are currently built from a single IASR input, the ``new_entrants_summary`` -table. This module splits that table into its two subsets and shapes each into the -identity columns of its target schema (see schemas/generators_new_entrant.yaml and -schemas/storage_new_entrant.yaml). - -There are two independent public orchestrators, one per output table, each taking -the full summary. They share the same shape: - 1. Filter the summary to the relevant technology group - 2. Rename the carried-over summary columns to their schema names - 3. Derive geo_id - 4. (Generators only) Derive resource_type - 5. Select the table's group-specific identity columns. +Both tables are built from the IASR ``new_entrants_summary`` table (for identity +columns) plus per-technology property tables. This module splits the summary into +its two subsets and shapes each into the columns of its target schema (see +schemas/generators_new_entrant.yaml and schemas/storage_new_entrant.yaml). + +There are two independent public orchestrators, one per output table. Each one: + 1. Filters the summary to its technology group (generators or storage) + 2. Renames the carried-over summary columns to their schema names + 3. Derives geo_id (REZ ID or sub-region) + 4. (Generators only) Derives resource_type from the VRE code in the IASR ID + 5. Merges in per-technology property values — each a single number looked up by + technology, via _merge_properties (see the property merge maps in mappings.py, + e.g. _GENERATORS_NEW_ENTRANT_PROPERTY_MAP). Generators and storage share a common + set of these (_COMMON_NEW_ENTRANT_PROPERTY_MAP). Storage additionally splits + into battery and pumped-hydro (PHES) rows, which take their storage-specific + properties from different IASR tables, then recombines them before merging + the common properties. + 6. Selects the table's schema columns. """ import logging @@ -19,9 +25,18 @@ import pandas as pd from ispypsa.templater.helpers import ( + _fuzzy_map_to_allowed_values, + _is_battery_row, + _is_pumped_hydro_row, _is_storage_row, _pick_location, ) +from ispypsa.templater.mappings import ( + _COMMON_NEW_ENTRANT_PROPERTY_MAP, + _GENERATORS_NEW_ENTRANT_PROPERTY_MAP, + _STORAGE_BATTERY_PROPERTY_MAP, + _STORAGE_PHES_PROPERTY_MAP, +) _GENERATOR_IDENTITY_COLUMNS = [ "name", @@ -32,6 +47,16 @@ "fuel_price_mapping", ] +# Explicit output order (schema order) +_GENERATOR_PROPERTY_COLUMNS = [ + "fom", + "vom", + "lifetime_technical", + "lifetime_economic", + "heat_rate", + "minimum_stable_level", +] + _STORAGE_IDENTITY_COLUMNS = [ "name", "technology", @@ -39,6 +64,20 @@ "fuel_type", ] +# Explicit output order (schema order) +_STORAGE_PROPERTY_COLUMNS = [ + "storage_hours", + "fom", + "efficiency_charge", + "efficiency_discharge", + "soc_max", + "soc_min", + "minimum_stable_level", + "lifetime_technical", + "lifetime_economic", + "degradation_annual", +] + # Source (IASR new_entrants_summary) column names → schema output column names. _SUMMARY_COLUMN_RENAMES = { "IASR ID / DLT names": "name", @@ -64,71 +103,241 @@ "|".join(sorted(_RESOURCE_QUALITY_CODE_TO_TYPE, key=len, reverse=True)) ) +# BOTN - Cethana is the one named, site-specific PHES project among the generic +# technologies. This mapping assists this special case handling through templating. +_BOTN_CETHANA_DETAILS = { + # common prefix of the two spellings used for this project: + # 'BOTN - Cethana - 20h' and 'BOTN - Cethana' + "name": "BOTN - Cethana", + "technology": "Pumped Hydro (24hrs storage)", +} + # --- public orchestrators --- -# NOTE: partial scope intentional - other columns to be added in next PRs! +# NOTE: partial scope intentional - lcf_* columns added in a later PR! def _template_generators_new_entrant( - new_entrants_summary: pd.DataFrame, + iasr_tables: dict[str, pd.DataFrame], ) -> pd.DataFrame: - """Templates the new entrant generators identity table from the IASR summary. + """Templates the new entrant generators table from the IASR summary and properties. Keeps only generator rows, renames the carried-over summary columns to schema names, derives geo_id (REZ ID or sub-region) and resource_type (from the VRE - resource code in the IASR ID), and returns the identity columns. - - I/O Example: - new_entrants_summary (abbr.): - IASR ID Power Station Technology Type REZ ID Sub-region Fuel type Fuel cost mapping - N3_WH_rez N3_WH_rez Wind N3 NNSW Wind Wind - N3 Battery N3 Battery Battery (2hrs) N3 NNSW Battery Battery - SQ CCGT SQ CCGT CCGT Not Applicable SQ Gas QLD new CCGT - - Returns: - name technology resource_type geo_id fuel_type fuel_price_mapping - N3_WH_rez Wind wind_high N3 Wind Wind - SQ CCGT CCGT SQ Gas QLD new CCGT - + resource code in the IASR ID), merges in the per-technology property columns + (see ``_GENERATORS_NEW_ENTRANT_PROPERTY_MAP``), and returns the identity + + property columns. + + Args: + iasr_tables: IASR tables; uses ``new_entrants_summary`` plus the property + tables named in ``_GENERATORS_NEW_ENTRANT_PROPERTY_MAP``. + + I/O Example (identity columns abbreviated to name/technology): + new_entrants_summary: + IASR ID / DLT names Technology Type ... + N3_WH_rez Wind ... + SQ CCGT CCGT ... + + property tables (one value per technology), e.g. heat_rates_new_entrants: + Technology Heat rate (GJ/MWh) + Wind 0.0 + CCGT 7.25 + + returns (property columns shown; identity columns also present): + name technology fom vom lifetime_technical ... heat_rate minimum_stable_level + N3_WH_rez Wind 18000.0 0.0 40 ... 0.0 0.0 + SQ CCGT CCGT 15303.0 4.18 40 ... 7.25 46.0 """ logging.info("Creating a template for new entrant generators") + new_entrants_summary = iasr_tables["new_entrants_summary"] gens = new_entrants_summary[~_is_storage_row(new_entrants_summary)].copy() gens = gens.rename(columns=_SUMMARY_COLUMN_RENAMES) gens = _set_geo_id(gens) gens = _add_resource_type(gens) - return gens[_GENERATOR_IDENTITY_COLUMNS] + gens = _merge_properties(gens, iasr_tables, _GENERATORS_NEW_ENTRANT_PROPERTY_MAP) + return gens[_GENERATOR_IDENTITY_COLUMNS + _GENERATOR_PROPERTY_COLUMNS] -# NOTE: partial scope intentional - other columns to be added in next PRs! +# NOTE: partial scope intentional - lcf_* columns added in a later PR! def _template_storage_new_entrant( - new_entrants_summary: pd.DataFrame, + iasr_tables: dict[str, pd.DataFrame], ) -> pd.DataFrame: - """Templates the new entrant storage identity table from the IASR summary. - - Keeps only storage rows, renames the carried-over summary columns to schema - names, derives geo_id (REZ ID or sub-region), and returns the identity columns. - - I/O Example: - new_entrants_summary (abbr.): - IASR ID Power Station Technology Type REZ ID Sub-region Fuel type Fuel cost mapping - N3_WH_rez N3_WH_rez Wind N3 NNSW Wind Wind - N3 Battery N3 Battery Battery (2hrs) N3 NNSW Battery Battery - SQ CCGT SQ CCGT CCGT Not Applicable SQ Gas QLD new CCGT - - Returns: - name technology geo_id fuel_type - N3 Battery Battery (2hrs) N3 Battery + """Templates the new entrant storage table from the IASR summary and properties. + + Keeps only storage rows, renames the carried-over summary columns to schema names + and derives geo_id (REZ ID or sub-region). Battery and pumped-hydro (PHES) rows draw + their storage-specific properties from different IASR tables, so each subset is merged + separately and recombined; the shared properties (see + ``_COMMON_NEW_ENTRANT_PROPERTY_MAP``) are then merged onto the combined set. + + Args: + iasr_tables: IASR tables; uses ``new_entrants_summary`` plus the property tables + named in the storage property maps and ``_COMMON_NEW_ENTRANT_PROPERTY_MAP``. + + I/O Example (identity columns abbreviated to name/technology): + new_entrants_summary: + IASR ID / DLT names Technology Type ... + NQ Battery - 2h Battery Storage (2hrs storage) ... + NQ Pumped Hydro-10h Pumped Hydro (10hrs storage) ... + SQ CCGT CCGT ... # generator, dropped + + returns (property columns shown; identity columns also present): + name technology storage_hours efficiency_charge ... + NQ Battery - 2h Battery Storage (2hrs storage) 2.0 92.0 ... + NQ Pumped Hydro-10h Pumped Hydro (10hrs storage) 10.0 87.2 ... """ logging.info("Creating a template for new entrant storage") + new_entrants_summary = iasr_tables["new_entrants_summary"] storage = new_entrants_summary[_is_storage_row(new_entrants_summary)].copy() storage = storage.rename(columns=_SUMMARY_COLUMN_RENAMES) storage = _set_geo_id(storage) - return storage[_STORAGE_IDENTITY_COLUMNS] + batteries = _merge_battery_properties( + storage[_is_battery_row(storage, col_to_check="technology")], iasr_tables + ) + phes = _merge_phes_properties( + storage[_is_pumped_hydro_row(storage, col_to_check="technology")], iasr_tables + ) + storage = pd.concat([batteries, phes], ignore_index=True) + _assert_botn_cethana_values_match_technology(iasr_tables) + storage = _merge_properties(storage, iasr_tables, _COMMON_NEW_ENTRANT_PROPERTY_MAP) + return storage[_STORAGE_IDENTITY_COLUMNS + _STORAGE_PROPERTY_COLUMNS] # --- shared helpers --- +def _merge_properties( + new_entrants: pd.DataFrame, + iasr_tables: dict[str, pd.DataFrame], + property_map: dict[str, dict], + key_col: str = "technology", +) -> pd.DataFrame: + """Merges every property in ``property_map`` onto ``new_entrants``. + + For each (new column, attrs) entry: validates the source IASR table, then looks up + one numeric value per row via ``_merge_technology_keyed_property``. ``key_col`` names + the column whose value is matched against each table's technology key (default + "technology"; PHES passes a name-or-technology key instead — see + ``_phes_lookup_key``). + + I/O Example (property_map = _STORAGE_BATTERY_PROPERTY_MAP, abbreviated): + new_entrants: + name technology + NQ Battery - 2h Battery Storage (2hrs storage) + + returns (adds one column per map key): + name technology storage_hours efficiency_charge ... + NQ Battery - 2h Battery Storage (2hrs storage) 2.0 92.0 ... + """ + for new_col, attrs in property_map.items(): + _assert_property_table_attrs( + table=iasr_tables[attrs["table"]], attrs=attrs, property_name=new_col + ) + new_entrants = _merge_technology_keyed_property( + new_entrants, + iasr_tables[attrs["table"]], + technology_col=attrs["technology_col"], + value_col=attrs["value_col"], + new_col=new_col, + scale=attrs.get("scale", 1.0), + key_col=key_col, + ) + return new_entrants + + +def _merge_technology_keyed_property( + new_entrants: pd.DataFrame, + property_table: pd.DataFrame, + technology_col: str, + value_col: str, + new_col: str, + scale: float = 1.0, + key_col: str = "technology", +) -> pd.DataFrame: + """Adds ``new_col``: one numeric property value per row, looked up by technology. + + The property table holds a single value per technology (``value_col`` keyed by + ``technology_col``). Each ``new_entrants`` row is matched on ``key_col`` (its + technology by default) — fuzzy-matched to the property table's key to manage + typos/capitalisation differences before lookup. The matched values are optionally + rescaled (e.g. ``scale=1000`` to convert $/kW → $/MW). NaN property values are + retained untouched. + + Raises if a key value has no match in the property table. + + I/O Example: + new_entrants: + name technology + A Wind + B CCGT + C Wind + + property_table: + Technology Base value + Wind 2.0 + CCGT 5.0 + + technology_col: "Technology" + value_col: "Base value" + new_col: "fom" + scale: 1000.0 # $/kW -> $/MW scaling + key_col: "technology" + + returns (new col "fom"): + name technology fom + A Wind 2000.0 + B CCGT 5000.0 + C Wind 2000.0 + """ + values_by_technology = pd.to_numeric( + property_table.set_index(technology_col)[value_col], errors="coerce" + ) + values_by_technology *= scale + matched_technology_name = _fuzzy_map_to_allowed_values( + new_entrants[key_col], + values_by_technology.index, + task_desc=f"merging new entrant '{new_col}' by technology", + ) + new_entrants = new_entrants.copy() + new_entrants[new_col] = matched_technology_name.map(values_by_technology) + return new_entrants + + +def _assert_property_table_attrs( + table: pd.DataFrame, attrs: dict[str, str | float], property_name: str +) -> None: + """Asserts that a property table has the required columns and isn't empty. + + Guards against two ways a property table can silently break the downstream + merge: missing required columns, or has no rows. Failing this assertion flags + a change in input IASR table structure that needs to be addressed. + + Args: + table: the property table to validate, e.g. + ``iasr_tables["fixed_opex_new_entrants"]``. + attrs: this property's entry from a property map (e.g. + ``_GENERATORS_NEW_ENTRANT_PROPERTY_MAP["fom"]``), giving the source + table's name plus its ``technology_col``/``value_col``. + property_name: the schema column name being merged (e.g. "fom"); used only + to name the property in the "table is empty" error. + + Raises: + ValueError: if ``table`` is missing ``technology_col`` and/or ``value_col``, + or if ``table`` has no rows. + """ + missing_cols = set([attrs["technology_col"], attrs["value_col"]]) - set( + table.columns + ) + if missing_cols: + raise ValueError( + f"'{attrs['table']}' table missing required columns: {sorted(missing_cols)}" + ) + if table.empty: + raise ValueError( + f"'{attrs['table']}' table is empty - cannot merge property '{property_name}'" + ) + + def _set_geo_id(new_entrants: pd.DataFrame) -> pd.DataFrame: """Adds 'geo_id' column to new_entrants containing REZ ID with Sub-region fallback. @@ -139,6 +348,99 @@ def _set_geo_id(new_entrants: pd.DataFrame) -> pd.DataFrame: return new_entrants +# --- storage-specific helpers --- + + +def _merge_battery_properties( + batteries: pd.DataFrame, iasr_tables: dict[str, pd.DataFrame] +) -> pd.DataFrame: + """Merges the battery-only storage properties onto the battery rows. + + Thin wrapper over ``_merge_properties`` for ``_STORAGE_BATTERY_PROPERTY_MAP`` — every + battery property (storage_hours, charge/discharge efficiency, soc_max/min, annual + degradation) is looked up by technology from the ``battery_properties`` table. + """ + return _merge_properties(batteries, iasr_tables, _STORAGE_BATTERY_PROPERTY_MAP) + + +def _merge_phes_properties( + phes: pd.DataFrame, iasr_tables: dict[str, pd.DataFrame] +) -> pd.DataFrame: + """Merges the pumped-hydro storage properties onto the PHES rows. + + PHES properties are keyed by name-or-technology (see ``_phes_lookup_key``). The table + gives storage_hours and a single round-trip efficiency directly; charge/discharge + efficiency are then derived from it (see ``_derive_phes_symmetric_efficiency``). The + temporary key and round-trip columns are dropped by the orchestrator's final select. + """ + phes = phes.copy() + phes["phes_key"] = _phes_lookup_key(phes) + phes = _merge_properties( + phes, iasr_tables, _STORAGE_PHES_PROPERTY_MAP, key_col="phes_key" + ) + phes = _derive_phes_symmetric_efficiency(phes) + return phes + + +def _phes_lookup_key(phes): + """Each PHES row's key into the pumped-hydro table: its 'name' for declared named + projects, its 'technology' otherwise. + + Currently: only 'BOTN - Cethana' is a declared named project; sometimes + spelled as 'BOTN - Cethana - 20h'. + """ + is_named_project = phes["name"].str.startswith(_BOTN_CETHANA_DETAILS["name"]) + return phes["name"].where(is_named_project, phes["technology"]) + + +def _derive_phes_symmetric_efficiency(phes: pd.DataFrame) -> pd.DataFrame: + """Splits the round-trip 'round_trip_efficiency' (%) into charge and discharge legs. + + The IASR PHES table gives only a single round-trip efficiency. Assuming symmetric + legs, each one-way efficiency is its square root, so e.g. a 76% round trip becomes + ~87.2% charge and ~87.2% discharge (sqrt(0.76) ≈ 0.872). + + I/O Example: + phes: + name round_trip_efficiency + NQ Pumped Hydro-10h 76.0 + + returns (adds the two efficiency columns): + name round_trip_efficiency efficiency_charge efficiency_discharge + NQ Pumped Hydro-10h 76.0 87.18 87.18 + """ + phes = phes.copy() + one_way_efficiency = (phes["round_trip_efficiency"] / 100) ** 0.5 * 100 + phes["efficiency_charge"] = one_way_efficiency + phes["efficiency_discharge"] = one_way_efficiency + return phes + + +def _assert_botn_cethana_values_match_technology(iasr_tables): + """Guard the assumption that BOTN - Cethana's value matches its `technology`'s + (Pumped Hydro (24hrs storage)) in each common property table. + + BOTN is keyed by name in these tables but merged via its technology (see the common + merge in _template_storage_new_entrant), so the two must agree. If a table diverges + them, raise — BOTN then needs explicit name-keyed handling rather than silently taking + the technology value. Both rows must be present: their absence is itself an unexpected + change in the IASR table structure, so a missing-key lookup is left to raise. The + common tables key BOTN by the bare 'BOTN - Cethana'. + """ + name, tech = _BOTN_CETHANA_DETAILS["name"], _BOTN_CETHANA_DETAILS["technology"] + for attrs in _COMMON_NEW_ENTRANT_PROPERTY_MAP.values(): + values = iasr_tables[attrs["table"]].set_index(attrs["technology_col"])[ + attrs["value_col"] + ] + botn_value = pd.to_numeric(values[name], errors="coerce") + tech_value = pd.to_numeric(values[tech], errors="coerce") + if pd.notna(botn_value) and pd.notna(tech_value) and botn_value != tech_value: + raise ValueError( + f"'{name}' diverges from its technology '{tech}' for " + f"'{attrs['value_col']}' in '{attrs['table']}'." + ) + + # --- generator-specific helpers --- diff --git a/tests/test_iasr_table_caching/test_local_cache.py b/tests/test_iasr_table_caching/test_local_cache.py index 5e43df7f..09a736d5 100644 --- a/tests/test_iasr_table_caching/test_local_cache.py +++ b/tests/test_iasr_table_caching/test_local_cache.py @@ -29,6 +29,15 @@ def test_build_required_tables_new_format(): assert "efficient_level_of_system_strength_cost" in result # New entrant generator summary feeds the generators_new_entrant templater assert "new_entrants_summary" in result + # Per-technology property tables merged into the new entrant generators template + assert "fixed_opex_new_entrants" in result + assert "variable_opex_new_entrants" in result + assert "lead_time_and_project_life" in result + assert "heat_rates_new_entrants" in result + assert "gpg_min_stable_level_new_entrants" in result + # Storage property tables merged into the new entrant storage template + assert "battery_properties" in result + assert "pumped_hydro_new_entrant_properties" in result def test_build_required_tables_old_format(): diff --git a/tests/test_templater/test_create_ispypsa_inputs_template.py b/tests/test_templater/test_create_ispypsa_inputs_template.py index 527acdec..bd33415a 100644 --- a/tests/test_templater/test_create_ispypsa_inputs_template.py +++ b/tests/test_templater/test_create_ispypsa_inputs_template.py @@ -57,6 +57,68 @@ def _stub_custom_constraints_tables() -> dict[str, pd.DataFrame]: } +# NOTE: temporary while new entrants not yet fully wired into templater - +# input tables defined here for brevity until then. +def _new_entrant_property_tables(csv_str_to_df) -> dict[str, pd.DataFrame]: + """Per-technology property tables the new_entrant templater merges. + + Covers the generator technologies used across the new-format fixtures below + (Wind, Large scale Solar PV, OCGT (small GT)) so the property merges resolve. + The storage property tables (battery_properties, pumped_hydro_new_entrant_properties) + are included too: the fixtures have no storage rows, so the storage subset is empty, + but the tables must still be present and non-empty for the merge asserts to pass. + Detailed merge behaviour is covered in test_new_entrants.py; here they just + need to be present for the wiring to run. + """ + return { + "fixed_opex_new_entrants": csv_str_to_df(""" + Technology Type, Base value ($/kW/year)), Unit + Wind, 20.0, $ + Large scale Solar PV, 15.0, $ + OCGT (small GT), 17.0, $ + Pumped Hydro (24hrs storage), 78.5, $ + BOTN - Cethana, 78.5, $ + """), + "variable_opex_new_entrants": csv_str_to_df(""" + Generator, Base value + Wind, 0.0 + Large scale Solar PV, 0.0 + OCGT (small GT), 16.4 + """), + "lead_time_and_project_life": csv_str_to_df(""" + Technology, Economic life (years), Technical life (years) + Wind, 5, 30 + Large scale Solar PV, 25, 30 + OCGT (small GT), 25, 40 + Pumped Hydro (24hrs storage), 40, 90 + BOTN - Cethana, 40, 90 + """), + "heat_rates_new_entrants": csv_str_to_df(""" + Technology, Heat rate (GJ/MWh) + Wind, 0.0 + Large scale Solar PV, 0.0 + OCGT (small GT), 10.6 + """), + "gpg_min_stable_level_new_entrants": csv_str_to_df(""" + Technology, Min Stable Level (% of nameplate) + Wind, 0.0 + Large scale Solar PV, 0.0 + OCGT (small GT), 50.0 + Pumped Hydro (24hrs storage), 40.0 + BOTN - Cethana, 40.0 + """), + "battery_properties": csv_str_to_df(""" + Technology, Energy capacity_Hours, Charge efficiency_%, Discharge efficiency_%, Allowable max state of charge_%, Allowable min state of charge_%, Annual degradation_% + Battery storage (2hrs storage), 2.0, 92.0, 92.0, 100, 0, 1.8 + """), + "pumped_hydro_new_entrant_properties": csv_str_to_df(""" + Power Station / Technology, Storage capacity (hours), Pumping efficiency (%) + Pumped Hydro (24hrs storage), 24, 76 + BOTN - Cethana - 20h, 20, 80 + """), + } + + def test_list_templater_output_files_includes_custom_constraints_only_at_sub_regions(): """Custom-constraint tables are declared as task outputs only at sub_regions. @@ -230,11 +292,12 @@ def test_create_ispypsa_inputs_template_new_format(csv_str_to_df): IBR, 10 """) new_entrants_summary = csv_str_to_df(""" - IASR ID / DLT names, Technology Type, Fuel type, Fuel cost mapping, REZ ID, Sub-region - Q1_WH_Far North QLD, Wind, Wind, Wind, Q1, NQ - Q1_SAT_Far North QLD, Large scale Solar PV, Solar, Solar, Q1, NQ - CNSW OCGT Small, OCGT (small GT), Gas, NSW new OCGT, Not Applicable, CNSW - SNW OCGT Small, OCGT (small GT), Gas, NSW new OCGT, Not Applicable, SNW + IASR ID / DLT names, Technology Type, Fuel type, Fuel cost mapping, REZ ID, Sub-region + Q1_WH_Far North QLD, Wind, Wind, Wind, Q1, NQ + Q1_SAT_Far North QLD, Large scale Solar PV, Solar, Solar, Q1, NQ + CNSW OCGT Small, OCGT (small GT), Gas, NSW new OCGT, Not Applicable, CNSW + SNW OCGT Small, OCGT (small GT), Gas, NSW new OCGT, Not Applicable, SNW + BOTN - Cethana - 20h, Pumped Hydro (24hrs storage), Water, Hydro, Not Applicable, TAS """) with ( @@ -266,6 +329,7 @@ def test_create_ispypsa_inputs_template_new_format(csv_str_to_df): "connection_cost_forecast_other": connection_cost_forecast_other, "efficient_level_of_system_strength_cost": efficient_level_of_system_strength_cost, "new_entrants_summary": new_entrants_summary, + **_new_entrant_property_tables(csv_str_to_df), }, # connection_capacity_non_vre is popped out of manually_extracted_tables # into iasr_tables by create_template; supplied so the @@ -424,11 +488,12 @@ def test_create_ispypsa_inputs_template_new_format_nem_regions(csv_str_to_df): IBR, 10 """) new_entrants_summary = csv_str_to_df(""" - IASR ID / DLT names, Technology Type, Fuel type, Fuel cost mapping, REZ ID, Sub-region - Q1_WH_Far North QLD, Wind, Wind, Wind, Q1, NQ - Q1_SAT_Far North QLD, Large scale Solar PV, Solar, Solar, Q1, NQ - CNSW OCGT Small, OCGT (small GT), Gas, NSW new OCGT, Not Applicable, CNSW - SNW OCGT Small, OCGT (small GT), Gas, NSW new OCGT, Not Applicable, SNW + IASR ID / DLT names, Technology Type, Fuel type, Fuel cost mapping, REZ ID, Sub-region + Q1_WH_Far North QLD, Wind, Wind, Wind, Q1, NQ + Q1_SAT_Far North QLD, Large scale Solar PV, Solar, Solar, Q1, NQ + CNSW OCGT Small, OCGT (small GT), Gas, NSW new OCGT, Not Applicable, CNSW + SNW OCGT Small, OCGT (small GT), Gas, NSW new OCGT, Not Applicable, SNW + BOTN - Cethana - 20h, Pumped Hydro (24hrs storage), Water, Hydro, Not Applicable, TAS """) with ( @@ -460,6 +525,7 @@ def test_create_ispypsa_inputs_template_new_format_nem_regions(csv_str_to_df): "connection_cost_forecast_other": connection_cost_forecast_other, "efficient_level_of_system_strength_cost": efficient_level_of_system_strength_cost, "new_entrants_summary": new_entrants_summary, + **_new_entrant_property_tables(csv_str_to_df), }, manually_extracted_tables={ "connection_capacity_non_vre": connection_capacity_non_vre, @@ -579,11 +645,12 @@ def test_create_ispypsa_inputs_template_new_format_single_region(csv_str_to_df): IBR, 10 """) new_entrants_summary = csv_str_to_df(""" - IASR ID / DLT names, Technology Type, Fuel type, Fuel cost mapping, REZ ID, Sub-region - Q1_WH_Far North QLD, Wind, Wind, Wind, Q1, NQ - Q1_SAT_Far North QLD, Large scale Solar PV, Solar, Solar, Q1, NQ - CNSW OCGT Small, OCGT (small GT), Gas, NSW new OCGT, Not Applicable, CNSW - SNW OCGT Small, OCGT (small GT), Gas, NSW new OCGT, Not Applicable, SNW + IASR ID / DLT names, Technology Type, Fuel type, Fuel cost mapping, REZ ID, Sub-region + Q1_WH_Far North QLD, Wind, Wind, Wind, Q1, NQ + Q1_SAT_Far North QLD, Large scale Solar PV, Solar, Solar, Q1, NQ + CNSW OCGT Small, OCGT (small GT), Gas, NSW new OCGT, Not Applicable, CNSW + SNW OCGT Small, OCGT (small GT), Gas, NSW new OCGT, Not Applicable, SNW + BOTN - Cethana - 20h, Pumped Hydro (24hrs storage), Water, Hydro, Not Applicable, TAS """) with ( @@ -613,6 +680,7 @@ def test_create_ispypsa_inputs_template_new_format_single_region(csv_str_to_df): "connection_cost_forecast_other": connection_cost_forecast_other, "efficient_level_of_system_strength_cost": efficient_level_of_system_strength_cost, "new_entrants_summary": new_entrants_summary, + **_new_entrant_property_tables(csv_str_to_df), }, manually_extracted_tables={ "connection_capacity_non_vre": connection_capacity_non_vre, diff --git a/tests/test_templater/test_fuzzy_matching.py b/tests/test_templater/test_fuzzy_matching.py index bd7d53b3..2c3c4eb5 100644 --- a/tests/test_templater/test_fuzzy_matching.py +++ b/tests/test_templater/test_fuzzy_matching.py @@ -3,7 +3,7 @@ from ispypsa.templater.helpers import ( _best_fuzzy_match, - _fuzzy_map_to_canonical, + _fuzzy_map_to_allowed_values, _fuzzy_match_names, ) @@ -244,13 +244,13 @@ def test_best_fuzzy_match_picks_highest_scoring_choice(): assert result == "Step Change" -# ── _fuzzy_map_to_canonical ────────────────────────────────────────────────── +# ── _fuzzy_map_to_allowed_values ────────────────────────────────────────────────── -def test_fuzzy_map_to_canonical_corrects_typo_and_logs_info(caplog): +def test_fuzzy_map_to_allowed_values_corrects_typo_and_logs_info(caplog): series = pd.Series(["Step Chaneg"]) with caplog.at_level("INFO"): - result = _fuzzy_map_to_canonical( + result = _fuzzy_map_to_allowed_values( series, ["Step Change", "Slower Growth"], "testing correction" ) expected = pd.Series(["Step Change"]) @@ -260,10 +260,10 @@ def test_fuzzy_map_to_canonical_corrects_typo_and_logs_info(caplog): ) in caplog.text -def test_fuzzy_map_to_canonical_exact_match_no_info_log(caplog): +def test_fuzzy_map_to_allowed_values_exact_match_no_info_log(caplog): series = pd.Series(["Step Change"]) with caplog.at_level("INFO"): - result = _fuzzy_map_to_canonical( + result = _fuzzy_map_to_allowed_values( series, ["Step Change", "Slower Growth"], "testing exact" ) expected = pd.Series(["Step Change"]) @@ -271,18 +271,18 @@ def test_fuzzy_map_to_canonical_exact_match_no_info_log(caplog): assert "matched to" not in caplog.text -def test_fuzzy_map_to_canonical_unmatched_raises_error(): +def test_fuzzy_map_to_allowed_values_unmatched_raises_error(): series = pd.Series(["Wind", "the sun", "Solar PV"]) - msg = r"Could not fuzzy match to a canonical value whilst testing unmatched: \['the sun'\]" + msg = r"Could not fuzzy match to an allowed value whilst testing unmatched: \['the sun'\]" with pytest.raises(ValueError, match=msg): - _fuzzy_map_to_canonical( + _fuzzy_map_to_allowed_values( series, ["Wind", "Solar PV"], "testing unmatched", threshold=85 ) -def test_fuzzy_map_to_canonical_empty_series(): +def test_fuzzy_map_to_allowed_values_empty_series(): series = pd.Series([], dtype=object) - result = _fuzzy_map_to_canonical( + result = _fuzzy_map_to_allowed_values( series, ["Step Change", "Slower Growth"], "testing empty" ) expected = pd.Series([], dtype=object) diff --git a/tests/test_templater/test_new_entrants.py b/tests/test_templater/test_new_entrants.py index 2b32d386..018dbb4a 100644 --- a/tests/test_templater/test_new_entrants.py +++ b/tests/test_templater/test_new_entrants.py @@ -3,8 +3,17 @@ from ispypsa.templater.new_entrants import ( _GENERATOR_IDENTITY_COLUMNS, + _GENERATOR_PROPERTY_COLUMNS, _STORAGE_IDENTITY_COLUMNS, + _STORAGE_PROPERTY_COLUMNS, _add_resource_type, + _assert_botn_cethana_values_match_technology, + _assert_property_table_attrs, + _derive_phes_symmetric_efficiency, + _merge_battery_properties, + _merge_phes_properties, + _merge_technology_keyed_property, + _phes_lookup_key, _set_geo_id, _template_generators_new_entrant, _template_storage_new_entrant, @@ -15,45 +24,413 @@ def test_template_generators_new_entrant(csv_str_to_df): # Wiring check only (per-helper behaviour is covered below): storage is dropped, - # the identity columns are produced, and one row per surviving generating unit - # is returned. Detailed content is covered by the per-helper tests. + # and the identity + property columns are produced, one row per generating unit. + # Detailed content is covered by the per-helper tests. new_entrants_summary = csv_str_to_df(""" - IASR ID / DLT names, Technology Type, Fuel type, Fuel cost mapping, REZ ID, Sub-region - Q1_WH_Far North QLD, Wind, Wind, Wind, Q1, NQ - Q1_WM_Far North QLD, Wind, Wind, Wind, Q1, NQ - Q1_SAT_Far North QLD, Large scale Solar PV, Solar, Solar, Q1, NQ - NQ OCGT Small, OCGT (small GT), Gas, QLD new OCGT, Not Applicable, NQ - NQ SAT - Distributed Resources, Distributed Resources Solar, Solar, Solar, Not Applicable, NQ - NQ Battery 2hrs, Battery Storage (2hrs storage), Battery, Battery, Not Applicable, NQ + IASR ID / DLT names, Technology Type, Fuel type, Fuel cost mapping, REZ ID, Sub-region + Q1_WH_Far North QLD, Wind, Wind, Wind, Q1, NQ + NQ OCGT Small, OCGT (small GT), Gas, QLD new OCGT, Not Applicable, NQ + NQ Battery 2hrs, Battery Storage (2hrs storage), Battery, Battery, Not Applicable, NQ """) + iasr_tables = { + "new_entrants_summary": new_entrants_summary, + "fixed_opex_new_entrants": csv_str_to_df(""" + Technology Type, Base value ($/kW/year)), Unit + Wind, 20.0, $ + OCGT (small GT), 17.0, $ + """), + "variable_opex_new_entrants": csv_str_to_df(""" + Generator, Base value + Wind, 0.0 + OCGT (small GT), 16.4 + """), + "lead_time_and_project_life": csv_str_to_df(""" + Technology, Economic life (years), Technical life (years) + Wind, 25, 30 + OCGT (small GT), 25, 40 + """), + "heat_rates_new_entrants": csv_str_to_df(""" + Technology, Heat rate (GJ/MWh) + Wind, 0.0 + OCGT (small GT), 10.6 + """), + "gpg_min_stable_level_new_entrants": csv_str_to_df(""" + Technology, Min Stable Level (% of nameplate) + Wind, 0.0 + OCGT (small GT), 50.0 + """), + } - result = _template_generators_new_entrant(new_entrants_summary) + result = _template_generators_new_entrant(iasr_tables) - # storage row dropped -> 5 of 6 rows survive; identity columns produced in order - assert list(result.columns) == _GENERATOR_IDENTITY_COLUMNS - assert len(result) == 5 + # storage row dropped -> 2 gen rows; identity + property columns produced in order + assert ( + list(result.columns) + == _GENERATOR_IDENTITY_COLUMNS + _GENERATOR_PROPERTY_COLUMNS + ) + assert len(result) == 2 + + +def _storage_property_tables(csv_str_to_df): + """The IASR property tables the storage orchestrator merges from (shared by tests).""" + return { + "battery_properties": csv_str_to_df(""" + Technology, Energy capacity_Hours, Charge efficiency_%, Discharge efficiency_%, Allowable max state of charge_%, Allowable min state of charge_%, Annual degradation_% + Battery storage (2hrs storage), 2.0, 92.0, 92.0, 100, 0, 1.8 + Distributed Resources Batteries, 2.0, 92.0, 92.0, 100, 0, 1.8 + """), + "pumped_hydro_new_entrant_properties": csv_str_to_df(""" + Power Station / Technology, Storage capacity (hours), Pumping efficiency (%) + Pumped Hydro (24hrs storage), 24, 76 + BOTN - Cethana - 20h, 20, 80 + """), + "fixed_opex_new_entrants": csv_str_to_df(""" + Technology Type, Base value ($/kW/year)), Unit + Battery storage (2hrs storage), 13.5, $ + Distributed Resources Batteries, 13.5, $ + Pumped Hydro (24hrs storage), 50.0, $ + BOTN - Cethana, 50.0, $ + """), + "lead_time_and_project_life": csv_str_to_df(""" + Technology, Economic life (years), Technical life (years) + Battery storage (2hrs storage), 20, 20 + Distributed Resources Batteries, 20, 20 + Pumped Hydro (24hrs storage), 40, 90 + BOTN - Cethana, 40, 90 + """), + "gpg_min_stable_level_new_entrants": csv_str_to_df(""" + Technology, Min Stable Level (% of nameplate) + Battery storage (2hrs storage), 0.0 + Distributed Resources Batteries, 0.0 + Pumped Hydro (24hrs storage), 40.0 + BOTN - Cethana, 40.0 + """), + } def test_template_storage_new_entrant(csv_str_to_df): # Wiring check only (per-helper behaviour is covered below): generators are - # dropped, the identity columns are produced, and one row per surviving storage - # unit is returned. Detailed content is covered by the per-helper tests. + # dropped, identity + property columns are produced, and one row per surviving + # storage unit (battery + PHES) is returned. Detailed content is covered by the + # per-helper tests. new_entrants_summary = csv_str_to_df(""" IASR ID / DLT names, Technology Type, Fuel type, Fuel cost mapping, REZ ID, Sub-region Q1_WH_Far North QLD, Wind, Wind, Wind, Q1, NQ NQ OCGT Small, OCGT (small GT), Gas, QLD new OCGT, Not Applicable, NQ NQ Battery 2hrs, Battery Storage (2hrs storage), Battery, Battery, N3, NQ NQ Battery - Distributed, Distributed Resources Batteries, Battery, Battery, Not Applicable, NQ - Snowy PH 24hr, Pumped Hydro (24hrs storage), Water, Water, Not Applicable, NQ + BOTN - Cethana - 20h, Pumped Hydro (24hrs storage), Water, Hydro, Not Applicable, NQ """) + iasr_tables = { + "new_entrants_summary": new_entrants_summary, + **_storage_property_tables(csv_str_to_df), + } - result = _template_storage_new_entrant(new_entrants_summary) + result = _template_storage_new_entrant(iasr_tables) - # generator rows dropped -> 3 of 5 rows survive; identity columns produced in order - assert list(result.columns) == _STORAGE_IDENTITY_COLUMNS + # generator rows dropped -> 3 of 5 rows survive; identity + property columns in order + assert list(result.columns) == _STORAGE_IDENTITY_COLUMNS + _STORAGE_PROPERTY_COLUMNS assert len(result) == 3 +# --- _assert_property_table_attrs --- + + +def test_assert_property_table_attrs_valid_table(csv_str_to_df): + # Table has both required columns and at least one row - no error raised. + table = csv_str_to_df(""" + Technology, Base value + Wind, 20.0 + """) + attrs = { + "table": "fixed_opex_new_entrants", + "technology_col": "Technology", + "value_col": "Base value", + "scale": 1000.0, + } + # should not raise + _assert_property_table_attrs(table, attrs, "fom") + + +def test_assert_property_table_attrs_raises_missing_columns(csv_str_to_df): + # Table is missing technology_col - raised message names the source table, + # and the missing columns - including the 'Base Value' column with different + # capitalisation to expected 'Base value'. + table = csv_str_to_df(""" + Base Value + 20.0 + """) + attrs = { + "table": "heat_rate", + "technology_col": "Technology", + "value_col": "Base value", + "scale": 1.0, + } + + with pytest.raises( + ValueError, + match=r"'heat_rate' table missing required columns: \['Base value', 'Technology'\]", + ): + _assert_property_table_attrs(table, attrs, "fom") + + +def test_assert_property_table_attrs_raises_empty_table(): + # Table has both required columns but no rows - raise + table = pd.DataFrame(columns=["Technology", "Base value"]) + attrs = { + "table": "fixed_opex_new_entrants", + "technology_col": "Technology", + "value_col": "Base value", + "scale": 1000.0, + } + + with pytest.raises( + ValueError, + match="'fixed_opex_new_entrants' table is empty - cannot merge property 'fom'", + ): + _assert_property_table_attrs(table, attrs, "fom") + + +# --- _merge_technology_property --- + + +def test_merge_technology_property(csv_str_to_df): + # Looks up one value per technology, fuzzy-matching spelling differences + # and applying the scale. Duplicate technologies all receive the value; + # canon spelling is kept. NaN property values are retained untouched. + new_entrants = csv_str_to_df(""" + name, technology + A, Wind + B, Battery Storage (2hrs storage) + C, Wind + D, CCGT + """) + property_table = csv_str_to_df(""" + Technology, Base value + Wind, 20.0 + Battery storage (2hrs storage), 17.0 + CCGT, NaN + """) + + result = _merge_technology_keyed_property( + new_entrants, + property_table, + "Technology", + "Base value", + "fom", + scale=1000.0, + key_col="technology", + ) + + expected = csv_str_to_df(""" + name, technology, fom + A, Wind, 20000.0 + B, Battery Storage (2hrs storage), 17000.0 + C, Wind, 20000.0 + D, CCGT, NaN + """) + pd.testing.assert_frame_equal(result, expected) + + +def test_merge_technology_property_empty_new_entrants(csv_str_to_df): + # Test that empty new_entrants df returns with new column added (empty) + new_entrants = csv_str_to_df(""" + name, technology + """) + property_table = csv_str_to_df(""" + Technology, Base value + Wind, 20.0 + Battery storage (2hrs storage), 17.0 + CCGT, NaN + """) + + expected_result = pd.DataFrame(columns=["name", "technology", "fom"]) + result = _merge_technology_keyed_property( + new_entrants, property_table, "Technology", "Base value", "fom", 1000.0 + ) + pd.testing.assert_frame_equal( + result, + expected_result, + check_dtype=False, + ) + + +# --- _merge_battery_properties --- + + +def test_merge_battery_properties(csv_str_to_df): + # Every battery property is looked up by technology from battery_properties, with the + # summary's capitalisation ("Battery Storage") fuzzy-matched to the table's spelling. + batteries = csv_str_to_df(""" + name, technology + NQ Battery - 2h, Battery Storage (2hrs storage) + """) + iasr_tables = { + "battery_properties": csv_str_to_df(""" + Technology, Energy capacity_Hours, Charge efficiency_%, Discharge efficiency_%, Allowable max state of charge_%, Allowable min state of charge_%, Annual degradation_% + Battery storage (2hrs storage), 2.0, 92.0, 92.0, 100, 0, 1.8 + """) + } + + result = _merge_battery_properties(batteries, iasr_tables) + + expected = csv_str_to_df(""" + name, technology, storage_hours, efficiency_charge, efficiency_discharge, soc_max, soc_min, degradation_annual + NQ Battery - 2h, Battery Storage (2hrs storage), 2.0, 92.0, 92.0, 100.0, 0.0, 1.8 + """) + pd.testing.assert_frame_equal(result, expected) + + +def test_merge_battery_properties_empty(csv_str_to_df): + # No battery rows -> returns empty with the battery property columns added. + batteries = pd.DataFrame(columns=["name", "technology"]) + iasr_tables = { + "battery_properties": csv_str_to_df(""" + Technology, Energy capacity_Hours, Charge efficiency_%, Discharge efficiency_%, Allowable max state of charge_%, Allowable min state of charge_%, Annual degradation_% + Battery storage (2hrs storage), 2.0, 92.0, 92.0, 100, 0, 1.8 + """) + } + + result = _merge_battery_properties(batteries, iasr_tables) + + expected = csv_str_to_df(""" + name, technology, storage_hours, efficiency_charge, efficiency_discharge, soc_max, soc_min, degradation_annual + """) + pd.testing.assert_frame_equal(result, expected, check_dtype=False) + + +# --- _merge_phes_properties / _phes_lookup_key / _derive_phes_symmetric_efficiency --- + + +def test_merge_phes_properties(csv_str_to_df): + # storage_hours is merged by name-or-technology key; charge/discharge efficiency are + # derived from the single round-trip pumping efficiency. + phes = csv_str_to_df(""" + name, technology + NQ Pumped Hydro - 24h, Pumped Hydro (24hrs storage) + BOTN - Cethana - 20h, Pumped Hydro (24hrs storage) + """) + iasr_tables = { + "pumped_hydro_new_entrant_properties": csv_str_to_df(""" + Power Station / Technology, Storage capacity (hours), Pumping efficiency (%) + Pumped Hydro (24hrs storage), 24, 64 + BOTN - Cethana - 20h, 20, 81 + """) + } + + result = _merge_phes_properties(phes, iasr_tables) + + expected = csv_str_to_df(""" + name, technology, phes_key, storage_hours, round_trip_efficiency, efficiency_charge, efficiency_discharge + NQ Pumped Hydro - 24h, Pumped Hydro (24hrs storage), Pumped Hydro (24hrs storage), 24.0, 64.0, 80.0, 80.0 + BOTN - Cethana - 20h, Pumped Hydro (24hrs storage), BOTN - Cethana - 20h, 20.0, 81.0, 90.0, 90.0 + """) + pd.testing.assert_frame_equal(result, expected, check_exact=False, rtol=1e-6) + + +def test_phes_lookup_key(csv_str_to_df): + # Named stations (present in ``_BOTN_CETHANA_DETAILS``) keys on name - only + # 'BOTN - Cethana - 20h' at v7.5; generic rows key on technology. + phes = csv_str_to_df(""" + name, technology + NQ Pumped Hydro - 24h, Pumped Hydro (24hrs storage) + BOTN - Cethana - 20h, Pumped Hydro (24hrs storage) + """) + + result = _phes_lookup_key(phes) + + expected = pd.Series( + ["Pumped Hydro (24hrs storage)", "BOTN - Cethana - 20h"], name="name" + ) + pd.testing.assert_series_equal(result, expected) + + +def test_derive_phes_symmetric_efficiency(csv_str_to_df): + # A single round-trip efficiency becomes equal charge and discharge legs, each its + # square root: sqrt(0.91) ≈ 0.9 -> 90.0%. + phes = csv_str_to_df(""" + name, round_trip_efficiency + NQ Pumped Hydro - 24h, 81.0 + """) + + result = _derive_phes_symmetric_efficiency(phes) + + expected = csv_str_to_df(""" + name, round_trip_efficiency, efficiency_charge, efficiency_discharge + NQ Pumped Hydro - 24h, 81.0, 90.0, 90.0 + """) + pd.testing.assert_frame_equal(result, expected, check_exact=False, rtol=1e-6) + + +def test_merge_phes_properties_empty(csv_str_to_df): + # No PHES rows -> returns empty with the PHES-derived columns added. + phes = pd.DataFrame(columns=["name", "technology"]) + iasr_tables = { + "pumped_hydro_new_entrant_properties": csv_str_to_df(""" + Power Station / Technology, Storage capacity (hours), Pumping efficiency (%) + Pumped Hydro (24hrs storage), 24, 76 + """) + } + + result = _merge_phes_properties(phes, iasr_tables) + + expected = csv_str_to_df(""" + name, technology, phes_key, storage_hours, round_trip_efficiency, efficiency_charge, efficiency_discharge + """) + pd.testing.assert_frame_equal(result, expected, check_dtype=False) + + +# --- _assert_botn_cethana_values_match_technology --- + + +def _botn_common_tables(csv_str_to_df, botn_fom="75.0"): + """The common tables the BOTN guard checks, with BOTN keyed by its bare name + alongside its 'Pumped Hydro (24hrs storage)' archetype. ``botn_fom`` lets a test make + BOTN's fom diverge from the matching technology's (75.0).""" + return { + "fixed_opex_new_entrants": csv_str_to_df(f""" + Technology Type, Base value ($/kW/year)), Unit + Pumped Hydro (24hrs storage), 75.0, $ + BOTN - Cethana, {botn_fom}, $ + """), + "lead_time_and_project_life": csv_str_to_df(""" + Technology, Economic life (years), Technical life (years) + Pumped Hydro (24hrs storage), 40, 90 + BOTN - Cethana, 40, 90 + """), + "gpg_min_stable_level_new_entrants": csv_str_to_df(""" + Technology, Min Stable Level (% of nameplate) + Pumped Hydro (24hrs storage), 40.0 + BOTN - Cethana, 40.0 + """), + } + + +def test_assert_botn_cethana_values_match_technology_passes_when_matching( + csv_str_to_df, +): + # BOTN's values equal its technology's across every common table -> no raise. + iasr_tables = _botn_common_tables(csv_str_to_df) + _assert_botn_cethana_values_match_technology(iasr_tables) + + +def test_assert_botn_cethana_values_match_technology_raises_on_divergence( + csv_str_to_df, +): + # BOTN's fom no longer matches the technology's -> raise, naming the property and table. + iasr_tables = _botn_common_tables(csv_str_to_df, botn_fom="99.0") + + with pytest.raises( + ValueError, + match=( + r"'BOTN - Cethana' diverges from its technology " + r"'Pumped Hydro \(24hrs storage\)' for 'Base value \(\$\/kW\/year\)\)' " + r"in 'fixed_opex_new_entrants'" + ), + ): + _assert_botn_cethana_values_match_technology(iasr_tables) + + # --- _set_geo_id --- diff --git a/tests/test_workbook_table_cache/7.5/battery_properties.csv b/tests/test_workbook_table_cache/7.5/battery_properties.csv new file mode 100644 index 00000000..b756737a --- /dev/null +++ b/tests/test_workbook_table_cache/7.5/battery_properties.csv @@ -0,0 +1,9 @@ +Technology,Maximum power_MW,Energy capacity_Hours,Charge efficiency_%,Discharge efficiency_%,Allowable max state of charge_%,Allowable min state of charge_%,Round trip efficiency_%,Annual degradation_% +Battery storage (1hr storage),1,1.0,92.0,92.0,100,0,84,1.8 +Battery storage (2hrs storage),1,2.0,92.0,92.0,100,0,84,1.8 +Battery storage (4hrs storage),1,4.0,92.5,92.5,100,0,85,1.4 +Battery storage (8hrs storage),1,8.0,93.0,93.0,100,0,85,1.2 +Compressed air,1,8.0,81.0,81.0,100,0,0,0.0 +Distributed Resources Batteries,1,2.0,92.0,92.0,100,0,84,1.8 +VPP (aggregated ESS) - Coordinated CER,1,2.2,92.2,92.2,85,0,85,1.6 +VPP (aggregated ESS) - V2G,1,2.2,92.0,92.0,85,0,85,1.6 diff --git a/tests/test_workbook_table_cache/7.5/fixed_opex_new_entrants.csv b/tests/test_workbook_table_cache/7.5/fixed_opex_new_entrants.csv new file mode 100644 index 00000000..e1434b06 --- /dev/null +++ b/tests/test_workbook_table_cache/7.5/fixed_opex_new_entrants.csv @@ -0,0 +1,22 @@ +Technology Type,Base value ($/kW/year)),Unit +OCGT (small GT),17.6858344,$ +OCGT (large GT),14.3234078,$ +CCGT,15.3030124,$ +CCGT with CCS,22.9545186,$ +Biomass,187.4537555,$ +Large scale Solar PV,12.2196,$ +Solar Thermal (16hrs storage),124.3211921,$ +Battery storage (1hr storage),9.1647,$ +Battery storage (2hrs storage),13.54339,$ +Battery storage (4hrs storage),21.58796,$ +Battery storage (8hrs storage),37.98259,$ +Wind,28.512400000000003,$ +Wind - offshore (fixed),177.7676859,$ +Wind - offshore (floating),251.8225351,$ +Pumped Hydro (10hrs storage),96.7385,$ +Pumped Hydro (24hrs storage),74.84505,$ +Pumped Hydro (48hrs storage),85.5372,$ +Distributed Resources Solar,12.2196,$ +Distributed Resources Batteries,11.85301,$ +Alkaline Electrolyser,50.10036,$ +BOTN - Cethana,74.84505,$ diff --git a/tests/test_workbook_table_cache/7.5/gpg_min_stable_level_new_entrants.csv b/tests/test_workbook_table_cache/7.5/gpg_min_stable_level_new_entrants.csv new file mode 100644 index 00000000..368db22f --- /dev/null +++ b/tests/test_workbook_table_cache/7.5/gpg_min_stable_level_new_entrants.csv @@ -0,0 +1,22 @@ +Technology,Min Stable Level (% of nameplate) +OCGT (small GT),50.0 +OCGT (large GT),50.0 +CCGT,46.0 +CCGT with CCS,46.0 +Biomass,40.0 +Large scale Solar PV,0.0 +Solar Thermal (16hrs storage),20.0 +Battery storage (1hr storage),0.0 +Battery storage (2hrs storage),0.0 +Battery storage (4hrs storage),0.0 +Battery storage (8hrs storage),0.0 +Wind,0.0 +Wind - offshore (fixed),0.0 +Wind - offshore (floating),0.0 +Pumped Hydro (10hrs storage),40.0 +Pumped Hydro (24hrs storage),40.0 +Pumped Hydro (48hrs storage),40.0 +Distributed Resources Solar,0.0 +Distributed Resources Batteries,0.0 +Alkaline Electrolyser,10.0 +BOTN - Cethana,40.0 diff --git a/tests/test_workbook_table_cache/7.5/heat_rates_new_entrants.csv b/tests/test_workbook_table_cache/7.5/heat_rates_new_entrants.csv new file mode 100644 index 00000000..b97811e0 --- /dev/null +++ b/tests/test_workbook_table_cache/7.5/heat_rates_new_entrants.csv @@ -0,0 +1,22 @@ +Technology,Heat rate (GJ/MWh) +OCGT (small GT),10.648730964467 +OCGT (large GT),10.9312436804853 +CCGT,7.24923076923076 +CCGT with CCS,9.03964757709251 +Biomass,17.5350593311758 +Battery Storage (1hr storage),0.0 +Battery Storage (2hrs storage),0.0 +Battery Storage (4hrs storage),0.0 +Battery Storage (8hrs storage),0.0 +Large scale Solar PV,0.0 +Pumped Hydro (10hrs storage),0.0 +Pumped Hydro (24hrs storage),0.0 +Pumped Hydro (48hrs storage),0.0 +Solar Thermal (16hrs storage),0.0 +Wind,0.0 +Wind - offshore (fixed),0.0 +Wind - offshore (floating),0.0 +Distributed Resources Solar,0.0 +Distributed Resources Batteries,0.0 +Alkaline Electrolyser,0.0 +BOTN - Cethana,0.0 diff --git a/tests/test_workbook_table_cache/7.5/lead_time_and_project_life.csv b/tests/test_workbook_table_cache/7.5/lead_time_and_project_life.csv new file mode 100644 index 00000000..b4106b8b --- /dev/null +++ b/tests/test_workbook_table_cache/7.5/lead_time_and_project_life.csv @@ -0,0 +1,22 @@ +Technology,"Lead time for development (years)1, 2",Lead time (years),Construction time (years),"Total lead time (years)4,",Economic life (years),Technical life (years) +OCGT (small GT),2.0,2.0,1.5,6,25,40 +OCGT (large GT),2.0,2.0,1.11538461538461,5,25,40 +CCGT,2.5,1.75,1.5,6,25,40 +CCGT with CCS,3.0,1.75,2.0,7,25,40 +Biomass,3.0,1.75,1.25,6,30,50 +Large scale Solar PV,0.0,1.0,0.5,2,30,30 +Solar Thermal (16hrs storage),2.5,1.75,1.75,6,30,40 +Battery storage (1hr storage),0.0,1.0,0.846153846153846,2,20,20 +Battery storage (2hrs storage),0.0,1.2,1.0,2,20,20 +Battery storage (4hrs storage),0.0,1.4,1.15384615384615,3,20,20 +Battery storage (8hrs storage),0.0,1.6,1.3076923076923,3,20,20 +Wind,0.0,1.0,1.73076923076923,3,25,30 +Wind - offshore (fixed),7.0,3.0,3.0,13,25,30 +Wind - offshore (floating),7.0,3.0,3.0,13,25,30 +Pumped Hydro (10hrs storage),4.0,2.0,2.40384615384615,8,40,90 +Pumped Hydro (24hrs storage),4.0,2.0,3.84615384615384,10,40,90 +Pumped Hydro (48hrs storage),4.0,2.0,3.84615384615384,10,40,90 +Distributed Resources Solar,0.0,0.5,0.5,1,30,30 +Distributed Resources Batteries,0.0,0.5,1.0,2,20,20 +Alkaline Electrolyser,2.5,2.0,0.5,5,10,25 +BOTN - Cethana,4.0,2.0,3.84615384615384,10,40,90 diff --git a/tests/test_workbook_table_cache/7.5/pumped_hydro_new_entrant_properties.csv b/tests/test_workbook_table_cache/7.5/pumped_hydro_new_entrant_properties.csv new file mode 100644 index 00000000..29bc1ee1 --- /dev/null +++ b/tests/test_workbook_table_cache/7.5/pumped_hydro_new_entrant_properties.csv @@ -0,0 +1,5 @@ +Power Station / Technology,Installed capacity (MW),Storage capacity (hours),Pumping efficiency (%) +BOTN - Cethana - 20h,750,20,80 +Pumped Hydro (10hrs storage),1,10,76 +Pumped Hydro (24hrs storage),1,24,76 +Pumped Hydro (48hrs storage),1,48,76 diff --git a/tests/test_workbook_table_cache/7.5/variable_opex_new_entrants.csv b/tests/test_workbook_table_cache/7.5/variable_opex_new_entrants.csv new file mode 100644 index 00000000..d5542e7c --- /dev/null +++ b/tests/test_workbook_table_cache/7.5/variable_opex_new_entrants.csv @@ -0,0 +1,22 @@ +Generator,Base value +OCGT (small GT),16.39463 +OCGT (large GT),8.24823 +CCGT,4.17503 +CCGT with CCS,8.1464 +Biomass,10.93654 +Large scale Solar PV,0.0 +Solar Thermal (16hrs storage),0.0 +Battery storage (1hr storage),0.0 +Battery storage (2hrs storage),0.0 +Battery storage (4hrs storage),0.0 +Battery storage (8hrs storage),0.0 +Wind,0.0 +Wind - offshore (fixed),0.0 +Wind - offshore (floating),0.0 +Pumped Hydro (10hrs storage),0.0 +Pumped Hydro (24hrs storage),0.0 +Pumped Hydro (48hrs storage),0.0 +Distributed Resources Solar,0.0 +Distributed Resources Batteries,0.0 +Alkaline Electrolyser,0.0 +BOTN - Cethana,0.0