From 340826881c1f0673c430a80598e4b88cf92a577f Mon Sep 17 00:00:00 2001 From: nick-gorman Date: Thu, 11 Jun 2026 12:23:56 +1000 Subject: [PATCH 1/2] Add schemas for the custom-constraint tables The templater has emitted custom_constraints, custom_constraints_lhs and custom_constraints_rhs since the PLEXOS templater landed, but the tables had no schemas, so user-supplied versions of them had no documented contract. The schemas pin the vocabulary the translator now relies on: direction senses (<=, >=, =), term types (including load, which the 7.5 PLEXOS extract emits for Node Load Coefficient terms), the timeslice cross-reference, and date_from semantics (the value active at the start of an investment period applies for that whole period). Also fixes network_expansion_options cross-referencing the non-existent constraints_rhs table. Co-Authored-By: Claude Fable 5 --- .../schemas/custom_constraints.yaml | 28 ++++++++ .../schemas/custom_constraints_lhs.yaml | 64 +++++++++++++++++++ .../schemas/custom_constraints_rhs.yaml | 51 +++++++++++++++ .../schemas/network_expansion_options.yaml | 4 +- 4 files changed, 145 insertions(+), 2 deletions(-) create mode 100644 src/ispypsa/validation/schemas/custom_constraints.yaml create mode 100644 src/ispypsa/validation/schemas/custom_constraints_lhs.yaml create mode 100644 src/ispypsa/validation/schemas/custom_constraints_rhs.yaml diff --git a/src/ispypsa/validation/schemas/custom_constraints.yaml b/src/ispypsa/validation/schemas/custom_constraints.yaml new file mode 100644 index 00000000..a06fd4ae --- /dev/null +++ b/src/ispypsa/validation/schemas/custom_constraints.yaml @@ -0,0 +1,28 @@ +table: custom_constraints +required: false +unique: + - [constraint_id] +description: > + Catalogue of custom (group) constraints and the sense of each constraint's + inequality. + + Each constraint's terms live in `custom_constraints_lhs` and its limit + values in `custom_constraints_rhs`; this table defines the constraint + itself. + + Source tables: + - PLEXOS model extract (`constraints.csv`), via the custom-constraints + templater. + + If absent: + No custom constraints are applied to the model. +columns: + constraint_id: + type: string + required: true + description: Unique identifier for the constraint (e.g. SWQLD1). + direction: + type: string + required: true + allowed_values: ["<=", ">=", "="] + description: Sense of the constraint inequality, applied as lhs rhs. diff --git a/src/ispypsa/validation/schemas/custom_constraints_lhs.yaml b/src/ispypsa/validation/schemas/custom_constraints_lhs.yaml new file mode 100644 index 00000000..7d35fc38 --- /dev/null +++ b/src/ispypsa/validation/schemas/custom_constraints_lhs.yaml @@ -0,0 +1,64 @@ +table: custom_constraints_lhs +required: false +custom_validation: + - name: no_duplicate_constraint_term_date + description: > + No two rows may have the same constraint_id, term_type, variable_name, + and date_from value. For NaN date_from rows, at most one row is + permitted per (constraint_id, term_type, variable_name) combination. +description: > + Left-hand-side terms of the custom constraints: one row per (constraint, + model variable) coefficient. + + Source tables: + - PLEXOS model extract (`lhs_terms.csv`), via the custom-constraints + templater. + + Source notes: + variable_name uses IASR IDs for generators and batteries and path_ids for + links. Generator and battery terms may reference units that are not in the + model (e.g. before generator templating lands, or units outside a filtered + region); such terms are skipped with a log line when constraints are + applied. + + If absent: + Constraints have no terms, so no custom constraints bind. +columns: + constraint_id: + type: string + required: true + allowed_values_from: + - custom_constraints: constraint_id + description: Constraint the term belongs to. + term_type: + type: string + required: true + allowed_values: [generator_output, storage_output, link_flow, load] + description: > + Model variable type the term applies to: a generator's dispatch, a + battery's discharge, a transmission path's flow, or a node's load + consumption (load terms are not yet modelled and are skipped, with a + log line, when constraints are applied). + variable_name: + type: string + required: true + description: > + Name of the component the term's variable belongs to: an IASR ID for + generator_output and storage_output terms, a path_id for link_flow + terms. + coefficient: + type: float + required: true + description: Coefficient the variable is multiplied by in the constraint. + date_from: + type: date + required: false + format: "%Y-%m-%dT%H:%M:%S" + description: > + Date this row's coefficient takes effect. For each investment period, + the coefficient active at the period's start applies for the whole + period. + + If absent (or empty): + The coefficient applies from the start of the model horizon (until + another row's date_from supersedes it). diff --git a/src/ispypsa/validation/schemas/custom_constraints_rhs.yaml b/src/ispypsa/validation/schemas/custom_constraints_rhs.yaml new file mode 100644 index 00000000..84965e98 --- /dev/null +++ b/src/ispypsa/validation/schemas/custom_constraints_rhs.yaml @@ -0,0 +1,51 @@ +table: custom_constraints_rhs +required: false +custom_validation: + - name: no_duplicate_constraint_timeslice_date + description: > + No two rows may have the same constraint_id, timeslice, and date_from + value. NaN timeslice and NaN date_from each count as a single value for + this purpose (e.g. at most one all-NaN row per constraint_id). +description: > + Right-hand-side limit values of the custom constraints, by timeslice. + + Source tables: + - PLEXOS model extract (`rhs_values.csv`), via the custom-constraints + templater. + + If absent: + Constraints have no limit values, so no custom constraints bind. +columns: + constraint_id: + type: string + required: true + allowed_values_from: + - custom_constraints: constraint_id + description: Constraint the limit value belongs to. + timeslice: + type: string + required: false + allowed_values_from: + - timeslices: timeslice_id + description: > + Demand condition the limit applies to: the constraint binds only at + snapshots inside the timeslice's active windows. A constraint with RHS + rows for only some timeslices does not bind outside them. + + If absent (or empty): + The limit applies at all snapshots. + rhs: + type: float + required: true + description: Limit value the constraint's LHS is compared against. + date_from: + type: date + required: false + format: "%Y-%m-%dT%H:%M:%S" + description: > + Date this row's limit takes effect. For each investment period, the + limit active at the period's start applies for the whole period. + + If absent (or empty): + The limit applies from the start of the model horizon (until another + row's date_from supersedes it). diff --git a/src/ispypsa/validation/schemas/network_expansion_options.yaml b/src/ispypsa/validation/schemas/network_expansion_options.yaml index 6e434267..02108c82 100644 --- a/src/ispypsa/validation/schemas/network_expansion_options.yaml +++ b/src/ispypsa/validation/schemas/network_expansion_options.yaml @@ -29,11 +29,11 @@ columns: required: true allowed_values_from: - network_transmission_paths: path_id - - constraints_rhs: constraint_id + - custom_constraints_rhs: constraint_id description: > Identifier for the expandable network element. Maps to path_id in network_transmission_paths for physical paths, or to constraint_id - in constraints_rhs for group constraints. + in custom_constraints_rhs for group constraints. expansion_type: type: string required: true From 5294cc89ffdc1de4a6b9faee5e7dae34913b3247 Mon Sep 17 00:00:00 2001 From: nick-gorman Date: Fri, 19 Jun 2026 10:04:11 +1000 Subject: [PATCH 2/2] Use unique: for nullable keys; make NaN timeslice a fallback The custom_validation rules on these schemas only existed to express "uniqueness with NaN counted as a single value". Pinning down unique: to treat NaN as equal (Open-ISP/ISPyPSA#122) lets a plain unique: carry that guarantee, so the three rules become unique: declarations. For network_transmission_path_limits and custom_constraints_rhs this also revises what a NaN timeslice means: rather than a static limit applying to all conditions (mutually exclusive with per-timeslice rows), it is now a fallback applying only where no per-timeslice row covers, so both forms may co-exist. Documented under the timeslice column. Also adds generator_capacity to the lhs term_type allowed values. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../schemas/custom_constraints_lhs.yaml | 16 ++++------- .../schemas/custom_constraints_rhs.yaml | 18 ++++++------ .../network_transmission_path_limits.yaml | 28 ++++++++++--------- 3 files changed, 30 insertions(+), 32 deletions(-) diff --git a/src/ispypsa/validation/schemas/custom_constraints_lhs.yaml b/src/ispypsa/validation/schemas/custom_constraints_lhs.yaml index 7d35fc38..29a1f15e 100644 --- a/src/ispypsa/validation/schemas/custom_constraints_lhs.yaml +++ b/src/ispypsa/validation/schemas/custom_constraints_lhs.yaml @@ -1,11 +1,7 @@ table: custom_constraints_lhs required: false -custom_validation: - - name: no_duplicate_constraint_term_date - description: > - No two rows may have the same constraint_id, term_type, variable_name, - and date_from value. For NaN date_from rows, at most one row is - permitted per (constraint_id, term_type, variable_name) combination. +unique: + - [constraint_id, term_type, variable_name, date_from] description: > Left-hand-side terms of the custom constraints: one row per (constraint, model variable) coefficient. @@ -33,12 +29,12 @@ columns: term_type: type: string required: true - allowed_values: [generator_output, storage_output, link_flow, load] + allowed_values: [generator_output, generator_capacity, storage_output, link_flow, load] description: > Model variable type the term applies to: a generator's dispatch, a - battery's discharge, a transmission path's flow, or a node's load - consumption (load terms are not yet modelled and are skipped, with a - log line, when constraints are applied). + generator's installed capacity, a battery's discharge, a transmission + path's flow, or a node's load consumption (load terms are not yet + modelled and are skipped, with a log line, when constraints are applied). variable_name: type: string required: true diff --git a/src/ispypsa/validation/schemas/custom_constraints_rhs.yaml b/src/ispypsa/validation/schemas/custom_constraints_rhs.yaml index 84965e98..d08aa20d 100644 --- a/src/ispypsa/validation/schemas/custom_constraints_rhs.yaml +++ b/src/ispypsa/validation/schemas/custom_constraints_rhs.yaml @@ -1,11 +1,7 @@ table: custom_constraints_rhs required: false -custom_validation: - - name: no_duplicate_constraint_timeslice_date - description: > - No two rows may have the same constraint_id, timeslice, and date_from - value. NaN timeslice and NaN date_from each count as a single value for - this purpose (e.g. at most one all-NaN row per constraint_id). +unique: + - [constraint_id, timeslice, date_from] description: > Right-hand-side limit values of the custom constraints, by timeslice. @@ -29,11 +25,15 @@ columns: - timeslices: timeslice_id description: > Demand condition the limit applies to: the constraint binds only at - snapshots inside the timeslice's active windows. A constraint with RHS - rows for only some timeslices does not bind outside them. + snapshots inside the timeslice's active windows. If absent (or empty): - The limit applies at all snapshots. + The limit is a fallback for this constraint_id, applying at snapshots not + covered by any non-NaN timeslice row for the same constraint_id. A + constraint may carry both a no-timeslice row and per-timeslice rows; where + a per-timeslice row covers a snapshot its limit takes precedence, and the + no-timeslice limit fills the remainder. A constraint with per-timeslice + rows but no no-timeslice row does not bind outside those timeslices. rhs: type: float required: true diff --git a/src/ispypsa/validation/schemas/network_transmission_path_limits.yaml b/src/ispypsa/validation/schemas/network_transmission_path_limits.yaml index bdeb2d5e..977a8bbf 100644 --- a/src/ispypsa/validation/schemas/network_transmission_path_limits.yaml +++ b/src/ispypsa/validation/schemas/network_transmission_path_limits.yaml @@ -1,20 +1,17 @@ table: network_transmission_path_limits required: false -custom_validation: - - name: no_duplicate_path_direction_timeslice - description: > - No two rows may have the same path_id, direction, and timeslice value. - For NaN timeslice rows, at most one row is permitted per path_id and - direction combination. +unique: + - [path_id, direction, timeslice] description: > Transmission capacity limits for each path, by direction and optionally by demand condition (timeslice). - A limit with no timeslice applies to all demand conditions; a limit with a - timeslice applies to that condition only. The schema permits either form for - any path. In the default IASR data, flow-path limits vary by demand condition - while REZ-to-subregion connection limits are static, but that reflects the - source tables rather than a schema rule. + A limit with a timeslice applies to that demand condition only; a limit with + no timeslice is a fallback for the conditions no per-timeslice limit covers + (see the timeslice column). A path may carry either or both. In the default + IASR data, flow-path limits vary by demand condition while REZ-to-subregion + connection limits are static, but that reflects the source tables rather than + a schema rule. Source tables: - `flow_path_transfer_capability` @@ -40,10 +37,15 @@ columns: allowed_values_from: - timeslices: timeslice_id description: > - Demand condition the limit applies to. + Demand condition the limit applies to: the limit binds only at snapshots + inside the timeslice's active windows. If absent (or empty): - Capacity is treated as a static limit applying to all conditions. + The limit is a fallback for this path_id and direction, applying at + snapshots not covered by any non-NaN timeslice row for the same path_id + and direction. A path_id and direction may carry both a no-timeslice row + and per-timeslice rows; where a per-timeslice row covers a snapshot its + limit takes precedence, and the no-timeslice limit fills the remainder. capacity: type: float required: false