From 8a0c3cef8e18758f7c019f37f3869c5d6f4b48a6 Mon Sep 17 00:00:00 2001 From: jakeross Date: Mon, 24 Aug 2026 11:31:30 -0700 Subject: [PATCH] feat(ogc): publish a well water-column layer A well's construction record says how deep the hole goes and its water-level record says how far down the water sits, but nothing in the catalogue published the difference -- the standing column of water in the well, which is the number that says whether a well still holds usable water. ogc_well_water_column (and its unfiltered ogc_internal_ twin) carries one row per water well with the same well and location fields the water_wells layer publishes, plus four derived depths in feet: the well depth less the latest depth to water, less the mean depth to water, less the shallowest reading on record, and less the deepest. Shallowest water leaves the most in the well and deepest the least, hence maximum/minimum naming the column rather than the reading. Readings are manual groundwater-level observations taken below ground surface as value minus measuring-point height, the same convention water_well_summary and latest_depth_to_water_wells already use, so the three layers cannot disagree about what a depth to water is. Continuous transducer readings are not counted. Negative results are clamped to zero: a reading deeper than the recorded well depth is a contradiction between two records rather than a well holding negative water. The contradiction itself stays visible in water_well_summary, which publishes the raw shallowest and deepest readings beside the well depth. Wells with no recorded depth, or no usable reading, are left out -- all four columns would be NULL and the row would say nothing. Materialized, because every column but the latest aggregates a well's whole reading history. The nightly pg_cron job refreshes every matview in the public schema by name, so the two are picked up with no change to the schedule, and both carry a unique index on id so a manual refresh can run CONCURRENTLY. Co-Authored-By: Claude Opus 5 --- ...1f2a3b4_add_well_water_column_ogc_views.py | 212 ++++++++++++++++++ core/ogc-field-descriptions.yml | 34 +++ core/pygeoapi-config-internal.yml | 42 ++++ core/pygeoapi-config.yml | 42 ++++ tests/test_ogc.py | 105 +++++++++ 5 files changed, 435 insertions(+) create mode 100644 alembic/versions/c9d0e1f2a3b4_add_well_water_column_ogc_views.py diff --git a/alembic/versions/c9d0e1f2a3b4_add_well_water_column_ogc_views.py b/alembic/versions/c9d0e1f2a3b4_add_well_water_column_ogc_views.py new file mode 100644 index 00000000..315146f3 --- /dev/null +++ b/alembic/versions/c9d0e1f2a3b4_add_well_water_column_ogc_views.py @@ -0,0 +1,212 @@ +"""add the well water-column OGC layer + +A water well's construction record says how deep the hole goes; its +groundwater-level record says how far down the water sits. The difference -- +the standing column of water inside the well -- is the number that says whether +a well still has usable water in it, and nothing in the catalogue published it. + +This creates ogc_well_water_column (public) and ogc_internal_well_water_column +(unfiltered), one row per water well, carrying the same well and location +fields the water_wells layer publishes plus four derived depths, all in feet: + + water_column_latest well depth minus the most recent depth to water + water_column_average well depth minus the mean depth to water + water_column_maximum well depth minus the shallowest depth to water + water_column_minimum well depth minus the deepest depth to water + +Shallowest water gives the largest column and deepest water the smallest, hence +the maximum/minimum naming: these are the extremes of the water column itself, +not of the readings behind them. + +Readings are manual groundwater-level observations, taken below ground surface +as (value - measuring_point_height) with a missing height treated as ground +level -- the same convention as ogc_water_well_summary and +ogc_latest_depth_to_water_wells, so the three layers cannot disagree about what +a depth to water is. Continuous transducer readings are not included. + +Negative results are clamped to zero. A reading deeper than the recorded well +depth is a contradiction between two records rather than a well holding +negative water, and the clamp keeps consumers from having to special-case it; +the contradiction itself stays visible in water_well_summary, which publishes +the raw shallowest and deepest readings next to the well depth. + +Rows are restricted to wells that have both a well depth and at least one +usable reading -- without either, all four columns would be NULL and the row +would say nothing. + +Materialized, because every column but the latest one aggregates a well's +entire reading history. The nightly pg_cron job (b6c7d8e9f0a1) refreshes every +matview in the public schema by name, so these two are picked up with no change +to the schedule. Both carry a unique index on id so the refresh can also be run +CONCURRENTLY by hand (`oco refresh-matview --concurrently`). + +Revision ID: c9d0e1f2a3b4 +Revises: b8c9d0e1f2a3 +Create Date: 2026-08-24 00:00:00.000000 +""" + +import re +from typing import Sequence, Union + +from alembic import op +from sqlalchemy import inspect, text + +# revision identifiers, used by Alembic. +revision: str = "c9d0e1f2a3b4" +down_revision: Union[str, Sequence[str], None] = "b8c9d0e1f2a3" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +REQUIRED_TABLES = { + "thing", + "location", + "location_thing_association", + "observation", + "sample", + "field_activity", + "field_event", +} + +LATEST_LOCATION_CTE = """ +SELECT DISTINCT ON (lta.thing_id) + lta.thing_id, + lta.location_id, + lta.effective_start +FROM location_thing_association AS lta +WHERE lta.effective_end IS NULL +ORDER BY lta.thing_id, lta.effective_start DESC +""".strip() + +VIEWS = [ + ("ogc_well_water_column", True), + ("ogc_internal_well_water_column", False), +] + + +def _safe_relation_name(name: str) -> str: + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", name): + raise ValueError(f"Unsafe relation name: {name!r}") + return name + + +def _check_required_tables() -> None: + bind = op.get_bind() + inspector = inspect(bind) + existing_tables = set(inspector.get_table_names(schema="public")) + missing = REQUIRED_TABLES - existing_tables + if missing: + raise RuntimeError( + "Cannot create the well water-column views. " + f"Missing required tables: {', '.join(sorted(missing))}" + ) + + +def _create_well_water_column_view(view_name: str, public_only: bool) -> str: + safe_view_name = _safe_relation_name(view_name) + release_filter = " AND t.release_status = 'public'" if public_only else "" + observation_release_filter = ( + "\n AND o.release_status = 'public'" if public_only else "" + ) + return f""" + CREATE MATERIALIZED VIEW {safe_view_name} AS + WITH latest_location AS ( +{LATEST_LOCATION_CTE} + ), + wl_obs AS ( + SELECT + fe.thing_id, + o.id AS observation_id, + o.observation_datetime, + (o.value - COALESCE(o.measuring_point_height, 0)) AS water_level + FROM observation AS o + JOIN sample AS s ON s.id = o.sample_id + JOIN field_activity AS fa ON fa.id = s.field_activity_id + JOIN field_event AS fe ON fe.id = fa.field_event_id + JOIN thing AS t ON t.id = fe.thing_id + WHERE + t.thing_type = 'water well' + AND fa.activity_type = 'groundwater level' + AND o.value IS NOT NULL + AND o.observation_datetime IS NOT NULL{observation_release_filter} + ), + wl_agg AS ( + SELECT + w.thing_id, + AVG(w.water_level) AS avg_water_level, + MIN(w.water_level) AS min_water_level, + MAX(w.water_level) AS max_water_level + FROM wl_obs AS w + GROUP BY w.thing_id + ), + wl_last AS ( + SELECT + ranked.thing_id, + ranked.water_level AS last_water_level + FROM ( + SELECT + w.thing_id, + w.water_level, + ROW_NUMBER() OVER ( + PARTITION BY w.thing_id + ORDER BY w.observation_datetime DESC, w.observation_id DESC + ) AS rn + FROM wl_obs AS w + ) AS ranked + WHERE ranked.rn = 1 + ) + SELECT + t.id AS id, + t.name, + t.first_visit_date, + t.nma_pk_welldata, + t.well_depth, + t.hole_depth, + t.well_casing_diameter, + t.well_casing_depth, + t.well_completion_date, + t.well_driller_name, + t.well_construction_method, + t.well_pump_type, + t.well_pump_depth, + t.formation_completion_code, + t.nma_formation_zone, + t.release_status, + GREATEST(t.well_depth - wl.last_water_level, 0) AS water_column_latest, + GREATEST(t.well_depth - wa.avg_water_level, 0) AS water_column_average, + -- The shallowest reading leaves the most water in the well, the + -- deepest the least, so min/max swap sides here. + GREATEST(t.well_depth - wa.min_water_level, 0) AS water_column_maximum, + GREATEST(t.well_depth - wa.max_water_level, 0) AS water_column_minimum, + l.elevation, + l.point + FROM thing AS t + JOIN latest_location AS ll ON ll.thing_id = t.id + JOIN location AS l ON l.id = ll.location_id + JOIN wl_agg AS wa ON wa.thing_id = t.id + JOIN wl_last AS wl ON wl.thing_id = t.id + WHERE + t.thing_type = 'water well' + AND t.well_depth IS NOT NULL{release_filter} + """ + + +def upgrade() -> None: + _check_required_tables() + + for view_name, public_only in VIEWS: + safe_view_name = _safe_relation_name(view_name) + op.execute(text(f"DROP MATERIALIZED VIEW IF EXISTS {safe_view_name}")) + op.execute(text(_create_well_water_column_view(view_name, public_only))) + # Unique index required for REFRESH MATERIALIZED VIEW CONCURRENTLY. + op.execute( + text( + f"CREATE UNIQUE INDEX ix_{safe_view_name}_id " + f"ON {safe_view_name} (id)" + ) + ) + + +def downgrade() -> None: + for view_name, _public_only in VIEWS: + safe_view_name = _safe_relation_name(view_name) + op.execute(text(f"DROP MATERIALIZED VIEW IF EXISTS {safe_view_name}")) diff --git a/core/ogc-field-descriptions.yml b/core/ogc-field-descriptions.yml index 90cf606e..c462b6dd 100644 --- a/core/ogc-field-descriptions.yml +++ b/core/ogc-field-descriptions.yml @@ -244,6 +244,40 @@ project_areas: Kind of grouping the record represents, such as a project or a geographic area. +well_water_column: + water_column_latest: + title: Water column, latest reading + description: >- + Standing water in the well at the most recent measurement: the well's + depth less that reading's depth to water. Reported as zero where the + reading is deeper than the recorded well depth. + x-ogc-unit: https://qudt.org/vocab/unit/FT + x-ogc-unitLang: QUDT + water_column_average: + title: Water column, average reading + description: >- + Standing water the well holds on average: the well's depth less the mean + depth to water across every reading on record. Each reading counts once, + however unevenly spaced in time they are. Reported as zero where the mean + reading is deeper than the recorded well depth. + x-ogc-unit: https://qudt.org/vocab/unit/FT + x-ogc-unitLang: QUDT + water_column_maximum: + title: Water column, fullest on record + description: >- + The most standing water the well is known to have held: the well's depth + less the shallowest depth to water on record. + x-ogc-unit: https://qudt.org/vocab/unit/FT + x-ogc-unitLang: QUDT + water_column_minimum: + title: Water column, emptiest on record + description: >- + The least standing water the well is known to have held: the well's depth + less the deepest depth to water on record. Reported as zero where that + reading is deeper than the recorded well depth. + x-ogc-unit: https://qudt.org/vocab/unit/FT + x-ogc-unitLang: QUDT + water_well_summary: elevation_method: title: Elevation method diff --git a/core/pygeoapi-config-internal.yml b/core/pygeoapi-config-internal.yml index 2396539e..e62723e0 100644 --- a/core/pygeoapi-config-internal.yml +++ b/core/pygeoapi-config-internal.yml @@ -284,6 +284,48 @@ resources: table: ogc_internal_water_well_summary geom_field: point + well_water_column: + type: collection + title: Well Water Column (Water Wells) + description: >- + One row per water well, reporting how much standing water the well + holds: the well's depth minus its depth to water, in feet, worked out + four ways -- from the most recent reading, from the average of every + reading, from the shallowest water level on record (the fullest the + well has been) and from the deepest (the emptiest). Depths to water are + manual readings below ground surface -- the measured depth minus the + height of the measuring point above ground, with readings that have no + recorded measuring-point height treated as taken at ground level. + Continuous logger readings are not counted. A reading deeper than the + recorded well depth would give a negative column and is reported as + zero instead; water_well_summary publishes the raw shallowest and + deepest readings beside the well depth if you need to see that + contradiction. Wells with no depth on record, or no usable reading, are + left out. Each row also carries the well's construction record and + surveyed ground elevation. Use it to judge remaining water column and + how far it has swung over the well's history. + keywords: [ + water-wells, water-column, groundwater-level, well-depth, + depth-to-water, saturated-thickness, drawdown + ] + extents: + spatial: + bbox: [-109.05, 31.33, -103.00, 37.00] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + providers: + - type: feature + name: core.feature_provider.DescribedPostgreSQLProvider + data: + host: {postgres_host} + port: {postgres_port} + dbname: {postgres_db} + user: {postgres_user} + password: {postgres_password_env} + search_path: [public] + id_field: id + table: ogc_internal_well_water_column + geom_field: point + major_chemistry_results: type: collection title: Major Chemistry (Water Wells) diff --git a/core/pygeoapi-config.yml b/core/pygeoapi-config.yml index 093cd8b8..0348aa04 100644 --- a/core/pygeoapi-config.yml +++ b/core/pygeoapi-config.yml @@ -194,6 +194,48 @@ resources: table: ogc_water_well_summary geom_field: point + well_water_column: + type: collection + title: Well Water Column (Water Wells) + description: >- + One row per water well, reporting how much standing water the well + holds: the well's depth minus its depth to water, in feet, worked out + four ways -- from the most recent reading, from the average of every + reading, from the shallowest water level on record (the fullest the + well has been) and from the deepest (the emptiest). Depths to water are + manual readings below ground surface -- the measured depth minus the + height of the measuring point above ground, with readings that have no + recorded measuring-point height treated as taken at ground level. + Continuous logger readings are not counted. A reading deeper than the + recorded well depth would give a negative column and is reported as + zero instead; water_well_summary publishes the raw shallowest and + deepest readings beside the well depth if you need to see that + contradiction. Wells with no depth on record, or no usable reading, are + left out. Each row also carries the well's construction record and + surveyed ground elevation. Use it to judge remaining water column and + how far it has swung over the well's history. + keywords: [ + water-wells, water-column, groundwater-level, well-depth, + depth-to-water, saturated-thickness, drawdown + ] + extents: + spatial: + bbox: [-109.05, 31.33, -103.00, 37.00] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + providers: + - type: feature + name: core.feature_provider.DescribedPostgreSQLProvider + data: + host: {postgres_host} + port: {postgres_port} + dbname: {postgres_db} + user: {postgres_user} + password: {postgres_password_env} + search_path: [public] + id_field: id + table: ogc_well_water_column + geom_field: point + major_chemistry_results: type: collection title: Major Chemistry (Water Wells) diff --git a/tests/test_ogc.py b/tests/test_ogc.py index 86265e5a..18c5fea7 100644 --- a/tests/test_ogc.py +++ b/tests/test_ogc.py @@ -458,6 +458,109 @@ def test_ogc_water_elevation_wells_normalizes_meter_observations_to_feet( session.commit() +def _seed_water_levels(session, sample, readings, release_status="public"): + """readings: (day, value, measuring_point_height) -> depth to water is + value - measuring_point_height, matching the layer's convention.""" + from db import Observation + from tests import get_parameter_id + + observations = [] + for day, value, measuring_point_height in readings: + observation = Observation( + observation_datetime=datetime(2025, 1, day, 12, 0, 0), + sample_id=sample.id, + parameter_id=get_parameter_id("groundwater level", "Field Parameter"), + release_status=release_status, + value=value, + unit="ft", + measuring_point_height=measuring_point_height, + groundwater_level_reason="Water level not affected", + ) + session.add(observation) + observations.append(observation) + session.commit() + return observations + + +def test_ogc_well_water_column_computes_the_four_depths( + water_well_thing, groundwater_level_sample +): + # The well is 10 ft deep. Readings give depths to water of 5, 2 and 14 ft + # below ground surface, the last one deeper than the well itself. + with session_ctx() as session: + observations = _seed_water_levels( + session, + groundwater_level_sample, + [(1, 6.0, 1.0), (2, 3.0, 1.0), (3, 15.0, 1.0)], + ) + session.execute(text("REFRESH MATERIALIZED VIEW ogc_well_water_column")) + session.commit() + + row = session.execute( + text( + "SELECT water_column_latest, water_column_average, " + "water_column_maximum, water_column_minimum " + "FROM ogc_well_water_column WHERE id = :thing_id" + ), + {"thing_id": water_well_thing.id}, + ).one() + + # Latest reading sits 4 ft below the bottom of the well, so the + # negative column is published as zero rather than -4. + assert float(row.water_column_latest) == 0.0 + # Mean depth to water is (5 + 2 + 14) / 3 = 7 ft. + assert abs(float(row.water_column_average) - 3.0) < 1e-9 + # Shallowest water (2 ft) leaves the most in the well. + assert float(row.water_column_maximum) == 8.0 + # Deepest water (14 ft) leaves none, clamped from -4. + assert float(row.water_column_minimum) == 0.0 + + for observation in observations: + session.delete(observation) + session.commit() + session.execute(text("REFRESH MATERIALIZED VIEW ogc_well_water_column")) + session.commit() + + +def test_ogc_well_water_column_counts_private_readings_only_on_the_internal_view( + water_well_thing, groundwater_level_sample +): + with session_ctx() as session: + observations = _seed_water_levels( + session, + groundwater_level_sample, + [(1, 6.0, 1.0)], + release_status="private", + ) + for relation in ("ogc_well_water_column", "ogc_internal_well_water_column"): + session.execute(text(f"REFRESH MATERIALIZED VIEW {relation}")) + session.commit() + + # No public reading, so the well has nothing to report publicly and + # drops out of the layer entirely. + public = session.execute( + text("SELECT COUNT(*) FROM ogc_well_water_column WHERE id = :thing_id"), + {"thing_id": water_well_thing.id}, + ).scalar() + assert public == 0 + + internal = session.execute( + text( + "SELECT water_column_latest FROM ogc_internal_well_water_column " + "WHERE id = :thing_id" + ), + {"thing_id": water_well_thing.id}, + ).scalar() + assert float(internal) == 5.0 + + for observation in observations: + session.delete(observation) + session.commit() + for relation in ("ogc_well_water_column", "ogc_internal_well_water_column"): + session.execute(text(f"REFRESH MATERIALIZED VIEW {relation}")) + session.commit() + + def test_ogc_actively_monitored_wells_exposes_water_level_network_group_wells( water_well_thing, groundwater_level_observation, @@ -704,6 +807,7 @@ def test_ogc_collections(ogc_client): "depth_to_water_trend_wells", "water_elevation_wells", "water_well_summary", + "well_water_column", "major_chemistry_results", "minor_chemistry_wells", "actively_monitored_wells", @@ -730,6 +834,7 @@ def test_ogc_new_collection_items_endpoints(ogc_client): "depth_to_water_trend_wells", "water_elevation_wells", "water_well_summary", + "well_water_column", "major_chemistry_results", "minor_chemistry_wells", "actively_monitored_wells",