diff --git a/alembic/versions/c9d0e1f2a3b4_add_well_water_column_ogc_views.py b/alembic/versions/c9d0e1f2a3b4_add_well_water_column_ogc_views.py new file mode 100644 index 00000000..315146f3 --- /dev/null +++ b/alembic/versions/c9d0e1f2a3b4_add_well_water_column_ogc_views.py @@ -0,0 +1,212 @@ +"""add the well water-column OGC layer + +A water well's construction record says how deep the hole goes; its +groundwater-level record says how far down the water sits. The difference -- +the standing column of water inside the well -- is the number that says whether +a well still has usable water in it, and nothing in the catalogue published it. + +This creates ogc_well_water_column (public) and ogc_internal_well_water_column +(unfiltered), one row per water well, carrying the same well and location +fields the water_wells layer publishes plus four derived depths, all in feet: + + water_column_latest well depth minus the most recent depth to water + water_column_average well depth minus the mean depth to water + water_column_maximum well depth minus the shallowest depth to water + water_column_minimum well depth minus the deepest depth to water + +Shallowest water gives the largest column and deepest water the smallest, hence +the maximum/minimum naming: these are the extremes of the water column itself, +not of the readings behind them. + +Readings are manual groundwater-level observations, taken below ground surface +as (value - measuring_point_height) with a missing height treated as ground +level -- the same convention as ogc_water_well_summary and +ogc_latest_depth_to_water_wells, so the three layers cannot disagree about what +a depth to water is. Continuous transducer readings are not included. + +Negative results are clamped to zero. A reading deeper than the recorded well +depth is a contradiction between two records rather than a well holding +negative water, and the clamp keeps consumers from having to special-case it; +the contradiction itself stays visible in water_well_summary, which publishes +the raw shallowest and deepest readings next to the well depth. + +Rows are restricted to wells that have both a well depth and at least one +usable reading -- without either, all four columns would be NULL and the row +would say nothing. + +Materialized, because every column but the latest one aggregates a well's +entire reading history. The nightly pg_cron job (b6c7d8e9f0a1) refreshes every +matview in the public schema by name, so these two are picked up with no change +to the schedule. Both carry a unique index on id so the refresh can also be run +CONCURRENTLY by hand (`oco refresh-matview --concurrently`). + +Revision ID: c9d0e1f2a3b4 +Revises: b8c9d0e1f2a3 +Create Date: 2026-08-24 00:00:00.000000 +""" + +import re +from typing import Sequence, Union + +from alembic import op +from sqlalchemy import inspect, text + +# revision identifiers, used by Alembic. +revision: str = "c9d0e1f2a3b4" +down_revision: Union[str, Sequence[str], None] = "b8c9d0e1f2a3" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +REQUIRED_TABLES = { + "thing", + "location", + "location_thing_association", + "observation", + "sample", + "field_activity", + "field_event", +} + +LATEST_LOCATION_CTE = """ +SELECT DISTINCT ON (lta.thing_id) + lta.thing_id, + lta.location_id, + lta.effective_start +FROM location_thing_association AS lta +WHERE lta.effective_end IS NULL +ORDER BY lta.thing_id, lta.effective_start DESC +""".strip() + +VIEWS = [ + ("ogc_well_water_column", True), + ("ogc_internal_well_water_column", False), +] + + +def _safe_relation_name(name: str) -> str: + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", name): + raise ValueError(f"Unsafe relation name: {name!r}") + return name + + +def _check_required_tables() -> None: + bind = op.get_bind() + inspector = inspect(bind) + existing_tables = set(inspector.get_table_names(schema="public")) + missing = REQUIRED_TABLES - existing_tables + if missing: + raise RuntimeError( + "Cannot create the well water-column views. " + f"Missing required tables: {', '.join(sorted(missing))}" + ) + + +def _create_well_water_column_view(view_name: str, public_only: bool) -> str: + safe_view_name = _safe_relation_name(view_name) + release_filter = " AND t.release_status = 'public'" if public_only else "" + observation_release_filter = ( + "\n AND o.release_status = 'public'" if public_only else "" + ) + return f""" + CREATE MATERIALIZED VIEW {safe_view_name} AS + WITH latest_location AS ( +{LATEST_LOCATION_CTE} + ), + wl_obs AS ( + SELECT + fe.thing_id, + o.id AS observation_id, + o.observation_datetime, + (o.value - COALESCE(o.measuring_point_height, 0)) AS water_level + FROM observation AS o + JOIN sample AS s ON s.id = o.sample_id + JOIN field_activity AS fa ON fa.id = s.field_activity_id + JOIN field_event AS fe ON fe.id = fa.field_event_id + JOIN thing AS t ON t.id = fe.thing_id + WHERE + t.thing_type = 'water well' + AND fa.activity_type = 'groundwater level' + AND o.value IS NOT NULL + AND o.observation_datetime IS NOT NULL{observation_release_filter} + ), + wl_agg AS ( + SELECT + w.thing_id, + AVG(w.water_level) AS avg_water_level, + MIN(w.water_level) AS min_water_level, + MAX(w.water_level) AS max_water_level + FROM wl_obs AS w + GROUP BY w.thing_id + ), + wl_last AS ( + SELECT + ranked.thing_id, + ranked.water_level AS last_water_level + FROM ( + SELECT + w.thing_id, + w.water_level, + ROW_NUMBER() OVER ( + PARTITION BY w.thing_id + ORDER BY w.observation_datetime DESC, w.observation_id DESC + ) AS rn + FROM wl_obs AS w + ) AS ranked + WHERE ranked.rn = 1 + ) + SELECT + t.id AS id, + t.name, + t.first_visit_date, + t.nma_pk_welldata, + t.well_depth, + t.hole_depth, + t.well_casing_diameter, + t.well_casing_depth, + t.well_completion_date, + t.well_driller_name, + t.well_construction_method, + t.well_pump_type, + t.well_pump_depth, + t.formation_completion_code, + t.nma_formation_zone, + t.release_status, + GREATEST(t.well_depth - wl.last_water_level, 0) AS water_column_latest, + GREATEST(t.well_depth - wa.avg_water_level, 0) AS water_column_average, + -- The shallowest reading leaves the most water in the well, the + -- deepest the least, so min/max swap sides here. + GREATEST(t.well_depth - wa.min_water_level, 0) AS water_column_maximum, + GREATEST(t.well_depth - wa.max_water_level, 0) AS water_column_minimum, + l.elevation, + l.point + FROM thing AS t + JOIN latest_location AS ll ON ll.thing_id = t.id + JOIN location AS l ON l.id = ll.location_id + JOIN wl_agg AS wa ON wa.thing_id = t.id + JOIN wl_last AS wl ON wl.thing_id = t.id + WHERE + t.thing_type = 'water well' + AND t.well_depth IS NOT NULL{release_filter} + """ + + +def upgrade() -> None: + _check_required_tables() + + for view_name, public_only in VIEWS: + safe_view_name = _safe_relation_name(view_name) + op.execute(text(f"DROP MATERIALIZED VIEW IF EXISTS {safe_view_name}")) + op.execute(text(_create_well_water_column_view(view_name, public_only))) + # Unique index required for REFRESH MATERIALIZED VIEW CONCURRENTLY. + op.execute( + text( + f"CREATE UNIQUE INDEX ix_{safe_view_name}_id " + f"ON {safe_view_name} (id)" + ) + ) + + +def downgrade() -> None: + for view_name, _public_only in VIEWS: + safe_view_name = _safe_relation_name(view_name) + op.execute(text(f"DROP MATERIALIZED VIEW IF EXISTS {safe_view_name}")) diff --git a/core/ogc-field-descriptions.yml b/core/ogc-field-descriptions.yml index 90cf606e..c462b6dd 100644 --- a/core/ogc-field-descriptions.yml +++ b/core/ogc-field-descriptions.yml @@ -244,6 +244,40 @@ project_areas: Kind of grouping the record represents, such as a project or a geographic area. +well_water_column: + water_column_latest: + title: Water column, latest reading + description: >- + Standing water in the well at the most recent measurement: the well's + depth less that reading's depth to water. Reported as zero where the + reading is deeper than the recorded well depth. + x-ogc-unit: https://qudt.org/vocab/unit/FT + x-ogc-unitLang: QUDT + water_column_average: + title: Water column, average reading + description: >- + Standing water the well holds on average: the well's depth less the mean + depth to water across every reading on record. Each reading counts once, + however unevenly spaced in time they are. Reported as zero where the mean + reading is deeper than the recorded well depth. + x-ogc-unit: https://qudt.org/vocab/unit/FT + x-ogc-unitLang: QUDT + water_column_maximum: + title: Water column, fullest on record + description: >- + The most standing water the well is known to have held: the well's depth + less the shallowest depth to water on record. + x-ogc-unit: https://qudt.org/vocab/unit/FT + x-ogc-unitLang: QUDT + water_column_minimum: + title: Water column, emptiest on record + description: >- + The least standing water the well is known to have held: the well's depth + less the deepest depth to water on record. Reported as zero where that + reading is deeper than the recorded well depth. + x-ogc-unit: https://qudt.org/vocab/unit/FT + x-ogc-unitLang: QUDT + water_well_summary: elevation_method: title: Elevation method diff --git a/core/pygeoapi-config-internal.yml b/core/pygeoapi-config-internal.yml index 2396539e..e62723e0 100644 --- a/core/pygeoapi-config-internal.yml +++ b/core/pygeoapi-config-internal.yml @@ -284,6 +284,48 @@ resources: table: ogc_internal_water_well_summary geom_field: point + well_water_column: + type: collection + title: Well Water Column (Water Wells) + description: >- + One row per water well, reporting how much standing water the well + holds: the well's depth minus its depth to water, in feet, worked out + four ways -- from the most recent reading, from the average of every + reading, from the shallowest water level on record (the fullest the + well has been) and from the deepest (the emptiest). Depths to water are + manual readings below ground surface -- the measured depth minus the + height of the measuring point above ground, with readings that have no + recorded measuring-point height treated as taken at ground level. + Continuous logger readings are not counted. A reading deeper than the + recorded well depth would give a negative column and is reported as + zero instead; water_well_summary publishes the raw shallowest and + deepest readings beside the well depth if you need to see that + contradiction. Wells with no depth on record, or no usable reading, are + left out. Each row also carries the well's construction record and + surveyed ground elevation. Use it to judge remaining water column and + how far it has swung over the well's history. + keywords: [ + water-wells, water-column, groundwater-level, well-depth, + depth-to-water, saturated-thickness, drawdown + ] + extents: + spatial: + bbox: [-109.05, 31.33, -103.00, 37.00] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + providers: + - type: feature + name: core.feature_provider.DescribedPostgreSQLProvider + data: + host: {postgres_host} + port: {postgres_port} + dbname: {postgres_db} + user: {postgres_user} + password: {postgres_password_env} + search_path: [public] + id_field: id + table: ogc_internal_well_water_column + geom_field: point + major_chemistry_results: type: collection title: Major Chemistry (Water Wells) diff --git a/core/pygeoapi-config.yml b/core/pygeoapi-config.yml index 093cd8b8..0348aa04 100644 --- a/core/pygeoapi-config.yml +++ b/core/pygeoapi-config.yml @@ -194,6 +194,48 @@ resources: table: ogc_water_well_summary geom_field: point + well_water_column: + type: collection + title: Well Water Column (Water Wells) + description: >- + One row per water well, reporting how much standing water the well + holds: the well's depth minus its depth to water, in feet, worked out + four ways -- from the most recent reading, from the average of every + reading, from the shallowest water level on record (the fullest the + well has been) and from the deepest (the emptiest). Depths to water are + manual readings below ground surface -- the measured depth minus the + height of the measuring point above ground, with readings that have no + recorded measuring-point height treated as taken at ground level. + Continuous logger readings are not counted. A reading deeper than the + recorded well depth would give a negative column and is reported as + zero instead; water_well_summary publishes the raw shallowest and + deepest readings beside the well depth if you need to see that + contradiction. Wells with no depth on record, or no usable reading, are + left out. Each row also carries the well's construction record and + surveyed ground elevation. Use it to judge remaining water column and + how far it has swung over the well's history. + keywords: [ + water-wells, water-column, groundwater-level, well-depth, + depth-to-water, saturated-thickness, drawdown + ] + extents: + spatial: + bbox: [-109.05, 31.33, -103.00, 37.00] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + providers: + - type: feature + name: core.feature_provider.DescribedPostgreSQLProvider + data: + host: {postgres_host} + port: {postgres_port} + dbname: {postgres_db} + user: {postgres_user} + password: {postgres_password_env} + search_path: [public] + id_field: id + table: ogc_well_water_column + geom_field: point + major_chemistry_results: type: collection title: Major Chemistry (Water Wells) diff --git a/tests/test_ogc.py b/tests/test_ogc.py index 86265e5a..18c5fea7 100644 --- a/tests/test_ogc.py +++ b/tests/test_ogc.py @@ -458,6 +458,109 @@ def test_ogc_water_elevation_wells_normalizes_meter_observations_to_feet( session.commit() +def _seed_water_levels(session, sample, readings, release_status="public"): + """readings: (day, value, measuring_point_height) -> depth to water is + value - measuring_point_height, matching the layer's convention.""" + from db import Observation + from tests import get_parameter_id + + observations = [] + for day, value, measuring_point_height in readings: + observation = Observation( + observation_datetime=datetime(2025, 1, day, 12, 0, 0), + sample_id=sample.id, + parameter_id=get_parameter_id("groundwater level", "Field Parameter"), + release_status=release_status, + value=value, + unit="ft", + measuring_point_height=measuring_point_height, + groundwater_level_reason="Water level not affected", + ) + session.add(observation) + observations.append(observation) + session.commit() + return observations + + +def test_ogc_well_water_column_computes_the_four_depths( + water_well_thing, groundwater_level_sample +): + # The well is 10 ft deep. Readings give depths to water of 5, 2 and 14 ft + # below ground surface, the last one deeper than the well itself. + with session_ctx() as session: + observations = _seed_water_levels( + session, + groundwater_level_sample, + [(1, 6.0, 1.0), (2, 3.0, 1.0), (3, 15.0, 1.0)], + ) + session.execute(text("REFRESH MATERIALIZED VIEW ogc_well_water_column")) + session.commit() + + row = session.execute( + text( + "SELECT water_column_latest, water_column_average, " + "water_column_maximum, water_column_minimum " + "FROM ogc_well_water_column WHERE id = :thing_id" + ), + {"thing_id": water_well_thing.id}, + ).one() + + # Latest reading sits 4 ft below the bottom of the well, so the + # negative column is published as zero rather than -4. + assert float(row.water_column_latest) == 0.0 + # Mean depth to water is (5 + 2 + 14) / 3 = 7 ft. + assert abs(float(row.water_column_average) - 3.0) < 1e-9 + # Shallowest water (2 ft) leaves the most in the well. + assert float(row.water_column_maximum) == 8.0 + # Deepest water (14 ft) leaves none, clamped from -4. + assert float(row.water_column_minimum) == 0.0 + + for observation in observations: + session.delete(observation) + session.commit() + session.execute(text("REFRESH MATERIALIZED VIEW ogc_well_water_column")) + session.commit() + + +def test_ogc_well_water_column_counts_private_readings_only_on_the_internal_view( + water_well_thing, groundwater_level_sample +): + with session_ctx() as session: + observations = _seed_water_levels( + session, + groundwater_level_sample, + [(1, 6.0, 1.0)], + release_status="private", + ) + for relation in ("ogc_well_water_column", "ogc_internal_well_water_column"): + session.execute(text(f"REFRESH MATERIALIZED VIEW {relation}")) + session.commit() + + # No public reading, so the well has nothing to report publicly and + # drops out of the layer entirely. + public = session.execute( + text("SELECT COUNT(*) FROM ogc_well_water_column WHERE id = :thing_id"), + {"thing_id": water_well_thing.id}, + ).scalar() + assert public == 0 + + internal = session.execute( + text( + "SELECT water_column_latest FROM ogc_internal_well_water_column " + "WHERE id = :thing_id" + ), + {"thing_id": water_well_thing.id}, + ).scalar() + assert float(internal) == 5.0 + + for observation in observations: + session.delete(observation) + session.commit() + for relation in ("ogc_well_water_column", "ogc_internal_well_water_column"): + session.execute(text(f"REFRESH MATERIALIZED VIEW {relation}")) + session.commit() + + def test_ogc_actively_monitored_wells_exposes_water_level_network_group_wells( water_well_thing, groundwater_level_observation, @@ -704,6 +807,7 @@ def test_ogc_collections(ogc_client): "depth_to_water_trend_wells", "water_elevation_wells", "water_well_summary", + "well_water_column", "major_chemistry_results", "minor_chemistry_wells", "actively_monitored_wells", @@ -730,6 +834,7 @@ def test_ogc_new_collection_items_endpoints(ogc_client): "depth_to_water_trend_wells", "water_elevation_wells", "water_well_summary", + "well_water_column", "major_chemistry_results", "minor_chemistry_wells", "actively_monitored_wells",