feat(ogc): expose last_observation_date on the Group A layers (BDMS-976/A13)

alembic/versions/b8c9d0e1f2a3_add_last_observation_date_to_thing_views.py

@@ -0,0 +1,239 @@
+"""add last_observation_date to the Group A thing views
+
+Ticket A13. The 11 thing-type layers (Group A) carry construction and location
+detail but no signal of data recency: a consumer could not tell a well measured
+last month from one last visited in 1994 without querying a second layer.
+
+This adds `last_observation_date` to the shared thing-view template -- the date
+of the most recent observation recorded against the thing, or NULL where the
+thing has no observations at all. All 11 public views and their 11
+`ogc_internal_` counterparts are rebuilt from the same template here, so the
+two mounts stay column-for-column identical.
+
+Scope of "observation": rows in the `observation` table, reached through the
+sample -> field_activity -> field_event chain that every other observation-
+backed view in this schema uses. Continuous transducer readings
+(`transducer_observation`) are deliberately *not* folded in: they live on a
+different chain (deployment -> thing), they exist for a handful of instrumented
+water wells rather than for Group A generally, and a max() over the largest
+table in the schema would need its own index on
+(deployment_id, observation_datetime) to stay cheap. Wells with logger data are
+served by ogc_actively_monitored_wells and the water-elevation layers. If
+Group A currency should later include instrument readings, that is a separate
+ticket and a separate index.
+
+The date is the UTC calendar date of the observation timestamp -- same
+convention as transducer_daily_data (v0w1x2y3z4a5) -- rather than a
+session-timezone cast, so the value does not depend on who is querying.
+
+Public views count only observations with release_status='public', matching how
+the public mount filters everything else; the internal views count all of them.
+A public well whose only observations are private therefore reads NULL on
+/ogcapi and carries a date on /ogcapi-internal.
+
+Per-thing lookup is a LEFT JOIN LATERAL rather than a grouped CTE so that a
+paginated or single-feature request touches only the observations of the rows
+it returns. That path had no indexes at all (Postgres does not index foreign
+keys on its own), so the four it needs are created here.
+
+The view bodies below are otherwise character-for-character the templates from
+f4a5b6c7d8e9 (public) and 2d3c3a268652 (internal); downgrade() restores them.
+
+Revision ID: b8c9d0e1f2a3
+Revises: 986e0eb85ab3
+Create Date: 2026-08-24 00:00:00.000000
+"""
+
+import re
+from typing import Sequence, Union
+
+from alembic import op
+from sqlalchemy import inspect, text
+
+# revision identifiers, used by Alembic.
+revision: str = "b8c9d0e1f2a3"
+down_revision: Union[str, Sequence[str], None] = "baba91fe5e83"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+REQUIRED_TABLES = {
+    "thing",
+    "location",
+    "location_thing_association",
+    "observation",
+    "sample",
+    "field_activity",
+    "field_event",
+}
+
+LATEST_LOCATION_CTE = """
+SELECT DISTINCT ON (lta.thing_id)
+    lta.thing_id,
+    lta.location_id,
+    lta.effective_start
+FROM location_thing_association AS lta
+WHERE lta.effective_end IS NULL
+ORDER BY lta.thing_id, lta.effective_start DESC
+""".strip()
+
+# Same 11 thing-type views as f4a5b6c7d8e9's THING_VIEWS.
+THING_VIEWS = [
+    ("water_wells", "water well"),
+    ("springs", "spring"),
+    ("diversions_surface_water", "diversion of surface water, etc."),
+    ("ephemeral_streams", "ephemeral stream"),
+    ("lakes_ponds_reservoirs", "lake, pond or reservoir"),
+    ("meteorological_stations", "meteorological station"),
+    ("other_things", "other"),
+    ("outfalls_wastewater_return_flow", "outfall of wastewater or return flow"),
+    ("perennial_streams", "perennial stream"),
+    ("rock_sample_locations", "rock sample location"),
+    ("soil_gas_sample_locations", "soil gas sample location"),
+]
+
+# (name, table, columns) for the observation chain the lateral walks
+# thing -> field_event -> field_activity -> sample -> observation.
+SUPPORTING_INDEXES = [
+    ("ix_field_event_thing_id", "field_event", "thing_id"),
+    ("ix_field_activity_field_event_id", "field_activity", "field_event_id"),
+    ("ix_sample_field_activity_id", "sample", "field_activity_id"),
+    (
+        "ix_observation_sample_id_observation_datetime",
+        "observation",
+        "sample_id, observation_datetime",
+    ),
+]
+
+
+def _safe_view_id(view_id: str) -> str:
+    if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", view_id):
+        raise ValueError(f"Unsafe view id: {view_id!r}")
+    return view_id
+
+
+def _check_required_tables() -> None:
+    bind = op.get_bind()
+    inspector = inspect(bind)
+    existing_tables = set(inspector.get_table_names(schema="public"))
+    missing = REQUIRED_TABLES - existing_tables
+    if missing:
+        raise RuntimeError(
+            "Cannot add last_observation_date to the OGC thing views. "
+            f"Missing required tables: {', '.join(sorted(missing))}"
+        )
+
+
+def _create_thing_view(
+    view_id: str, thing_type: str, public_only: bool, table_prefix: str
+) -> str:
+    """The Group A view template, with last_observation_date."""
+    safe_view_id = _safe_view_id(f"{table_prefix}{view_id}")
+    escaped_thing_type = thing_type.replace("'", "''")
+    release_filter = " AND t.release_status = 'public'" if public_only else ""
+    observation_release_filter = (
+        "\n                  AND o.release_status = 'public'" if public_only else ""
+    )
+    return f"""
+        CREATE VIEW {safe_view_id} AS
+        WITH latest_location AS (
+{LATEST_LOCATION_CTE}
+        )
+        SELECT
+            t.id,
+            t.name,
+            t.first_visit_date,
+            (
+                last_obs.last_observation_datetime AT TIME ZONE 'UTC'
+            )::date AS last_observation_date,
+            t.nma_pk_welldata,
+            t.well_depth,
+            t.hole_depth,
+            t.well_casing_diameter,
+            t.well_casing_depth,
+            t.well_completion_date,
+            t.well_driller_name,
+            t.well_construction_method,
+            t.well_pump_type,
+            t.well_pump_depth,
+            t.formation_completion_code,
+            t.nma_formation_zone,
+            t.release_status,
+            l.elevation,
+            l.point
+        FROM thing AS t
+        JOIN latest_location AS ll ON ll.thing_id = t.id
+        JOIN location AS l ON l.id = ll.location_id
+        LEFT JOIN LATERAL (
+            SELECT MAX(o.observation_datetime) AS last_observation_datetime
+            FROM observation AS o
+            JOIN sample AS s ON s.id = o.sample_id
+            JOIN field_activity AS fa ON fa.id = s.field_activity_id
+            JOIN field_event AS fe ON fe.id = fa.field_event_id
+            WHERE fe.thing_id = t.id{observation_release_filter}
+        ) AS last_obs ON TRUE
+        WHERE t.thing_type = '{escaped_thing_type}'{release_filter}
+    """
+
+
+def _create_thing_view_pre_a13(
+    view_id: str, thing_type: str, public_only: bool, table_prefix: str
+) -> str:
+    """The template as it stood in f4a5b6c7d8e9/2d3c3a268652, for downgrade."""
+    safe_view_id = _safe_view_id(f"{table_prefix}{view_id}")
+    escaped_thing_type = thing_type.replace("'", "''")
+    release_filter = " AND t.release_status = 'public'" if public_only else ""
+    return f"""
+        CREATE VIEW {safe_view_id} AS
+        WITH latest_location AS (
+{LATEST_LOCATION_CTE}
+        )
+        SELECT
+            t.id,
+            t.name,
+            t.first_visit_date,
+            t.nma_pk_welldata,
+            t.well_depth,
+            t.hole_depth,
+            t.well_casing_diameter,
+            t.well_casing_depth,
+            t.well_completion_date,
+            t.well_driller_name,
+            t.well_construction_method,
+            t.well_pump_type,
+            t.well_pump_depth,
+            t.formation_completion_code,
+            t.nma_formation_zone,
+            t.release_status,
+            l.elevation,
+            l.point
+        FROM thing AS t
+        JOIN latest_location AS ll ON ll.thing_id = t.id
+        JOIN location AS l ON l.id = ll.location_id
+        WHERE t.thing_type = '{escaped_thing_type}'{release_filter}
+    """
+
+
+def _rebuild_thing_views(builder) -> None:
+    for table_prefix, public_only in (("ogc_", True), ("ogc_internal_", False)):
+        for view_id, thing_type in THING_VIEWS:
+            view_name = _safe_view_id(f"{table_prefix}{view_id}")
+            op.execute(text(f"DROP VIEW IF EXISTS {view_name}"))
+            op.execute(text(builder(view_id, thing_type, public_only, table_prefix)))
+
+
+def upgrade() -> None:
+    _check_required_tables()
+
+    for index_name, table_name, columns in SUPPORTING_INDEXES:
+        op.execute(
+            text(f"CREATE INDEX IF NOT EXISTS {index_name} ON {table_name} ({columns})")
+        )
+
+    _rebuild_thing_views(_create_thing_view)
+
+
+def downgrade() -> None:
+    _rebuild_thing_views(_create_thing_view_pre_a13)
+
+    for index_name, _table_name, _columns in SUPPORTING_INDEXES:
+        op.execute(text(f"DROP INDEX IF EXISTS {index_name}"))

core/ogc-field-descriptions.yml

@@ -40,6 +40,15 @@ _defaults:
   first_visit_date:
     title: First visit date
     description: Date of the earliest Bureau visit on record for this feature.
+  last_observation_date:
+    title: Last observation date
+    description: >-
+      Date of the most recent measurement recorded against this feature, as a
+      UTC calendar date. Null where no measurement is on record for it. Counts
+      readings and laboratory results held in the observation record; continuous
+      instrument readings from a deployed logger are not included, so an
+      instrumented well can carry newer data than this date shows. On the public
+      mount only measurements released to the public are counted.
   nma_pk_welldata:
     title: Legacy NM_Aquifer well key
     description: >-

tests/features/environment.py

@@ -645,6 +645,20 @@ def _alembic_config() -> Config:
     return cfg
 
 
+def reset_pygeoapi_reflection() -> None:
+    """Drop pygeoapi's process-wide cache of reflected table models.
+
+    pygeoapi.provider.sql.get_table_model is functools.cache'd, so a provider
+    keeps serving the column list it reflected the first time a collection was
+    queried. Scenarios that move the schema under a running app (the
+    @migration-mutates-schema ones) would otherwise build SELECTs naming
+    columns the downgraded views no longer have.
+    """
+    from pygeoapi.provider.sql import get_table_model
+
+    get_table_model.cache_clear()
+
+
 def _initialize_test_schema() -> None:
     with session_ctx() as session:
         recreate_public_schema(session)
@@ -876,6 +890,7 @@ def before_scenario(context, scenario):
         # Defense in depth against a previous, unrelated failure having
         # already left the database below head.
         command.upgrade(_alembic_config(), "head")
+        reset_pygeoapi_reflection()
 
 
 def after_scenario(context, scenario):
@@ -885,6 +900,7 @@ def after_scenario(context, scenario):
         # this database. Deliberately not gated on DROP_AND_REBUILD_DB,
         # since these scenarios mutate schema regardless of that flag.
         command.upgrade(_alembic_config(), "head")
+        reset_pygeoapi_reflection()
 
     if not get_bool_env("DROP_AND_REBUILD_DB"):
         return

tests/features/ogc-cleanup-sprint1.feature

@@ -203,7 +203,7 @@ Feature: OGC Feature Layer Cleanup — Sprint 1
   # A13 — Add last_observation_date column to Group A view template
   # ---------------------------------------------------------------------------
 
-  @backend @ogc-data-currency @sprint-1 @medium-priority @A13
+  @backend @ogc-data-currency @sprint-1 @medium-priority @A13 @production
   Scenario: last_observation_date column is present in all Group A layers
     When a client requests items from each of the following layers:
       | layer-id                          |
@@ -221,7 +221,7 @@ Feature: OGC Feature Layer Cleanup — Sprint 1
     # other_things is not listed: it is in the Group A view template, but A18
     # took it off the public catalog — it is only reachable on /ogcapi-internal.
 
-  @backend @ogc-data-currency @sprint-1 @medium-priority @A13
+  @backend @ogc-data-currency @sprint-1 @medium-priority @A13 @production
   Scenario: last_observation_date is NULL for things with no associated observations
     Given monitoring locations with no linked observations exist in each of the following layers:
       | layer-id                          |
@@ -240,7 +240,7 @@ Feature: OGC Feature Layer Cleanup — Sprint 1
     # other_things is not listed: it is in the Group A view template, but A18
     # took it off the public catalog — it is only reachable on /ogcapi-internal.
 
-  @backend @ogc-data-currency @sprint-1 @medium-priority @A13
+  @backend @ogc-data-currency @sprint-1 @medium-priority @A13 @production
   Scenario: Consumers can filter Group A layers by last_observation_date
     Given each of the following Group A layers has features with last_observation_date values "2019-06-01" and "2023-06-01":
       | layer-id                        |