diff --git a/CLAUDE.md b/CLAUDE.md index a5f9358c..bcc472bb 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -204,6 +204,18 @@ ArcGIS Pro cannot send a bearer token at all and neither desktop client can refresh an Authentik token. Read **`docs/internal-ogc-desktop-gis.md`** before changing the credential paths. +**`/ogcapi-internal` carries landowner PII.** The `water_well_field_operations` +collection publishes contact name, organization, role, phone and email for well +owners and operators, plus staff-written access notes. Every credential the +mount accepts reaches it, the static keys included. It is internal-only with +**no public twin** — `ogc_water_well_field_operations` does not exist and must +never be created. The layer also honours `end_date` when reading history +tables, unlike `ogc_actively_monitored_wells`, so "may we sample here" cannot +outlive the permission that granted it. Read +**`docs/water-well-field-operations-layer.md`** before changing it, and +**`docs/water-well-field-operations-columns.md`** for where each column comes +from. + ### OGC field descriptions Per-column `title`/`description`/unit for every collection lives in diff --git a/alembic/versions/e1f2a3b4c5d6_add_water_well_field_operations_layer.py b/alembic/versions/e1f2a3b4c5d6_add_water_well_field_operations_layer.py new file mode 100644 index 00000000..12276cf4 --- /dev/null +++ b/alembic/versions/e1f2a3b4c5d6_add_water_well_field_operations_layer.py @@ -0,0 +1,718 @@ +"""add the water well field operations layer + +An internal-only OGC layer for field crews: one row per water well, carrying +what a crew needs to plan and execute a visit -- where the well is, who owns it +and how to reach them, what the crew is permitted to do there, what is +installed in it, and when it was last measured. Every other well layer answers +a scientific question; this one answers an operational one. + +Two relations, not one: + + ogc_internal_water_well_field_operations_stats MATERIALIZED VIEW + Counts, first/last dates and aggregates over `observation` and + `transducer_observation`. Expensive, refreshed nightly by the pg_cron + job (b6c7d8e9f0a1), which discovers matviews from the catalog and so + needs no schedule change. + + ogc_internal_water_well_field_operations VIEW <- pygeoapi reads this + Live join of thing, location, status_history, permission_history, + measuring_point_history, monitoring_frequency_history, deployment, + sensor, contact and notes, LEFT JOINed to the stats matview. + +The split is not tidiness. Staleness is dangerous on exactly the columns that +are cheap to read: a revoked sampling permission that still reads `true` until +the next nightly refresh sends a crew onto land they are no longer welcome on. +And the current-record rule below is written against CURRENT_DATE, which in a +materialized view would freeze at refresh time -- a permission that expired +this morning would still read current tonight. Those columns have to be in the +plain view. + +Internal only, and deliberately without a public twin: the layer publishes +landowner contact details and staff-written access notes. +`ogc_water_well_field_operations` does not exist and must never be created. The +parity guard in tests/test_migration_view_parity.py reads two named migration +files (f4a5b6c7d8e9 and 2d3c3a268652) and is unaffected by a relation created +here. + +Current-record rule, applied identically to all four history tables +(status_history, permission_history, measuring_point_history, +monitoring_frequency_history): + + start_date <= CURRENT_DATE + AND (end_date IS NULL OR end_date >= CURRENT_DATE) + ORDER BY start_date DESC, id DESC -- via DISTINCT ON + +This diverges from ogc_actively_monitored_wells, which takes the greatest +start_date and ignores end_date entirely, so a status closed in 2019 still +reads as current there. Tolerable on a summary layer; not on one whose job is +to say what is true today. See docs/ogc_conventions.md. + +The three permission columns are three-valued and must stay that way: true = +a current grant says allowed, false = a current grant says not allowed, NULL = +no permission on record. NULL is not "denied" -- it means nobody has asked the +landowner yet. + +Depth to water uses (value - COALESCE(measuring_point_height, 0)) -- the +reading minus the height of the measuring point above ground, a missing height +treated as ground level. Character-for-character the convention in +ogc_water_well_summary, ogc_latest_depth_to_water_wells and +ogc_well_water_column, so the four layers cannot disagree about what a depth to +water is. + +Multi-valued columns are emitted as comma-joined text rather than as arrays +(which ogc_actively_monitored_wells uses). This layer exists to be pulled into +ArcGIS Pro and QGIS and exported to a File Geodatabase or GeoPackage for +offline field use, and neither format has a list type. + +Both relations are new here, so downgrade() drops them rather than restoring a +prior state. The supporting indexes are also created here: none of these +foreign keys was indexed (Postgres does not index foreign keys on its own). + +Revision ID: e1f2a3b4c5d6 +Revises: c9d0e1f2a3b4 +Create Date: 2026-08-31 00:00:00.000000 +""" + +import re +from typing import Sequence, Union + +from alembic import op +from sqlalchemy import inspect, text + +# revision identifiers, used by Alembic. +revision: str = "e1f2a3b4c5d6" +down_revision: Union[str, Sequence[str], None] = "c9d0e1f2a3b4" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +REQUIRED_TABLES = { + "thing", + "location", + "location_thing_association", + "observation", + "sample", + "field_activity", + "field_event", + "data_provenance", + "status_history", + "permission_history", + "measuring_point_history", + "monitoring_frequency_history", + "deployment", + "sensor", + "transducer_observation", + "contact", + "thing_contact_association", + "phone", + "email", + "notes", + "group", + "group_thing_association", + "well_purpose", + "well_casing_material", + "well_screen", + "thing_id_link", +} + +STATS_VIEW = "ogc_internal_water_well_field_operations_stats" +FEATURE_VIEW = "ogc_internal_water_well_field_operations" + +LATEST_LOCATION_CTE = """ +SELECT DISTINCT ON (lta.thing_id) + lta.thing_id, + lta.location_id, + lta.effective_start +FROM location_thing_association AS lta +WHERE lta.effective_end IS NULL +ORDER BY lta.thing_id, lta.effective_start DESC +""".strip() + +# sensor_type values that mean "this well is logging by itself". Anything else +# deployed at a well (a barometer, a camera) is equipment, not a logger, and +# must not make has_datalogger true. +LOGGER_SENSOR_TYPES = ( + "'Data Logger'", + "'Pressure Transducer'", + "'DiverLink'", + "'Diver Cable'", +) + +# Indexes the per-well lookups need. Each is created IF NOT EXISTS: none of +# these existed when this migration was written, but the thing_type one in +# particular is the sort of thing another migration may add first. +SUPPORTING_INDEXES = [ + ( + "ix_status_history_target_type_start", + "status_history (target_table, target_id, status_type, start_date DESC)", + ), + ( + "ix_permission_history_target_type_start", + "permission_history " + "(target_table, target_id, permission_type, start_date DESC)", + ), + ( + "ix_measuring_point_history_thing_start", + "measuring_point_history (thing_id, start_date DESC)", + ), + ( + "ix_monitoring_frequency_history_thing_start", + "monitoring_frequency_history (thing_id, start_date DESC)", + ), + ("ix_deployment_thing_id", "deployment (thing_id)"), + ("ix_thing_contact_association_thing_id", "thing_contact_association (thing_id)"), + ( + "ix_thing_contact_association_contact_id", + "thing_contact_association (contact_id)", + ), + ("ix_well_purpose_thing_id", "well_purpose (thing_id)"), + ("ix_well_casing_material_thing_id", "well_casing_material (thing_id)"), + ("ix_well_screen_thing_id", "well_screen (thing_id)"), + ("ix_thing_id_link_thing_id", "thing_id_link (thing_id)"), + ("ix_thing_thing_type", "thing (thing_type)"), + # No index on transducer_observation (deployment_id, ...): the existing + # uq_transducer_observation_deployment_parameter_datetime leads with + # deployment_id and already serves the per-deployment aggregate. +] + + +def _safe_relation_name(name: str) -> str: + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", name): + raise ValueError(f"Unsafe relation name: {name!r}") + return name + + +def _check_required_tables() -> None: + bind = op.get_bind() + inspector = inspect(bind) + existing_tables = set(inspector.get_table_names(schema="public")) + missing = REQUIRED_TABLES - existing_tables + if missing: + raise RuntimeError( + "Cannot create the water well field operations layer. " + f"Missing required tables: {', '.join(sorted(missing))}" + ) + + +def _create_stats_view() -> str: + """The expensive half: aggregates over the observation-scale tables.""" + safe_name = _safe_relation_name(STATS_VIEW) + return f""" + CREATE MATERIALIZED VIEW {safe_name} AS + WITH wells AS ( + SELECT t.id AS thing_id + FROM thing AS t + WHERE t.thing_type = 'water well' + ), + manual_obs AS ( + SELECT + fe.thing_id, + o.id AS observation_id, + o.observation_datetime, + (o.value - COALESCE(o.measuring_point_height, 0)) AS depth_to_water + FROM observation AS o + JOIN sample AS s ON s.id = o.sample_id + JOIN field_activity AS fa ON fa.id = s.field_activity_id + JOIN field_event AS fe ON fe.id = fa.field_event_id + JOIN wells AS w ON w.thing_id = fe.thing_id + WHERE + fa.activity_type = 'groundwater level' + AND o.value IS NOT NULL + AND o.observation_datetime IS NOT NULL + ), + manual_agg AS ( + SELECT + m.thing_id, + COUNT(*)::integer AS manual_water_level_count, + ( + MIN(m.observation_datetime) AT TIME ZONE 'UTC' + )::date AS manual_water_level_first_date, + ( + MAX(m.observation_datetime) AT TIME ZONE 'UTC' + )::date AS manual_water_level_last_date + FROM manual_obs AS m + GROUP BY m.thing_id + ), + manual_last AS ( + SELECT DISTINCT ON (m.thing_id) + m.thing_id, + m.depth_to_water AS last_depth_to_water_ft + FROM manual_obs AS m + ORDER BY m.thing_id, m.observation_datetime DESC, m.observation_id DESC + ), + chemistry_agg AS ( + SELECT + fe.thing_id, + COUNT(DISTINCT s.id)::integer AS chemistry_sample_count, + ( + MAX(s.sample_date) AT TIME ZONE 'UTC' + )::date AS chemistry_sample_last_date + FROM sample AS s + JOIN field_activity AS fa ON fa.id = s.field_activity_id + JOIN field_event AS fe ON fe.id = fa.field_event_id + JOIN wells AS w ON w.thing_id = fe.thing_id + WHERE fa.activity_type = 'water chemistry' + GROUP BY fe.thing_id + ), + field_event_agg AS ( + SELECT + fe.thing_id, + COUNT(*)::integer AS field_event_count, + ( + MAX(fe.event_date) AT TIME ZONE 'UTC' + )::date AS field_event_last_date + FROM field_event AS fe + JOIN wells AS w ON w.thing_id = fe.thing_id + GROUP BY fe.thing_id + ), + continuous_agg AS ( + SELECT + d.thing_id, + COUNT(*)::bigint AS continuous_reading_count, + MIN(tobs.observation_datetime) AS continuous_first_datetime, + MAX(tobs.observation_datetime) AS continuous_last_datetime + FROM transducer_observation AS tobs + JOIN deployment AS d ON d.id = tobs.deployment_id + JOIN wells AS w ON w.thing_id = d.thing_id + GROUP BY d.thing_id + ) + SELECT + w.thing_id, + COALESCE(ma.manual_water_level_count, 0) AS manual_water_level_count, + ma.manual_water_level_first_date, + ma.manual_water_level_last_date, + ml.last_depth_to_water_ft, + COALESCE(ca.chemistry_sample_count, 0) AS chemistry_sample_count, + ca.chemistry_sample_last_date, + COALESCE(fea.field_event_count, 0) AS field_event_count, + fea.field_event_last_date, + COALESCE(co.continuous_reading_count, 0) AS continuous_reading_count, + co.continuous_first_datetime, + co.continuous_last_datetime + FROM wells AS w + LEFT JOIN manual_agg AS ma ON ma.thing_id = w.thing_id + LEFT JOIN manual_last AS ml ON ml.thing_id = w.thing_id + LEFT JOIN chemistry_agg AS ca ON ca.thing_id = w.thing_id + LEFT JOIN field_event_agg AS fea ON fea.thing_id = w.thing_id + LEFT JOIN continuous_agg AS co ON co.thing_id = w.thing_id + """ + + +def _create_feature_view() -> str: + """The live half: everything a stale answer would misreport.""" + safe_name = _safe_relation_name(FEATURE_VIEW) + safe_stats = _safe_relation_name(STATS_VIEW) + logger_types = ", ".join(LOGGER_SENSOR_TYPES) + return f""" + CREATE VIEW {safe_name} AS + WITH latest_location AS ( +{LATEST_LOCATION_CTE} + ), + current_status AS ( + SELECT DISTINCT ON (sh.target_id, sh.status_type) + sh.target_id AS thing_id, + sh.status_type, + sh.status_value, + sh.start_date, + sh.reason + FROM status_history AS sh + WHERE + sh.target_table = 'thing' + AND sh.start_date <= CURRENT_DATE + AND (sh.end_date IS NULL OR sh.end_date >= CURRENT_DATE) + ORDER BY sh.target_id, sh.status_type, sh.start_date DESC, sh.id DESC + ), + current_permission AS ( + SELECT DISTINCT ON (ph.target_id, ph.permission_type) + ph.target_id AS thing_id, + ph.permission_type, + ph.permission_allowed, + ph.contact_id + FROM permission_history AS ph + WHERE + ph.target_table = 'thing' + AND ph.start_date <= CURRENT_DATE + AND (ph.end_date IS NULL OR ph.end_date >= CURRENT_DATE) + ORDER BY + ph.target_id, ph.permission_type, ph.start_date DESC, ph.id DESC + ), + current_measuring_point AS ( + SELECT DISTINCT ON (mp.thing_id) + mp.thing_id, + mp.measuring_point_height, + mp.measuring_point_description, + mp.start_date + FROM measuring_point_history AS mp + WHERE + mp.start_date <= CURRENT_DATE + AND (mp.end_date IS NULL OR mp.end_date >= CURRENT_DATE) + ORDER BY mp.thing_id, mp.start_date DESC, mp.id DESC + ), + current_monitoring_frequency AS ( + SELECT DISTINCT ON (mf.thing_id) + mf.thing_id, + mf.monitoring_frequency, + mf.start_date + FROM monitoring_frequency_history AS mf + WHERE + mf.start_date <= CURRENT_DATE + AND (mf.end_date IS NULL OR mf.end_date >= CURRENT_DATE) + ORDER BY mf.thing_id, mf.start_date DESC, mf.id DESC + ), + logger_deployments AS ( + SELECT + d.id AS deployment_id, + d.thing_id, + d.installation_date, + d.recording_interval, + d.recording_interval_units, + d.hanging_point_description, + se.sensor_type, + se.model, + se.serial_no, + se.sensor_status + FROM deployment AS d + JOIN sensor AS se ON se.id = d.sensor_id + WHERE + d.installation_date IS NOT NULL + AND d.removal_date IS NULL + AND se.sensor_type IN ({logger_types}) + ), + current_logger AS ( + SELECT DISTINCT ON (ld.thing_id) ld.* + FROM logger_deployments AS ld + ORDER BY ld.thing_id, ld.installation_date DESC, ld.deployment_id DESC + ), + logger_count AS ( + SELECT + ld.thing_id, + COUNT(*)::integer AS datalogger_deployment_count + FROM logger_deployments AS ld + GROUP BY ld.thing_id + ), + thing_contacts AS ( + SELECT + tca.thing_id, + c.id AS contact_id, + c.name, + c.organization, + c.role, + c.contact_type + FROM thing_contact_association AS tca + JOIN contact AS c ON c.id = tca.contact_id + ), + contact_agg AS ( + SELECT + tc.thing_id, + COUNT(DISTINCT tc.contact_id)::integer AS contact_count, + string_agg( + DISTINCT tc.name, ', ' ORDER BY tc.name + ) AS contact_names + FROM thing_contacts AS tc + GROUP BY tc.thing_id + ), + primary_contact AS ( + -- Prefer a contact recorded as Primary, but fall back to any + -- contact on the well rather than publishing a blank name next to + -- a non-zero contact_count. primary_contact_type says which case + -- this is, so the fallback is visible rather than implied. + SELECT DISTINCT ON (tc.thing_id) + tc.thing_id, + tc.contact_id, + tc.name, + tc.organization, + tc.role, + tc.contact_type + FROM thing_contacts AS tc + ORDER BY + tc.thing_id, + (tc.contact_type = 'Primary') DESC NULLS LAST, + tc.contact_id + ) + SELECT + t.id AS id, + t.name, + 'water well'::text AS thing_type, + t.release_status, + t.nma_pk_welldata, + ( + SELECT string_agg( + DISTINCT + COALESCE(tl.alternate_organization, 'unknown') + || ': ' + || tl.alternate_id, + ', ' + ORDER BY + COALESCE(tl.alternate_organization, 'unknown') + || ': ' + || tl.alternate_id + ) + FROM thing_id_link AS tl + WHERE tl.thing_id = t.id + ) AS alternate_ids, + l.county, + l.state, + l.quad_name, + -- Decimal degrees alongside the geometry. A crew types these into + -- a handheld GPS or reads them over the radio; the geometry column + -- is for the map, and a CSV export of this layer drops it. + ST_Y(l.point) AS latitude, + ST_X(l.point) AS longitude, + l.elevation, + dpl.collection_method AS elevation_method, + + -- Construction, same column names as the thing-type views. + t.well_depth, + t.hole_depth, + t.well_casing_diameter, + t.well_casing_depth, + t.well_completion_date, + t.well_driller_name, + t.well_construction_method, + t.well_pump_type, + t.well_pump_depth, + t.formation_completion_code, + t.nma_formation_zone, + ( + SELECT string_agg(DISTINCT wp.purpose, ', ' ORDER BY wp.purpose) + FROM well_purpose AS wp + WHERE wp.thing_id = t.id + ) AS well_purposes, + ( + SELECT string_agg( + DISTINCT wcm.material, ', ' ORDER BY wcm.material + ) + FROM well_casing_material AS wcm + WHERE wcm.thing_id = t.id + ) AS well_casing_materials, + scr.screen_count, + scr.screen_depth_top, + scr.screen_depth_bottom, + + -- Measuring point, current record. + cmp.measuring_point_height, + cmp.measuring_point_description, + cmp.start_date AS measuring_point_start_date, + + -- Status, current record per status type. + well_st.status_value AS well_status, + well_st.start_date AS well_status_since, + mon_st.status_value AS monitoring_status, + mon_st.start_date AS monitoring_status_since, + mon_st.reason AS monitoring_status_reason, + acc_st.status_value AS access_status, + acc_st.start_date AS access_status_since, + open_st.status_value AS open_status, + open_st.start_date AS open_status_since, + dl_st.status_value AS datalogger_suitability_status, + dl_st.start_date AS datalogger_suitability_status_since, + + -- Permission, current grants. Three-valued: NULL means no + -- permission is on record, which is not the same as denied. + wl_perm.permission_allowed AS may_measure_water_level, + chem_perm.permission_allowed AS may_sample_water_chemistry, + dl_perm.permission_allowed AS may_install_datalogger, + granter.name AS permission_granted_by, + + -- Monitoring programme. + cmf.monitoring_frequency, + cmf.start_date AS monitoring_frequency_since, + grp.group_names, + grp.group_types, + + -- Manual water levels (from the stats matview). + COALESCE(st.manual_water_level_count, 0) AS manual_water_level_count, + st.manual_water_level_first_date, + st.manual_water_level_last_date, + ( + CURRENT_DATE - st.manual_water_level_last_date + ) AS days_since_manual_water_level, + st.last_depth_to_water_ft, + + -- Chemistry sampling (from the stats matview). + COALESCE(st.chemistry_sample_count, 0) AS chemistry_sample_count, + st.chemistry_sample_last_date, + ( + CURRENT_DATE - st.chemistry_sample_last_date + ) AS days_since_chemistry_sample, + + -- Field visits (from the stats matview). + COALESCE(st.field_event_count, 0) AS field_event_count, + st.field_event_last_date, + + -- Data logger, current deployment. + (cl.deployment_id IS NOT NULL) AS has_datalogger, + COALESCE(lc.datalogger_deployment_count, 0) + AS datalogger_deployment_count, + cl.sensor_type AS datalogger_sensor_type, + cl.model AS datalogger_model, + cl.serial_no AS datalogger_serial_no, + cl.sensor_status AS datalogger_sensor_status, + cl.installation_date AS datalogger_installed_date, + cl.recording_interval AS datalogger_recording_interval, + cl.recording_interval_units AS datalogger_recording_interval_units, + cl.hanging_point_description AS datalogger_hanging_point_description, + COALESCE(st.continuous_reading_count, 0) AS continuous_reading_count, + st.continuous_first_datetime, + st.continuous_last_datetime, + ( + CURRENT_DATE + - (st.continuous_last_datetime AT TIME ZONE 'UTC')::date + ) AS days_since_continuous_reading, + + -- Contacts. + COALESCE(cag.contact_count, 0) AS contact_count, + pc.name AS primary_contact_name, + pc.organization AS primary_contact_organization, + pc.role AS primary_contact_role, + pc.contact_type AS primary_contact_type, + ( + SELECT p.phone_number + FROM phone AS p + WHERE p.contact_id = pc.contact_id + ORDER BY p.id + LIMIT 1 + ) AS primary_contact_phone, + ( + SELECT e.email + FROM email AS e + WHERE e.contact_id = pc.contact_id + ORDER BY e.id + LIMIT 1 + ) AS primary_contact_email, + cag.contact_names, + + -- Visit instructions. Separator is ' | ' rather than ', ' because + -- the content is free text and routinely contains commas. + ( + SELECT string_agg(n.content, ' | ' ORDER BY n.id DESC) + FROM notes AS n + WHERE + n.target_table = 'thing' + AND n.target_id = t.id + AND n.note_type = 'Access' + ) AS access_notes, + ( + SELECT string_agg(n.content, ' | ' ORDER BY n.id DESC) + FROM notes AS n + WHERE + n.target_table = 'thing' + AND n.target_id = t.id + AND n.note_type = 'Directions' + ) AS directions_notes, + + l.point + FROM thing AS t + JOIN latest_location AS ll ON ll.thing_id = t.id + JOIN location AS l ON l.id = ll.location_id + LEFT JOIN {safe_stats} AS st ON st.thing_id = t.id + LEFT JOIN current_status AS well_st + ON well_st.thing_id = t.id AND well_st.status_type = 'Well Status' + LEFT JOIN current_status AS mon_st + ON mon_st.thing_id = t.id + AND mon_st.status_type = 'Monitoring Status' + LEFT JOIN current_status AS acc_st + ON acc_st.thing_id = t.id AND acc_st.status_type = 'Access Status' + LEFT JOIN current_status AS open_st + ON open_st.thing_id = t.id AND open_st.status_type = 'Open Status' + LEFT JOIN current_status AS dl_st + ON dl_st.thing_id = t.id + AND dl_st.status_type = 'Datalogger Suitability Status' + LEFT JOIN current_permission AS wl_perm + ON wl_perm.thing_id = t.id + AND wl_perm.permission_type = 'Water Level Sample' + LEFT JOIN current_permission AS chem_perm + ON chem_perm.thing_id = t.id + AND chem_perm.permission_type = 'Water Chemistry Sample' + LEFT JOIN current_permission AS dl_perm + ON dl_perm.thing_id = t.id + AND dl_perm.permission_type = 'Datalogger Installation' + LEFT JOIN contact AS granter ON granter.id = wl_perm.contact_id + LEFT JOIN current_measuring_point AS cmp ON cmp.thing_id = t.id + LEFT JOIN current_monitoring_frequency AS cmf ON cmf.thing_id = t.id + LEFT JOIN current_logger AS cl ON cl.thing_id = t.id + LEFT JOIN logger_count AS lc ON lc.thing_id = t.id + LEFT JOIN contact_agg AS cag ON cag.thing_id = t.id + LEFT JOIN primary_contact AS pc ON pc.thing_id = t.id + LEFT JOIN LATERAL ( + -- Character-for-character the elevation-provenance lookup in + -- ogc_water_well_summary (2d3c3a268652), so the two layers cannot + -- disagree about where a well's elevation came from. + SELECT dp.collection_method + FROM data_provenance AS dp + WHERE + dp.target_table = 'location' + AND dp.target_id = l.id + AND dp.field_name = 'elevation' + ORDER BY dp.id DESC + LIMIT 1 + ) AS dpl ON true + LEFT JOIN LATERAL ( + SELECT + COUNT(*)::integer AS screen_count, + MIN(ws.screen_depth_top) AS screen_depth_top, + MAX(ws.screen_depth_bottom) AS screen_depth_bottom + FROM well_screen AS ws + WHERE ws.thing_id = t.id + ) AS scr ON TRUE + LEFT JOIN LATERAL ( + -- group_thing_association has no unique constraint on + -- (group_id, thing_id), so DISTINCT before aggregating. Both + -- strings are ordered by the same group name so they stay + -- index-aligned with each other. + SELECT + string_agg(dm.group_name, ', ' ORDER BY dm.group_name) + AS group_names, + string_agg(dm.group_type, ', ' ORDER BY dm.group_name) + AS group_types + FROM ( + SELECT DISTINCT + g.id AS group_id, + g.name AS group_name, + g.group_type + FROM group_thing_association AS gta + JOIN "group" AS g ON g.id = gta.group_id + WHERE gta.thing_id = t.id + ) AS dm + ) AS grp ON TRUE + WHERE t.thing_type = 'water well' + """ + + +def upgrade() -> None: + _check_required_tables() + + safe_stats = _safe_relation_name(STATS_VIEW) + safe_feature = _safe_relation_name(FEATURE_VIEW) + + # The feature view depends on the stats matview, so it is dropped first and + # created last. + op.execute(text(f"DROP VIEW IF EXISTS {safe_feature}")) + op.execute(text(f"DROP MATERIALIZED VIEW IF EXISTS {safe_stats}")) + + op.execute(text(_create_stats_view())) + # Unique index required for REFRESH MATERIALIZED VIEW CONCURRENTLY. + op.execute( + text( + f"CREATE UNIQUE INDEX ix_{safe_stats}_thing_id " + f"ON {safe_stats} (thing_id)" + ) + ) + op.execute(text(_create_feature_view())) + + for index_name, definition in SUPPORTING_INDEXES: + op.execute( + text( + f"CREATE INDEX IF NOT EXISTS {_safe_relation_name(index_name)} ON {definition}" + ) + ) + + +def downgrade() -> None: + safe_stats = _safe_relation_name(STATS_VIEW) + safe_feature = _safe_relation_name(FEATURE_VIEW) + + op.execute(text(f"DROP VIEW IF EXISTS {safe_feature}")) + op.execute(text(f"DROP MATERIALIZED VIEW IF EXISTS {safe_stats}")) + + for index_name, _definition in SUPPORTING_INDEXES: + op.execute(text(f"DROP INDEX IF EXISTS {_safe_relation_name(index_name)}")) diff --git a/core/ogc-field-descriptions.yml b/core/ogc-field-descriptions.yml index c462b6dd..244525f8 100644 --- a/core/ogc-field-descriptions.yml +++ b/core/ogc-field-descriptions.yml @@ -2099,6 +2099,397 @@ minor_chemistry_wells: recorded them. +# water well field operations. Internal mount only -- the layer carries landowner +# contact details and staff-written access notes, so it has no public twin. +# See docs/water-well-field-operations-layer.md. +water_well_field_operations: + quad_name: + title: USGS quadrangle + description: >- + Name of the USGS 7.5-minute topographic quadrangle the well falls in. + elevation_method: + title: Elevation method + description: >- + How the ground-surface elevation was determined, such as GPS survey or + read from a digital elevation model. Governs how much precision the + elevation deserves. + enum-lexicon: collection_method + latitude: + title: Latitude + description: >- + Latitude of the well in decimal degrees on the WGS 84 datum, the same + coordinate the feature geometry carries. Published as a plain number so + it survives a spreadsheet or CSV export of this layer, which drops the + geometry, and so a crew can read it into a handheld GPS. + x-ogc-unit: https://qudt.org/vocab/unit/DEG + x-ogc-unitLang: QUDT + longitude: + title: Longitude + description: >- + Longitude of the well in decimal degrees on the WGS 84 datum, negative + west of Greenwich, the same coordinate the feature geometry carries. + Published as a plain number for the same reason as the latitude. + x-ogc-unit: https://qudt.org/vocab/unit/DEG + x-ogc-unitLang: QUDT + alternate_ids: + title: Alternate identifiers + description: >- + Identifiers other organisations use for this well, each written as + organisation followed by the identifier and separated by commas. Empty + where no cross-reference has been recorded. + well_purposes: + title: Well purposes + description: >- + What the well is used for, from the controlled well-purpose vocabulary -- + domestic, irrigation, monitoring, and so on. A well may serve more than + one purpose, in which case the purposes are separated by commas. + well_casing_materials: + title: Casing materials + description: >- + Materials the well casing is made of, from the controlled casing-material + vocabulary. Separated by commas where a well has more than one. + screen_count: + title: Screened intervals + description: >- + Number of discrete screened intervals recorded for the well. Zero means + no screen record exists, which is not the same as an unscreened well. + screen_depth_top: + title: Shallowest screen top + description: >- + Depth to the top of the shallowest screened interval, below ground + surface. + x-ogc-unit: https://qudt.org/vocab/unit/FT + x-ogc-unitLang: QUDT + screen_depth_bottom: + title: Deepest screen bottom + description: >- + Depth to the bottom of the deepest screened interval, below ground + surface. With the shallowest screen top this gives the outer extent of + the screened section, not the length of any one screen. + x-ogc-unit: https://qudt.org/vocab/unit/FT + x-ogc-unitLang: QUDT + measuring_point_height: + title: Measuring point height + description: >- + Height of the current measuring point above the ground surface. Manual + depth-to-water readings are taken from this point, so it is what converts + a raw reading to a depth below ground surface. + x-ogc-unit: https://qudt.org/vocab/unit/FT + x-ogc-unitLang: QUDT + measuring_point_description: + title: Measuring point description + description: >- + Where on the wellhead the measuring point is, as written for the person + standing at the well -- for example the north side of the casing at the + top of the PVC. The single most useful string on this layer for a crew + about to take a reading. + measuring_point_start_date: + title: Measuring point in use since + description: >- + Date the current measuring point configuration took effect. A recent date + means the wellhead was altered and readings before it are referenced to a + different point. + well_status: + title: Well status + description: >- + Current condition of the well itself. Null where no status is on record. + enum: + - Active, pumping well + - Inactive, exists but not used + - Destroyed, exists but not usable + - Abandoned + well_status_since: + title: Well status since + description: Date the current well status took effect. + monitoring_status: + title: Monitoring status + description: >- + Whether the well is currently on a monitoring schedule. Null where no + status is on record. + enum: + - Currently monitored + - Not currently monitored + monitoring_status_since: + title: Monitoring status since + description: Date the current monitoring status took effect. + monitoring_status_reason: + title: Monitoring status reason + description: >- + Why the well carries its current monitoring status, in the words of the + staff member who recorded it. This is where a landowner's request to stop + visiting is written down. + access_status: + title: Access status + description: >- + Current recorded state of physical access to the well. No values are + scoped to this status type in the controlled vocabulary yet, so the + column is expected to be sparse; treat a null as "not recorded" and + confirm access from the access notes and the landowner contact. + access_status_since: + title: Access status since + description: Date the current access status took effect. + open_status: + title: Open status + description: >- + Whether the well is physically open to a sounder or tape. A closed well + cannot be measured by hand however good the access. Null where no status + is on record. + enum: + - Open + - Open (unequipped) + - Closed + open_status_since: + title: Open status since + description: Date the current open status took effect. + datalogger_suitability_status: + title: Datalogger suitability + description: >- + Whether the well is physically suitable for a logger installation. This + is a judgement about the well, separate from whether the landowner has + given permission to install one. Null where no status is on record. + enum: + - Datalogger can be installed + - Datalogger cannot be installed + datalogger_suitability_status_since: + title: Datalogger suitability since + description: Date the current datalogger suitability status took effect. + may_measure_water_level: + title: May measure water level + description: >- + Whether a current permission allows a manual water-level measurement at + this well. True means a permission on record allows it; false means a + permission on record refuses it; null means no permission has been + recorded either way, which is not the same as a refusal -- it means + nobody has asked the landowner yet. + may_sample_water_chemistry: + title: May sample water chemistry + description: >- + Whether a current permission allows a water-chemistry sample to be + collected at this well. True allows, false refuses, null means no + permission is on record -- not a refusal. + may_install_datalogger: + title: May install datalogger + description: >- + Whether a current permission allows a logger to be installed in this + well. True allows, false refuses, null means no permission is on record + -- not a refusal. Separate from datalogger suitability, which says + whether the well could take one. + permission_granted_by: + title: Permission granted by + description: >- + Name of the contact who granted the current water-level permission, so a + crew can say who agreed to the visit. Null where no water-level + permission is on record. + monitoring_frequency: + title: Monitoring frequency + description: >- + How often the well is scheduled to be visited under its current + monitoring plan. Null where no frequency is on record. + enum-lexicon: monitoring_frequency + monitoring_frequency_since: + title: Monitoring frequency since + description: Date the current monitoring frequency took effect. + group_names: + title: Groups + description: >- + Names of the monitoring plans, geographic areas and historical groupings + the well belongs to, separated by commas. A well often sits in several. + group_types: + title: Group types + description: >- + Type of each group named in the groups column, in the same order and + separated by commas -- a monitoring plan, a geographic area, or a + historical grouping. + manual_water_level_count: + title: Manual water level readings + description: >- + Number of hand-measured groundwater level readings on record for the + well. Continuous readings from an installed logger are counted separately. + manual_water_level_first_date: + title: First manual reading + description: >- + Date of the earliest hand-measured groundwater level on record, as a UTC + calendar date. + manual_water_level_last_date: + title: Last manual reading + description: >- + Date of the most recent hand-measured groundwater level on record, as a + UTC calendar date. + days_since_manual_water_level: + title: Days since last manual reading + description: >- + Days elapsed since the most recent hand-measured groundwater level. + Computed against today, not against the last refresh, so it is current on + every request. Null where the well has never been measured by hand. + last_depth_to_water_ft: + title: Latest depth to water + description: >- + Most recent hand-measured depth to water, below ground surface -- the + measured value minus the height of the measuring point above ground, with + a reading that has no recorded measuring-point height treated as taken at + ground level. The same convention as the water-level summary and water + column layers. + x-ogc-unit: https://qudt.org/vocab/unit/FT + x-ogc-unitLang: QUDT + chemistry_sample_count: + title: Chemistry samples + description: >- + Number of water-chemistry samples collected at the well. Counts samples, + not the individual analyte results measured from them. + chemistry_sample_last_date: + title: Last chemistry sample + description: >- + Date the most recent water-chemistry sample was collected, as a UTC + calendar date. + days_since_chemistry_sample: + title: Days since last chemistry sample + description: >- + Days elapsed since the most recent water-chemistry sample. Computed + against today. Null where the well has never been sampled. + field_event_count: + title: Field visits + description: >- + Number of recorded visits to the well. Broader than either measurement + count: a visit that produced no reading, because the gate was locked or + the well was dry, is still a visit. + field_event_last_date: + title: Last field visit + description: >- + Date of the most recent recorded visit, as a UTC calendar date. A gap + between this and the last reading dates means someone went and came back + with nothing. + has_datalogger: + title: Has a datalogger + description: >- + Whether a logger is currently installed in the well -- a deployment with + an installation date and no removal date, of a data logger, pressure + transducer, DiverLink or diver cable. Other equipment deployed at the + well does not count. + datalogger_deployment_count: + title: Current logger deployments + description: >- + Number of logger deployments currently open at the well. Normally zero or + one; a higher number means the single logger columns describe only the + most recently installed of several. + datalogger_sensor_type: + title: Logger type + description: Kind of logger currently installed. + enum: + - Data Logger + - Pressure Transducer + - DiverLink + - Diver Cable + datalogger_model: + title: Logger model + description: Model of the logger currently installed. + datalogger_serial_no: + title: Logger serial number + description: >- + Serial number of the logger currently installed, for matching the + instrument in hand against the record. + datalogger_sensor_status: + title: Logger status + description: >- + Service state of the installed logger. A logger still recorded as + installed but marked retired or lost is a record worth correcting in the + field. + enum-lexicon: sensor_status + datalogger_installed_date: + title: Logger installed + description: Date the current logger deployment began. + datalogger_recording_interval: + title: Logger recording interval + description: >- + How often the installed logger records a reading, in the units given by + the recording interval units column. + datalogger_recording_interval_units: + title: Logger recording interval units + description: Time units the logger recording interval is expressed in. + datalogger_hanging_point_description: + title: Logger hanging point + description: >- + Where the logger cable is secured at the wellhead, as written for the + person about to pull it. Null where the deployment did not record one. + continuous_reading_count: + title: Continuous readings + description: >- + Number of readings logged automatically by instruments deployed in this + well, across all of its deployments. Separate from the hand-measured + count. + continuous_first_datetime: + title: First continuous reading + description: >- + Timestamp of the earliest automatically logged reading held for the well. + continuous_last_datetime: + title: Last continuous reading + description: >- + Timestamp of the most recent automatically logged reading held for the + well. A well whose logger has stopped reporting shows an old value here + while still reading as having a logger installed. + days_since_continuous_reading: + title: Days since last continuous reading + description: >- + Days elapsed since the most recent automatically logged reading, computed + against today. Null where the well has no logged readings at all. Rising + values on an instrumented well mean the logger needs attention. + contact_count: + title: Contacts + description: >- + Number of contacts associated with the well -- landowners, operators, and + anyone else recorded as a point of contact for it. + primary_contact_name: + title: Primary contact + description: >- + Name of the contact to approach about this well. This is the contact + recorded as primary where one exists; where none does, it is another + contact on the well, and the primary contact type column says which case + applies. + primary_contact_organization: + title: Primary contact organisation + description: Organisation the primary contact belongs to, where recorded. + primary_contact_role: + title: Primary contact role + description: >- + Role the primary contact holds in relation to the well, such as owner, + manager, or operator. + enum-lexicon: role + primary_contact_type: + title: Primary contact type + description: >- + Whether the contact shown is the one recorded as primary for the well or + a fallback used because no primary contact is on record. Read it before + relying on the contact details beside it. + enum-lexicon: contact_type + primary_contact_phone: + title: Primary contact phone + description: >- + Phone number for the primary contact. Where several are on record this is + the first one recorded, so a crew that cannot reach it should check the + full contact record. + primary_contact_email: + title: Primary contact email + description: >- + Email address for the primary contact. Where several are on record this + is the first one recorded. + contact_names: + title: All contacts + description: >- + Names of every contact associated with the well, separated by commas, so + a crew can see who else can be approached if the primary contact cannot + be reached. + access_notes: + title: Access notes + description: >- + Staff-written instructions for getting to the well and getting into it -- + gates, keys, locks, livestock, who to call first. Where several notes + exist they are separated by vertical bars, most recently recorded first. + directions_notes: + title: Directions + description: >- + Staff-written directions for finding the well on the ground, which + coordinates alone do not give. Where several notes exist they are + separated by vertical bars, most recently recorded first. + # EDR collections. Keys here are parameter names read out of the data, not # column names -- ogc_waterlevels stamps a single literal, while # ogc_water_chemistry carries the analyte text exactly as the laboratory diff --git a/core/pygeoapi-config-internal.yml b/core/pygeoapi-config-internal.yml index e62723e0..9b0d080b 100644 --- a/core/pygeoapi-config-internal.yml +++ b/core/pygeoapi-config-internal.yml @@ -77,6 +77,50 @@ resources: table: ogc_internal_locations geom_field: point + water_well_field_operations: + type: collection + title: Water Well Field Operations + description: >- + One record per water well, assembled for the crews who visit them. Where + every other well layer answers a scientific question, this one answers an + operational one: whether a crew may go to a well, what they are permitted + to do when they get there, and whether it is overdue. It pairs the well's + construction and measuring-point detail with its current well, + monitoring, access, open and datalogger-suitability status, the standing + landowner permissions for water-level measurement, chemistry sampling and + logger installation, the logger installed in it if any, how long it has + been since it was last measured and last sampled, and the landowner + contact and access notes needed to reach it. Published on the internal + mount only, because it carries personal contact details for landowners + and operators alongside staff-written access instructions. + keywords: + [ + water-wells, + field-operations, + landowner-contacts, + access-permissions, + monitoring-status, + data-logger, + internal, + ] + extents: + spatial: + bbox: [-109.05, 31.33, -103.00, 37.00] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + providers: + - type: feature + name: core.feature_provider.DescribedPostgreSQLProvider + data: + host: {postgres_host} + port: {postgres_port} + dbname: {postgres_db} + user: {postgres_user} + password: {postgres_password_env} + search_path: [public] + id_field: id + table: ogc_internal_water_well_field_operations + geom_field: point + latest_depth_to_water_wells: type: collection title: Latest Depth to Water (Water Wells) diff --git a/docs/internal-ogc-desktop-gis.md b/docs/internal-ogc-desktop-gis.md index 7efa6640..045bebd3 100644 --- a/docs/internal-ogc-desktop-gis.md +++ b/docs/internal-ogc-desktop-gis.md @@ -5,6 +5,16 @@ collections. It is gated by `core/internal_ogc_auth.py`, an ASGI middleware that runs in front of the raw Starlette Mount — FastAPI's `Depends()` machinery never sees these requests, so none of the `*_dependency` role parameters apply here. +> **An internal credential dispenses personal information.** The +> `water_well_field_operations` collection publishes landowner and operator +> contact details — name, organisation, role, phone number, email address — +> alongside staff-written access notes that routinely contain gate codes and +> names. Every credential accepted by this mount reaches it: an Authentik JWT +> carrying `OGCInternal`, and the static keys in `INTERNAL_OGC_API_KEYS`, which +> are shared secrets revocable only by redeploy. Any credential path added +> later inherits the same reach. Issue keys with that in mind, and read +> `docs/water-well-field-operations-layer.md` before widening who holds one. + ## Why there are static API keys at all The mount originally accepted only `Authorization: Bearer `. diff --git a/docs/ogc_conventions.md b/docs/ogc_conventions.md index 6ce6c8be..268f2cc6 100644 --- a/docs/ogc_conventions.md +++ b/docs/ogc_conventions.md @@ -155,6 +155,33 @@ human-readable display name). | `locations` | N/A | Locations | N/A | N/A | Conforms. Hidden from public catalog — scope decision, not a naming fix | | `avg_tds_wells` | `water_well_average_total_dissolved_solids` | Average TDS (Water Wells) | Average Total Dissolved Solids (Water Wells) | 1 and 2 | ⚠️ *needs review*: `avg` and `tds` both unexplained abbreviations; hidden from public catalog but the rule still applies for internal consumers | | `latest_depth_to_water_wells` | N/A | Latest Depth to Water (Water Wells) | N/A | N/A | Conforms. Hidden from public catalog — audit flagged it as redundant with `water_well_summary`, not a naming defect | +| `water_well_field_operations` | N/A | Water Well Field Operations | N/A | N/A | Conforms. Internal-only with no public form at all — it publishes landowner contact details and staff-written access notes | + +## Current-record semantics + +Layers that read a history table (`status_history`, `permission_history`, +`measuring_point_history`, `monitoring_frequency_history`) disagree about what +"current" means, and the disagreement is deliberate. + +`ogc_actively_monitored_wells` takes the row with the greatest `start_date` and +**ignores `end_date`**, so a monitoring status closed in 2019 still reads as +current there. That is tolerable on a summary layer, where the question is +roughly "is this well in the programme". + +`ogc_internal_water_well_field_operations` honours the window: + +```sql +WHERE h.start_date <= CURRENT_DATE + AND (h.end_date IS NULL OR h.end_date >= CURRENT_DATE) +ORDER BY h.start_date DESC, h.id DESC +``` + +Its columns answer "may a crew do this today", and a permission that ran out +last month must not read as a permission. A row whose window has not opened yet +is not current either, and reads null. + +New layers should follow the second form. The first is kept only because +changing it would move rows in a published layer. ## References diff --git a/docs/water-well-field-operations-columns.md b/docs/water-well-field-operations-columns.md new file mode 100644 index 00000000..423ad6eb --- /dev/null +++ b/docs/water-well-field-operations-columns.md @@ -0,0 +1,106 @@ +# water well field operations — column sources + +Every column published by `ogc_internal_water_well_field_operations`, in view +order, and where its value comes from. Columns marked *(stats)* are read from +the `ogc_internal_water_well_field_operations_stats` materialized view and are as +fresh as its last refresh; everything else is joined live on each request. + +"Current record" means the history row satisfying +`start_date <= CURRENT_DATE AND (end_date IS NULL OR end_date >= CURRENT_DATE)`, +latest `start_date` first. "Most recent association" means the +`location_thing_association` row with no `effective_end`, latest +`effective_start` first. + +Generated from the view definition in +`alembic/versions/e1f2a3b4c5d6_add_water_well_field_operations_layer.py`. Field +prose lives in `core/ogc-field-descriptions.yml`; the design rationale lives in +`docs/water-well-field-operations-layer.md`. + +| Column | Source | +| --- | --- | +| `id` | `thing.id` | +| `name` | `thing.name` | +| `thing_type` | Literal `'water well'` — the view's row filter | +| `release_status` | `thing.release_status` | +| `nma_pk_welldata` | `thing.nma_pk_welldata` | +| `alternate_ids` | `thing_id_link.alternate_organization` + `.alternate_id`, comma-joined | +| `county` | `location.county`, most recent association | +| `state` | `location.state`, most recent association | +| `quad_name` | `location.quad_name`, most recent association | +| `latitude` | `ST_Y(location.point)` — decimal degrees, WGS 84 | +| `longitude` | `ST_X(location.point)` — decimal degrees, WGS 84 | +| `elevation` | `location.elevation`, most recent association | +| `elevation_method` | `data_provenance.collection_method` where `target_table = 'location'` and `field_name = 'elevation'`, latest `id` | +| `well_depth` | `thing.well_depth` | +| `hole_depth` | `thing.hole_depth` | +| `well_casing_diameter` | `thing.well_casing_diameter` | +| `well_casing_depth` | `thing.well_casing_depth` | +| `well_completion_date` | `thing.well_completion_date` | +| `well_driller_name` | `thing.well_driller_name` | +| `well_construction_method` | `thing.well_construction_method` | +| `well_pump_type` | `thing.well_pump_type` | +| `well_pump_depth` | `thing.well_pump_depth` | +| `formation_completion_code` | `thing.formation_completion_code` | +| `nma_formation_zone` | `thing.nma_formation_zone` | +| `well_purposes` | `well_purpose.purpose`, comma-joined | +| `well_casing_materials` | `well_casing_material.material`, comma-joined | +| `screen_count` | `count(well_screen)` | +| `screen_depth_top` | `min(well_screen.screen_depth_top)` | +| `screen_depth_bottom` | `max(well_screen.screen_depth_bottom)` | +| `measuring_point_height` | `measuring_point_history.measuring_point_height`, current record | +| `measuring_point_description` | `measuring_point_history.measuring_point_description`, current record | +| `measuring_point_start_date` | `measuring_point_history.start_date`, current record | +| `well_status` | `status_history.status_value` where `status_type = 'Well Status'`, current record | +| `well_status_since` | `status_history.start_date`, same record | +| `monitoring_status` | `status_history.status_value` where `status_type = 'Monitoring Status'`, current record | +| `monitoring_status_since` | `status_history.start_date`, same record | +| `monitoring_status_reason` | `status_history.reason`, same record | +| `access_status` | `status_history.status_value` where `status_type = 'Access Status'`, current record | +| `access_status_since` | `status_history.start_date`, same record | +| `open_status` | `status_history.status_value` where `status_type = 'Open Status'`, current record | +| `open_status_since` | `status_history.start_date`, same record | +| `datalogger_suitability_status` | `status_history.status_value` where `status_type = 'Datalogger Suitability Status'`, current record | +| `datalogger_suitability_status_since` | `status_history.start_date`, same record | +| `may_measure_water_level` | `permission_history.permission_allowed` where `permission_type = 'Water Level Sample'`, current record | +| `may_sample_water_chemistry` | `permission_history.permission_allowed` where `permission_type = 'Water Chemistry Sample'`, current record | +| `may_install_datalogger` | `permission_history.permission_allowed` where `permission_type = 'Datalogger Installation'`, current record | +| `permission_granted_by` | `contact.name` via `permission_history.contact_id` on the current water-level grant | +| `monitoring_frequency` | `monitoring_frequency_history.monitoring_frequency`, current record | +| `monitoring_frequency_since` | `monitoring_frequency_history.start_date`, current record | +| `group_names` | `group.name` via `group_thing_association`, comma-joined | +| `group_types` | `group.group_type`, same order as `group_names` | +| `manual_water_level_count` | `count(observation)` via `sample` → `field_activity` → `field_event`, `activity_type = 'groundwater level'` *(stats)* | +| `manual_water_level_first_date` | `min(observation.observation_datetime)`, UTC date, same chain *(stats)* | +| `manual_water_level_last_date` | `max(observation.observation_datetime)`, UTC date, same chain *(stats)* | +| `days_since_manual_water_level` | `CURRENT_DATE - manual_water_level_last_date` | +| `last_depth_to_water_ft` | `observation.value - COALESCE(observation.measuring_point_height, 0)` on the latest reading *(stats)* | +| `chemistry_sample_count` | `count(DISTINCT sample.id)`, `activity_type = 'water chemistry'` *(stats)* | +| `chemistry_sample_last_date` | `max(sample.sample_date)`, UTC date, same filter *(stats)* | +| `days_since_chemistry_sample` | `CURRENT_DATE - chemistry_sample_last_date` | +| `field_event_count` | `count(field_event)` for the well *(stats)* | +| `field_event_last_date` | `max(field_event.event_date)`, UTC date *(stats)* | +| `has_datalogger` | `true` when an open logger deployment exists — see `datalogger_sensor_type` | +| `datalogger_deployment_count` | `count(deployment)` open logger deployments | +| `datalogger_sensor_type` | `sensor.sensor_type` on the current deployment, restricted to Data Logger / Pressure Transducer / DiverLink / Diver Cable | +| `datalogger_model` | `sensor.model`, same deployment | +| `datalogger_serial_no` | `sensor.serial_no`, same deployment | +| `datalogger_sensor_status` | `sensor.sensor_status`, same deployment | +| `datalogger_installed_date` | `deployment.installation_date`, same deployment | +| `datalogger_recording_interval` | `deployment.recording_interval`, same deployment | +| `datalogger_recording_interval_units` | `deployment.recording_interval_units`, same deployment | +| `datalogger_hanging_point_description` | `deployment.hanging_point_description`, same deployment | +| `continuous_reading_count` | `count(transducer_observation)` via `deployment` *(stats)* | +| `continuous_first_datetime` | `min(transducer_observation.observation_datetime)` *(stats)* | +| `continuous_last_datetime` | `max(transducer_observation.observation_datetime)` *(stats)* | +| `days_since_continuous_reading` | `CURRENT_DATE - continuous_last_datetime::date` | +| `contact_count` | `count(DISTINCT contact)` via `thing_contact_association` | +| `primary_contact_name` | `contact.name`, `contact_type = 'Primary'` preferred, else lowest `contact.id` | +| `primary_contact_organization` | `contact.organization`, same contact | +| `primary_contact_role` | `contact.role`, same contact | +| `primary_contact_type` | `contact.contact_type`, same contact — says whether the row above is a real primary or a fallback | +| `primary_contact_phone` | `phone.phone_number`, lowest `phone.id` for that contact | +| `primary_contact_email` | `email.email`, lowest `email.id` for that contact | +| `contact_names` | `contact.name` for every associated contact, comma-joined | +| `access_notes` | `notes.content` where `note_type = 'Access'`, newest first, joined with ` | ` | +| `directions_notes` | `notes.content` where `note_type = 'Directions'`, newest first, joined with ` | ` | +| `point` | `location.point`, most recent association (PostGIS Point, EPSG:4326) | diff --git a/docs/water-well-field-operations-layer.md b/docs/water-well-field-operations-layer.md new file mode 100644 index 00000000..9e2c6130 --- /dev/null +++ b/docs/water-well-field-operations-layer.md @@ -0,0 +1,519 @@ +# Water well field operations layer — design + +Status: **implemented** on `feat/amp-field-operations-ogc-layer`. Migration +`e1f2a3b4c5d6`; column-by-column source table in +`docs/water-well-field-operations-columns.md`. + +Two questions this document opened were settled before implementation: "AMP" is +a label for the crews' view of the register, not a subset of it, so the layer +covers every water well; and the contact detail in §4 is published in full +(option A). + +## 0. Why the name does not say AMP + +The layer was requested as `amp_field_well_operations` and renamed before it +merged, per the pre-merge check in `docs/ogc_conventions.md`. Three rules +pushed the same way: + +- `amp` is an unexpanded abbreviation, which the do/don't rules forbid + outright ("Spell abbreviations out in full ... If the accurate name is long, + that's fine"). Its expansion is recorded nowhere in this codebase, so a + consumer could not resolve it even in principle. +- AMP is a label for who uses the layer, not a filter on what is in it — the + row set is every water well, with no group predicate. A name carrying `amp` + would claim a scope the data does not have, which is its own Don't. +- Group B analytic layers prefer a `water_well_` prefix over a `_wells` + suffix, and this is one. + +The AMP crews remain the layer's audience; that belongs in the description, +which says so, rather than in an identifier that is a long-term commitment. + +A new OGC API - Features collection, `water_well_field_operations`, published on +`/ogcapi-internal` only. One feature per water well, carrying everything a +field crew needs to plan and execute a visit: where the well is, who owns it +and how to reach them, what the crew is allowed to do there, what is installed +in it, and when it was last measured. + +Every other well layer in the catalogue answers a scientific question — what is +the water level, what is the chemistry, what is the trend. This one answers an +operational one: **can we go to this well, what are we allowed to do when we +get there, and is it overdue?** + +## 1. Why a new layer rather than a column pass on an existing one + +`water_wells` is the construction register and is public. `water_well_summary` +and `actively_monitored_wells` summarise the measurement record and are public. +The operational fields — landowner phone numbers, access notes, sampling +permissions — cannot go on any of those, because those layers are published +anonymously. The split is a publication boundary, not a modelling preference. + +## 2. Scope + +**In:** water wells. `thing_type = 'water well'`, the same predicate every +other well layer in this catalogue uses. Springs, streams, and met stations are +out even though crews visit them — this is the wells layer, per the request. + +`thing_type` in the lexicon also carries `observation well`, `piezometer`, +`monitoring well`, `production well`, `injection well`, `exploration well`, +`test well`, `abandoned well`, `dry hole`, `artesian well`, and `dug well`. +None of them are included. That matches `ogc_water_wells`, +`ogc_water_well_summary`, and `ogc_well_water_column`, so the layers cannot +disagree about which rows are "wells" — see §11, open question 1. + +**Row set:** every water well with a current location, including wells with no +measurement record at all. A well nobody has ever measured is exactly the well +a field crew needs to find, so this layer must not inherit +`water_well_summary`'s `total_water_levels > 0` restriction. + +**Release status:** unfiltered. The internal mount is unfiltered by design and +this layer has no public twin, so there is no `release_status = 'public'` +predicate anywhere in it. `release_status` is published as a column so a +consumer can see what they are looking at. + +## 3. Placement: internal-only, no public counterpart + +Every other `ogc_internal_*` relation is the unfiltered twin of a public +`ogc_*` one. This layer deliberately breaks that: `ogc_internal_water_well_field_operations` +exists and `ogc_water_well_field_operations` does not, and must never be created. + +This does not disturb the existing parity guard. +`tests/test_migration_view_parity.py::test_internal_migration_mirrors_every_public_relation` +reads two named migration files (`f4a5b6c7d8e9` and `2d3c3a268652`) and asserts +that internal file mirrors that public file's 24 relations. A relation created +by a *new* migration is outside what that test reads, so the count assertion +stays at 24 and stays true. + +Precedent for an internal-only collection: `locations`, `avg_tds_wells`, +`latest_depth_to_water_wells` (static entries in the internal config template +only) and `other_things` (`internal_only: True` in `THING_COLLECTIONS`). + +Because this layer is not a thing-type register, it is wired the same way +`locations` is: a static block in `core/pygeoapi-config-internal.yml`, not an +entry in `THING_COLLECTIONS`. + +## 4. The PII decision — read this before implementing + +This layer publishes landowner and operator contact details: name, +organization, role, phone number, email address. That is what "contact info" +means for a field crew, and it is why the layer is internal-only. + +The consequence is not confined to this layer. `/ogcapi-internal` is reachable +two ways on this branch (`core/internal_ogc_auth.py`): an Authentik JWT +carrying `OGCInternal`, or a static API key from `INTERNAL_OGC_API_KEYS`. +**Shipping this layer means every one of those credentials becomes a credential +that dispenses landowner personal information**, and any credential path added +later -- the user-issued keys on `feat/api-key-management` among them -- +inherits the same reach. The static desktop-GIS keys in particular are shared +secrets that can only be revoked by redeploy. + +pygeoapi has no per-field authorization — a column is either in the view or it +is not — so this is all-or-nothing per layer. Three options: + +| Option | What crews get | Cost | +| --- | --- | --- | +| **A. Full contact detail** (recommended) | Name, organization, role, phone, email | Every internal credential now dispenses PII | +| B. Presence only | `contact_count`, `primary_contact_organization`, `primary_contact_role` — no name, phone, or email | Crews must look the owner up in the UI before every visit; layer half-fails its purpose | +| C. Second layer | `water_well_field_operations` without PII, `water_well_field_contacts` with it | Two layers, two configs, two sets of field docs, same credentials still reach both — the split buys nothing without per-layer auth | + +**Decision: A.** The request was explicit, the mount is authenticated, and B +makes the layer not worth building. Shipped with: + +- A notice at the top of `docs/internal-ogc-desktop-gis.md` stating that an + internal credential now conveys landowner PII, so key issuance is a decision + made with that in mind. **`docs/api-key-management.md` still needs the same + notice** — that file lives on `feat/api-key-management` and does not exist on + this branch, so it could not be edited here. +- The collection `description` says so too, so it is visible in the catalogue + itself. +- The layer does not filter contacts on `release_status`. `Contact` carries one, + but the internal mount is unfiltered by design and this layer has no public + twin to keep in step with. + +## 5. Relations: one matview, one view + +Two relations, not one, because the columns divide cleanly by cost *and* by how +badly staleness hurts: + +``` +ogc_internal_water_well_field_operations_stats MATERIALIZED VIEW, keyed thing_id + counts, first/last dates, aggregates over observation and + transducer_observation. Expensive. Refreshed nightly. + +ogc_internal_water_well_field_operations VIEW <- pygeoapi points here + live join of thing, location, status_history, permission_history, + deployment, sensor, contact, notes + LEFT JOIN the stats matview +``` + +Reasons for the split: + +1. **Staleness is dangerous on exactly the cheap columns.** "May we sample this + well" and "is access currently granted" must not be up to 24 hours old. A + revoked permission that still reads `true` until the next nightly refresh + sends a crew onto land they are no longer welcome on. `permission_history` + and `status_history` are small and indexed by `(target_id, target_table)`; + reading them live costs nothing. +2. **`CURRENT_DATE` freezes in a matview.** The current-record rule in §7 is + date-relative. Evaluated inside a materialized view it would be pinned to + the refresh time, and a permission that expired this morning would still + read as current tonight. It has to be in the plain view. +3. **The expensive columns don't need to be fresh.** A reading count and a + last-measured date are fine at 24 hours old. + +Plain views already back the eleven thing-type collections, so pygeoapi serving +a non-materialized view is established. Filtering and bbox pushdown resolve +against the base tables' indexes. + +`_stats` carries a unique index on `thing_id` so it can be refreshed +`CONCURRENTLY` (`oco refresh-matview --concurrently`). The nightly pg_cron job +discovers every matview in the `public` schema by name, so it is picked up with +no schedule change. Add it to `MATERIALIZED_VIEWS` in +`services/materialized_views.py` for the CLI path. + +## 6. Columns + +`id` is `thing.id` and is unique — pygeoapi's `id_field: id` requires exactly +one row per id for `/items/{id}`. Every one-to-many is therefore aggregated. + +Multi-valued columns are emitted as **`text`, comma-space joined**, not as +Postgres arrays. `ogc_actively_monitored_wells` uses `text[]` and that is fine +for a layer read in a browser, but this one exists to be pulled into ArcGIS Pro +and QGIS and exported to a File Geodatabase or GeoPackage for offline field +use, and neither format has a list type. A joined string survives the round +trip; an array does not. + +### Identity and location + +| Column | Source | +| --- | --- | +| `id` | `thing.id` | +| `name` | `thing.name` | +| `thing_type` | `'water well'::text` | +| `release_status` | `thing.release_status` | +| `nma_pk_welldata` | `thing.nma_pk_welldata` — legacy NM_Aquifer key, still what crews say out loud | +| `alternate_ids` | `thing_id_link`, joined as `organization:alternate_id` pairs | +| `county`, `quad_name`, `state` | `location` | +| `latitude`, `longitude` | `ST_Y`/`ST_X` of the same point, decimal degrees on WGS 84 -- a plain number survives a CSV export, which drops the geometry, and a crew can read it into a handheld GPS | +| `elevation` | `location.elevation` | +| `elevation_method` | `data_provenance.collection_method` for that location's elevation -- the same lookup `water_well_summary` uses, so the two layers cannot disagree about how an elevation was obtained | +| `point` | `location.point`, most recent association (the shared `LATEST_LOCATION_CTE`) | + +### Well construction + +`well_depth`, `hole_depth`, `well_casing_diameter`, `well_casing_depth`, +`well_completion_date`, `well_driller_name`, `well_construction_method`, +`well_pump_type`, `well_pump_depth`, `formation_completion_code`, +`nma_formation_zone` — all straight from `thing`, same columns and same names +as the thing-view template, so a crew reading both layers sees one vocabulary. + +Plus, aggregated: + +| Column | Source | +| --- | --- | +| `well_purposes` | `well_purpose.purpose`, joined | +| `well_casing_materials` | `well_casing_material.material`, joined | +| `screen_count` | `count(well_screen)` | +| `screen_depth_top`, `screen_depth_bottom` | `min`/`max` across that well's screens, ft below ground surface | + +### Measuring point (current record only) + +| Column | Source | +| --- | --- | +| `measuring_point_height` | `measuring_point_history.measuring_point_height`, ft above ground surface | +| `measuring_point_description` | e.g. "North side of casing, top of PVC" | +| `measuring_point_start_date` | when the current configuration took effect | + +The description is the single most useful string on the layer for a crew +standing at a wellhead. Current record per §7. + +### Status (current record per status type) + +`status_history.status_type` has five values in the lexicon, and all five are +published, each as `` plus `_since`: + +| Column | `status_type` | Values | +| --- | --- | --- | +| `well_status` | Well Status | Abandoned; Active, pumping well; Destroyed, exists but not usable; Inactive, exists but not used | +| `monitoring_status` | Monitoring Status | Currently monitored; Not currently monitored | +| `access_status` | Access Status | (lexicon does not yet scope values to type — open question 3) | +| `open_status` | Open Status | Open; Open (unequipped); Closed | +| `datalogger_suitability_status` | Datalogger Suitability Status | Datalogger can be installed; Datalogger cannot be installed | + +`monitoring_status_reason` is also published — `status_history.reason`, which +is where "landowner asked us to stop" is written down. + +### Permission (current grants) + +From `permission_history`, whose `permission_type` lexicon is exactly the three +things a crew does: + +| Column | `permission_type` | +| --- | --- | +| `may_measure_water_level` | Water Level Sample | +| `may_sample_water_chemistry` | Water Chemistry Sample | +| `may_install_datalogger` | Datalogger Installation | + +**These are three-valued and must stay that way.** `true` = a current grant +says allowed. `false` = a current grant says *not* allowed. `NULL` = **no +permission on record**, which is not the same as denied and must not be +rendered as "no". Collapsing NULL to false would tell a crew a well is off +limits when the truth is that nobody has asked yet; collapsing it to true is +worse. Every consumer-facing rendering of these columns has to carry the third +state. + +`permission_granted_by` — the granting contact's name from the current +water-level grant — is published alongside, so a crew can name who said yes. + +### Monitoring programme + +| Column | Source | +| --- | --- | +| `monitoring_frequency` | current `monitoring_frequency_history` record | +| `monitoring_frequency_since` | its `start_date` | +| `group_names` | `group` via `group_thing_association`, joined, de-duplicated | +| `group_types` | Monitoring Plan / Geographic Area / Historical, joined, index-aligned with `group_names` | + +### Manual measurement statistics + +Manual groundwater levels, reached the way every other water-level view in this +schema reaches them: `observation -> sample -> field_activity -> field_event`, +`field_activity.activity_type = 'groundwater level'`. + +| Column | Definition | +| --- | --- | +| `manual_water_level_count` | count of readings with a value and a timestamp | +| `manual_water_level_first_date` | earliest `observation_datetime`, UTC calendar date | +| `manual_water_level_last_date` | latest, UTC calendar date | +| `days_since_manual_water_level` | `CURRENT_DATE - manual_water_level_last_date`, computed in the plain view so it does not freeze at refresh | +| `last_depth_to_water_ft` | the latest reading, below ground surface | + +`last_depth_to_water_ft` uses `(o.value - COALESCE(o.measuring_point_height, 0))` +— the reading minus the height of the measuring point above ground, with a +missing height treated as taken at ground level. This is the exact convention +in `ogc_water_well_summary`, `ogc_latest_depth_to_water_wells`, and +`ogc_well_water_column`, and it must stay identical or the four layers will +disagree about what a depth to water is. See +`docs/measuring-point-height-null-handling.md`. + +### Chemistry sampling statistics + +Same chain, `field_activity.activity_type = 'water chemistry'`: +`chemistry_sample_count`, `chemistry_sample_last_date`, +`days_since_chemistry_sample`. Sample-level, not analyte-level — a crew wants +"when was this well last sampled", not "what was the sulfate". + +### Field visit statistics + +`field_event_count`, `field_event_last_date` from `field_event`. Broader than +either measurement chain: a visit that produced no reading is still a visit, +and the gap between `field_event_last_date` and +`manual_water_level_last_date` is itself a signal. + +### Data logger / continuous record + +"Has a logger" is a deployment question, not a sensor-inventory question. A +deployment is current when `installation_date IS NOT NULL` and `removal_date IS +NULL`, and the sensor is one of `Data Logger`, `Pressure Transducer`, +`DiverLink`, `Diver Cable` (`sensor.sensor_type`). + +| Column | Definition | +| --- | --- | +| `has_datalogger` | boolean, true when a current logger deployment exists | +| `datalogger_sensor_type`, `datalogger_model`, `datalogger_serial_no` | from `sensor` on the current deployment | +| `datalogger_sensor_status` | In Service / In Repair / Retired / Lost | +| `datalogger_installed_date` | `deployment.installation_date` | +| `datalogger_recording_interval`, `datalogger_recording_interval_units` | how often it logs | +| `datalogger_hanging_point_description` | where the cable hangs from — a field instruction | +| `continuous_reading_count` | rows in `transducer_observation` for this well's deployments | +| `continuous_first_datetime`, `continuous_last_datetime` | series extent | +| `days_since_continuous_reading` | live, in the plain view | + +Where a well has more than one current logger deployment, the singular +`datalogger_*` columns take the most recently installed one, and +`datalogger_deployment_count` says how many there are so nothing is silently +hidden. + +`continuous_reading_count` aggregates the largest table in the schema. It is +the reason `_stats` is materialized. If the refresh proves too slow, the +fallback is to source the three continuous columns from `transducer_daily_data` +instead — cheaper, one row per day, and adequate for "is the logger still +reporting". + +### Contacts + +Subject to §4. + +| Column | Definition | +| --- | --- | +| `contact_count` | contacts associated with the well | +| `primary_contact_name`, `_organization`, `_role` | prefer the `contact_type = 'Primary'` contact, else any contact on the well, lowest `contact.id` as tie-break | +| `primary_contact_type` | which of those two cases produced the row above | +| `primary_contact_phone` | lowest-id `phone.phone_number` for that contact | +| `primary_contact_email` | lowest-id `email.email` for that contact | +| `contact_names` | every associated contact, joined | + +The fallback is a change from this document's first draft, which took the +primary contact strictly. A well whose only contact is recorded as Secondary +would then show `contact_count = 1` beside a blank name and phone — the exact +failure mode that ruled out option B in §4. `primary_contact_type` keeps the +fallback visible rather than implied. + +Deliberately **not** published: `address`. A mailing address is not how a crew +reaches a landowner, and it is the most sensitive field on the record. + +### Notes + +`notes` is polymorphic on `(target_id, target_table = 'thing')` with a +`note_type` lexicon. Two types are operational and are published as their own +columns, most recent first, joined: + +| Column | `note_type` | +| --- | --- | +| `access_notes` | Access | +| `directions_notes` | Directions | + +The rest (Construction, Maintenance, Historical, General, Water, Water Quality, +Sampling Procedure, Coordinate, OwnerComment, Site Notes (legacy)) stay out. +They are a well's history, not its visit instructions, and free text is where +PII leaks — see open question 4. + +## 7. The "current record" rule + +Four histories feed this layer — `status_history`, `permission_history`, +`measuring_point_history`, `monitoring_frequency_history` — and all four have +the same `(start_date, end_date)` shape. One rule for all of them: + +```sql +WHERE h.start_date <= CURRENT_DATE + AND (h.end_date IS NULL OR h.end_date >= CURRENT_DATE) +ORDER BY h.start_date DESC, h.id DESC +LIMIT 1 -- via DISTINCT ON per thing +``` + +**This diverges from the existing catalogue on purpose.** +`ogc_actively_monitored_wells` takes the row with the greatest `start_date` and +ignores `end_date` entirely, so a monitoring status that was closed in 2019 +still reads as current there. That is tolerable on a summary layer. It is not +tolerable on a layer whose whole job is to tell a crew what is true today — +"you may install a logger here" must not survive the expiry of the permission +that said so. + +The divergence has to be written into `docs/ogc_conventions.md`, or the next +person will read the two views side by side and assume one is a mistake. + +Rows whose window has not opened yet (`start_date > CURRENT_DATE`) are not +current and read NULL. + +## 8. Field-level documentation + +`core/ogc_field_metadata.py` keys `core/ogc-field-descriptions.yml` by backing +relation with the `ogc_internal_` prefix stripped, so the entry is +`water_well_field_operations`. This is not optional: +`tests/test_ogc_field_descriptions.py::test_every_published_column_has_an_entry` +walks both mounts and fails on any published column without a `description`. + +Columns already in `_defaults` (`id`, `name`, `thing_type`, `release_status`, +`elevation`, `well_depth`, `hole_depth`, `well_casing_diameter`, …) are +inherited and must not be restated. Everything new to this layer needs an +entry, and the entry says what the value *means*, never how the view is built. + +Controlled vocabularies use `enum-lexicon` rather than a duplicated `enum` +list: `status_value` for the five status columns, `monitoring_frequency`, +`sensor_type`, `sensor_status`, `well_purpose`, `casing_material`, `role`, +`group_type`. Depth and height columns carry +`x-ogc-unit: https://qudt.org/vocab/unit/FT` with +`x-ogc-unitLang: QUDT`, matching the rest of the file. + +The three permission booleans need their NULL meaning spelled out in the +`description` — that is the only place a consumer will ever read it. + +## 9. Wiring + +1. **Migration** — new Alembic revision on the current head (`c9d0e1f2a3b4`), + creating the matview, its unique index, and the view, plus the supporting + indexes in §10. `downgrade()` drops all three; nothing pre-existing is + modified, so there is no prior state to restore. Follow the house style in + `2d3c3a268652`: self-contained, no cross-migration imports, `REQUIRED_TABLES` + guard up front. +2. **`core/pygeoapi-config-internal.yml`** — one static resource block, next to + `locations`. `name: core.feature_provider.DescribedPostgreSQLProvider`, + `id_field: id`, `table: ogc_internal_water_well_field_operations`, + `geom_field: point`. No `time_field`: no single column is the feature's + time, and picking one would make `datetime=` filtering quietly mean + something arbitrary. +3. **`core/pygeoapi-config.yml`** — no change. Ever. +4. **`core/ogc-field-descriptions.yml`** — the `water_well_field_operations` block. +5. **`services/materialized_views.py`** — append + `ogc_internal_water_well_field_operations_stats` to `MATERIALIZED_VIEWS`. +6. **`core/gis-curated-layers.yml`** — *no* entry. Every layer in that file + ships as a `.qlr` and a `.lyrx` artifact, and this one is PII-bearing and + internal; it should be reached by browsing the authenticated connection, not + handed out as a downloadable file. Revisit only if crews ask. +7. **Docs** — `docs/api-key-management.md` and `docs/internal-ogc-desktop-gis.md` + gain the PII notice from §4; `docs/ogc_conventions.md` gains the §7 + divergence. + +Description text has to clear +`tests/test_pygeoapi_mount.py::test_every_collection_description_explains_the_layer`: +at least 200 characters, ends in a full stop, no placeholder words, no +hyphenated word split across a YAML line break, and at least four unique +lowercase-hyphenated keywords. + +## 10. Indexes + +The per-thing lookups this layer adds are foreign-key joins, and Postgres does +not index foreign keys on its own. Check each before creating it — `b8c9d0e1f2a3` +already added four on the observation chain. + +- `status_history (target_table, target_id, status_type, start_date DESC)` +- `permission_history (target_table, target_id, permission_type, start_date DESC)` +- `measuring_point_history (thing_id, start_date DESC)` +- `monitoring_frequency_history (thing_id, start_date DESC)` +- `notes (target_id, target_table)` — already exists as `ix_notes_polymorphic_link` +- `deployment (thing_id)` +- `thing_contact_association (thing_id)`, `(contact_id)` +- `transducer_observation (deployment_id, observation_datetime)` — for the + continuous aggregates; the largest table in the schema and the one that will + decide whether the nightly refresh is acceptable + +## 11. Settled, and still open + +Settled before implementation: + +- **"AMP" is a label, not a subset.** The layer covers every water well; there + is no group predicate. +- **Contact detail is published in full** (§4, option A). + +Still open: + +1. **Which `thing_type` values count as a well?** Shipped as `'water well'` + only, for consistency with every existing well layer. If AMP crews also + visit piezometers and monitoring wells, the predicate widens to a list — and + then it should widen in the other four layers too, as its own ticket, rather + than this layer quietly disagreeing with them. +2. **Access Status has no values.** `status_type` carries `Access Status` but + the `status_value` lexicon has no access-related terms — the eleven values + map to Well, Monitoring, Open, and Datalogger Suitability status. The column + is published and will read null until the vocabulary gains terms scoped to + it; the field description says so. Either access statuses are recorded with + values from another category, or the column is empty in practice, and the + answer needs a look at production data. +3. **Free-text PII.** `access_notes` and `directions_notes` are staff-written + and will contain phone numbers, gate codes, and names. A further reason the + layer is internal-only, and the §4 decision covers them too. +4. **Refresh cadence.** Nightly, via the existing pg_cron job. If crews plan + the next day's route the evening before, nightly is fine; if they re-plan in + the field, the stats columns are the ones that go stale. +5. **`continuous_reading_count` cost at production scale.** It aggregates the + largest table in the schema. If the nightly refresh proves too slow, the + fallback is to source the three continuous columns from + `transducer_daily_data` instead. + +## 12. Out of scope + +- Any public counterpart. +- Write access. This is a read layer; edits happen in the Ocotillo UI. +- A curated `.qlr` / `.lyrx` artifact (§9.6). +- Springs and other non-well monitoring points. +- Per-field authorization. pygeoapi has no such hook; the layer boundary is the + authorization boundary. diff --git a/services/materialized_views.py b/services/materialized_views.py index 9b7e3740..75c5239d 100644 --- a/services/materialized_views.py +++ b/services/materialized_views.py @@ -19,5 +19,9 @@ # two pivot views above, at per-result grain. "ogc_water_chemistry", "ogc_internal_water_chemistry", + # Field-operations aggregates for the internal-only + # ogc_internal_water_well_field_operations layer (e1f2a3b4c5d6). The + # feature view itself is a plain view and needs no refresh. + "ogc_internal_water_well_field_operations_stats", "transducer_daily_data", ) diff --git a/tests/test_cli_commands.py b/tests/test_cli_commands.py index 97534a60..9923fb38 100644 --- a/tests/test_cli_commands.py +++ b/tests/test_cli_commands.py @@ -72,10 +72,11 @@ def __exit__(self, exc_type, exc, tb): "REFRESH MATERIALIZED VIEW ogc_minor_chemistry_wells", "REFRESH MATERIALIZED VIEW ogc_water_chemistry", "REFRESH MATERIALIZED VIEW ogc_internal_water_chemistry", + "REFRESH MATERIALIZED VIEW ogc_internal_water_well_field_operations_stats", "REFRESH MATERIALIZED VIEW transducer_daily_data", ] assert commit_called["value"] is True - assert "Refreshed 10 materialized view(s)." in result.output + assert "Refreshed 11 materialized view(s)." in result.output def test_refresh_materialized_views_custom_and_concurrently( diff --git a/tests/test_ogc.py b/tests/test_ogc.py index 18c5fea7..5d6d3c83 100644 --- a/tests/test_ogc.py +++ b/tests/test_ogc.py @@ -818,12 +818,15 @@ def test_ogc_collections(ogc_client): # and latest_depth_to_water_wells repeats water_well_summary # (BDMS-977), and other_things is internal vocabulary (BDMS-979). The # backing relations are retained and still served on /ogcapi-internal. + # water_well_field_operations is internal-only for a stronger reason than + # the other four: it publishes landowner contact details. assert ids.isdisjoint( { "locations", "avg_tds_wells", "latest_depth_to_water_wells", "other_things", + "water_well_field_operations", } ) diff --git a/tests/test_ogc_water_well_field_operations.py b/tests/test_ogc_water_well_field_operations.py new file mode 100644 index 00000000..a2b65541 --- /dev/null +++ b/tests/test_ogc_water_well_field_operations.py @@ -0,0 +1,603 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""The internal-only water well field operations layer (e1f2a3b4c5d6). + +What is worth testing here is not that the columns exist but that the layer's +three load-bearing rules hold: a well with no measurements still appears, a +permission with no record reads NULL rather than false, and a history record +whose window has closed is not treated as current. + +See docs/water-well-field-operations-layer.md. +""" + +from datetime import date, datetime, timedelta +from importlib.util import find_spec + +import pytest +from fastapi.testclient import TestClient +from sqlalchemy import text + +from core.dependencies import ( + admin_function, + amp_admin_function, + amp_editor_function, + amp_viewer_function, + editor_function, + viewer_function, +) +from core.factory import create_api_app +from db import ( + DataProvenance, + Deployment, + Notes, + PermissionHistory, + Sensor, + StatusHistory, +) +from db.engine import session_ctx +from tests import override_authentication + +pytestmark = pytest.mark.skipif( + find_spec("pygeoapi") is None, + reason="pygeoapi is not installed in this environment", +) + +VIEW = "ogc_internal_water_well_field_operations" +STATS_VIEW = "ogc_internal_water_well_field_operations_stats" + + +@pytest.fixture(scope="module") +def today(): + """The database's idea of today, not Python's. + + The view computes its `days_since_*` columns against CURRENT_DATE, which is + evaluated in the database session's timezone. That is UTC here while the + developer running the tests may not be, so a date read from Python is off + by one for part of every day. Read it from the same place the view does. + """ + with session_ctx() as session: + return session.execute(text("SELECT CURRENT_DATE")).scalar() + + +@pytest.fixture(scope="module") +def yesterday(today): + return today - timedelta(days=1) + + +@pytest.fixture(scope="module") +def last_year(today): + return today - timedelta(days=365) + + +@pytest.fixture(scope="module") +def ogc_client(): + app = create_api_app() + for dependency in ( + admin_function, + editor_function, + amp_admin_function, + amp_editor_function, + ): + app.dependency_overrides[dependency] = override_authentication( + default={"name": "foobar", "sub": "1234567890"} + ) + for dependency in (viewer_function, amp_viewer_function): + app.dependency_overrides[dependency] = override_authentication() + + with TestClient(app) as client: + yield client + + app.dependency_overrides = {} + + +def _row(session, thing_id, columns): + return session.execute( + text(f"SELECT {columns} FROM {VIEW} WHERE id = :thing_id"), + {"thing_id": thing_id}, + ).one() + + +def _refresh_stats(session): + session.execute(text(f"REFRESH MATERIALIZED VIEW {STATS_VIEW}")) + session.commit() + + +# ------------------------------------------------------------------ row set + + +def test_a_well_with_no_measurements_still_appears(water_well_thing): + # water_well_summary drops wells with no readings, because a summary of + # nothing says nothing. This layer must not: a well nobody has measured is + # exactly the well a crew needs to find. + with session_ctx() as session: + _refresh_stats(session) + row = _row( + session, + water_well_thing.id, + "name, manual_water_level_count, chemistry_sample_count, " + "continuous_reading_count, days_since_manual_water_level", + ) + + assert row.name == "Test Well" + assert row.manual_water_level_count == 0 + assert row.chemistry_sample_count == 0 + assert row.continuous_reading_count == 0 + # No reading means no elapsed time to report, not zero days since one. + assert row.days_since_manual_water_level is None + + +def test_latitude_and_longitude_match_the_geometry(water_well_thing): + with session_ctx() as session: + row = session.execute( + text( + "SELECT latitude, longitude, ST_Y(point) AS geom_y, " + f"ST_X(point) AS geom_x FROM {VIEW} WHERE id = :thing_id" + ), + {"thing_id": water_well_thing.id}, + ).one() + + # Decimal degrees on WGS 84, read off the same point the geometry carries. + assert row.latitude == row.geom_y + assert row.longitude == row.geom_x + assert -180 <= row.longitude <= 180 + assert -90 <= row.latitude <= 90 + + +def test_elevation_method_reads_the_location_provenance(water_well_thing, location): + with session_ctx() as session: + row = _row(session, water_well_thing.id, "elevation_method") + # No provenance record is "not recorded", not a method of "unknown". + assert row.elevation_method is None + + provenance = DataProvenance( + target_id=location.id, + target_table="location", + field_name="elevation", + collection_method="Survey-grade GPS", + ) + session.add(provenance) + session.commit() + + row = _row(session, water_well_thing.id, "elevation_method") + assert row.elevation_method == "Survey-grade GPS" + + session.delete(provenance) + session.commit() + + +def test_elevation_method_ignores_provenance_for_other_fields( + water_well_thing, location +): + with session_ctx() as session: + provenance = DataProvenance( + target_id=location.id, + target_table="location", + field_name="point", + collection_method="Survey-grade GPS", + ) + session.add(provenance) + session.commit() + + row = _row(session, water_well_thing.id, "elevation_method") + # How the coordinates were obtained says nothing about the elevation. + assert row.elevation_method is None + + session.delete(provenance) + session.commit() + + +# --------------------------------------------------------------- permissions + + +def test_permission_with_no_record_is_null_not_false(water_well_thing): + with session_ctx() as session: + row = _row( + session, + water_well_thing.id, + "may_measure_water_level, may_sample_water_chemistry, " + "may_install_datalogger", + ) + + # NULL means nobody has asked the landowner. Rendering it as "no" would + # tell a crew the well is off limits when the truth is unknown. + assert row.may_measure_water_level is None + assert row.may_sample_water_chemistry is None + assert row.may_install_datalogger is None + + +def test_permission_distinguishes_granted_from_refused( + water_well_thing, contact, last_year +): + with session_ctx() as session: + granted = PermissionHistory( + contact_id=contact.id, + target_id=water_well_thing.id, + target_table="thing", + permission_type="Water Level Sample", + permission_allowed=True, + start_date=last_year, + ) + refused = PermissionHistory( + contact_id=contact.id, + target_id=water_well_thing.id, + target_table="thing", + permission_type="Datalogger Installation", + permission_allowed=False, + start_date=last_year, + ) + session.add_all([granted, refused]) + session.commit() + + row = _row( + session, + water_well_thing.id, + "may_measure_water_level, may_install_datalogger, " + "may_sample_water_chemistry, permission_granted_by", + ) + + assert row.may_measure_water_level is True + assert row.may_install_datalogger is False + # Untouched permission type stays unknown rather than inheriting either. + assert row.may_sample_water_chemistry is None + assert row.permission_granted_by == contact.name + + session.delete(granted) + session.delete(refused) + session.commit() + + +def test_expired_permission_is_not_current( + water_well_thing, contact, last_year, yesterday +): + with session_ctx() as session: + expired = PermissionHistory( + contact_id=contact.id, + target_id=water_well_thing.id, + target_table="thing", + permission_type="Water Level Sample", + permission_allowed=True, + start_date=last_year, + end_date=yesterday, + ) + session.add(expired) + session.commit() + + row = _row(session, water_well_thing.id, "may_measure_water_level") + # A permission that ran out yesterday is not a permission today. This + # is the divergence from ogc_actively_monitored_wells, which ignores + # end_date entirely. + assert row.may_measure_water_level is None + + session.delete(expired) + session.commit() + + +# -------------------------------------------------------------------- status + + +def test_status_reads_the_current_record_not_the_latest( + water_well_thing, last_year, yesterday +): + with session_ctx() as session: + closed = StatusHistory( + target_id=water_well_thing.id, + target_table="thing", + status_type="Monitoring Status", + status_value="Currently monitored", + start_date=last_year, + end_date=yesterday, + reason="programme ended", + ) + open_status = StatusHistory( + target_id=water_well_thing.id, + target_table="thing", + status_type="Well Status", + status_value="Active, pumping well", + start_date=last_year, + ) + session.add_all([closed, open_status]) + session.commit() + + row = _row( + session, + water_well_thing.id, + "monitoring_status, monitoring_status_since, well_status, " + "well_status_since", + ) + + assert row.monitoring_status is None + assert row.monitoring_status_since is None + assert row.well_status == "Active, pumping well" + assert row.well_status_since == last_year + + session.delete(closed) + session.delete(open_status) + session.commit() + + +def test_status_types_do_not_bleed_into_each_other(water_well_thing, last_year): + with session_ctx() as session: + monitoring = StatusHistory( + target_id=water_well_thing.id, + target_table="thing", + status_type="Monitoring Status", + status_value="Not currently monitored", + start_date=last_year, + reason="landowner asked us to stop", + ) + session.add(monitoring) + session.commit() + + row = _row( + session, + water_well_thing.id, + "monitoring_status, monitoring_status_reason, well_status, " + "open_status, access_status, datalogger_suitability_status", + ) + + assert row.monitoring_status == "Not currently monitored" + assert row.monitoring_status_reason == "landowner asked us to stop" + assert row.well_status is None + assert row.open_status is None + assert row.access_status is None + assert row.datalogger_suitability_status is None + + session.delete(monitoring) + session.commit() + + +# ---------------------------------------------------------------- datalogger + + +def test_has_datalogger_counts_only_logger_equipment( + water_well_thing, sensor_to_water_well_thing_deployment +): + with session_ctx() as session: + row = _row( + session, + water_well_thing.id, + "has_datalogger, datalogger_deployment_count, " + "datalogger_sensor_type, datalogger_serial_no, " + "datalogger_recording_interval", + ) + + assert row.has_datalogger is True + assert row.datalogger_deployment_count == 1 + assert row.datalogger_sensor_type == "Pressure Transducer" + assert row.datalogger_serial_no == "123456" + assert row.datalogger_recording_interval == 24 + + +def test_non_logger_equipment_does_not_make_a_well_instrumented( + water_well_thing, last_year +): + with session_ctx() as session: + barometer = Sensor( + name="Test Barometer", + sensor_type="Barometer", + sensor_status="In Service", + release_status="draft", + ) + session.add(barometer) + session.commit() + deployment = Deployment( + sensor_id=barometer.id, + thing_id=water_well_thing.id, + installation_date=last_year, + removal_date=None, + ) + session.add(deployment) + session.commit() + + row = _row( + session, + water_well_thing.id, + "has_datalogger, datalogger_deployment_count", + ) + # A barometer at the well is equipment, not a logger in the well. + assert row.has_datalogger is False + assert row.datalogger_deployment_count == 0 + + session.delete(deployment) + session.delete(barometer) + session.commit() + + +def test_a_removed_logger_leaves_the_well_uninstrumented( + water_well_thing, sensor, sensor_to_water_well_thing_deployment, yesterday +): + with session_ctx() as session: + deployment = session.get(Deployment, sensor_to_water_well_thing_deployment.id) + deployment.removal_date = yesterday + session.commit() + + row = _row(session, water_well_thing.id, "has_datalogger") + assert row.has_datalogger is False + + deployment.removal_date = None + session.commit() + + +# ------------------------------------------------------------- measurements + + +def test_last_depth_to_water_uses_the_measuring_point_convention( + water_well_thing, groundwater_level_sample, today +): + from db import Observation + from tests import get_parameter_id + + with session_ctx() as session: + readings = [] + for day, value, measuring_point_height in ((1, 6.0, 1.0), (2, 9.0, 2.0)): + observation = Observation( + observation_datetime=datetime(2025, 1, day, 12, 0, 0), + sample_id=groundwater_level_sample.id, + parameter_id=get_parameter_id("groundwater level", "Field Parameter"), + release_status="public", + value=value, + unit="ft", + measuring_point_height=measuring_point_height, + groundwater_level_reason="Water level not affected", + ) + session.add(observation) + readings.append(observation) + session.commit() + _refresh_stats(session) + + row = _row( + session, + water_well_thing.id, + "manual_water_level_count, manual_water_level_first_date, " + "manual_water_level_last_date, last_depth_to_water_ft, " + "days_since_manual_water_level", + ) + + assert row.manual_water_level_count == 2 + assert row.manual_water_level_first_date == date(2025, 1, 1) + assert row.manual_water_level_last_date == date(2025, 1, 2) + # Latest reading is 9 ft from a measuring point 2 ft above ground. + assert abs(float(row.last_depth_to_water_ft) - 7.0) < 1e-9 + # Computed against today, not against the last refresh. + assert row.days_since_manual_water_level == (today - date(2025, 1, 2)).days + + for observation in readings: + session.delete(observation) + session.commit() + _refresh_stats(session) + + +def test_private_readings_are_counted_on_this_internal_layer( + water_well_thing, groundwater_level_sample +): + from db import Observation + from tests import get_parameter_id + + with session_ctx() as session: + observation = Observation( + observation_datetime=datetime(2025, 2, 1, 12, 0, 0), + sample_id=groundwater_level_sample.id, + parameter_id=get_parameter_id("groundwater level", "Field Parameter"), + release_status="private", + value=4.0, + unit="ft", + measuring_point_height=1.0, + groundwater_level_reason="Water level not affected", + ) + session.add(observation) + session.commit() + _refresh_stats(session) + + row = _row(session, water_well_thing.id, "manual_water_level_count") + # The internal mount is unfiltered by design; this layer has no public + # twin to keep in step with. + assert row.manual_water_level_count == 1 + + session.delete(observation) + session.commit() + _refresh_stats(session) + + +# ------------------------------------------------------ multi-valued columns + + +def test_multi_valued_columns_are_comma_joined_text( + water_well_thing, domestic_well_purpose, irrigation_well_purpose +): + with session_ctx() as session: + row = _row(session, water_well_thing.id, "well_purposes") + + # Text, not an array: this layer is exported to File Geodatabase and + # GeoPackage for offline field use, and neither format has a list type. + assert isinstance(row.well_purposes, str) + assert row.well_purposes == "Domestic, Irrigation" + + +def test_contacts_and_access_notes_are_published(water_well_thing, contact, phone): + with session_ctx() as session: + note = Notes( + target_id=water_well_thing.id, + target_table="thing", + note_type="Access", + content="Gate is locked, call ahead", + ) + session.add(note) + session.commit() + + row = _row( + session, + water_well_thing.id, + "contact_count, primary_contact_name, primary_contact_type, " + "primary_contact_role, primary_contact_phone, access_notes", + ) + + assert row.contact_count == 1 + assert row.primary_contact_name == contact.name + assert row.primary_contact_type == "Primary" + assert row.primary_contact_role == "Owner" + assert row.primary_contact_phone == phone.phone_number + assert row.access_notes == "Gate is locked, call ahead" + + session.delete(note) + session.commit() + + +# ------------------------------------------------------------------- mounts + + +def test_layer_is_served_on_the_internal_mount_only(ogc_client): + internal = ogc_client.get( + "/ogcapi-internal/collections/water_well_field_operations/items?limit=1" + ) + assert internal.status_code == 200 + assert internal.json()["type"] == "FeatureCollection" + + public = ogc_client.get( + "/ogcapi/collections/water_well_field_operations/items?limit=1" + ) + # The layer publishes landowner contact details; it must not exist at all + # on the anonymous mount. + assert public.status_code == 404 + + +def test_every_column_is_documented_on_the_internal_mount(ogc_client): + response = ogc_client.get( + "/ogcapi-internal/collections/water_well_field_operations/schema" + ) + assert response.status_code == 200 + + properties = response.json()["properties"] + gaps = [ + name + for name, prop in properties.items() + if name != "geometry" and not prop.get("description") + ] + assert not gaps, f"columns with no YAML entry: {gaps}" + + +def test_permission_columns_explain_their_null_meaning(ogc_client): + response = ogc_client.get( + "/ogcapi-internal/collections/water_well_field_operations/schema" + ) + properties = response.json()["properties"] + + # The three-valued meaning only reaches a consumer through this prose. + for column in ( + "may_measure_water_level", + "may_sample_water_chemistry", + "may_install_datalogger", + ): + assert "null" in properties[column]["description"].lower() diff --git a/tests/test_pygeoapi_mount.py b/tests/test_pygeoapi_mount.py index e267d180..2572874e 100644 --- a/tests/test_pygeoapi_mount.py +++ b/tests/test_pygeoapi_mount.py @@ -115,12 +115,15 @@ def test_loading_a_mount_restores_config_env_vars(): # Layers hidden from the public catalog but still served to staff GIS # clients on /ogcapi-internal: locations duplicates the thing-type layers # (BDMS-978), avg_tds_wells and latest_depth_to_water_wells are misleading -# or redundant (BDMS-977), other_things is internal vocabulary (BDMS-979). +# or redundant (BDMS-977), other_things is internal vocabulary (BDMS-979), +# and water_well_field_operations carries landowner contact details and +# staff-written access notes, so it has no public form at all. INTERNAL_ONLY_COLLECTIONS = { "locations", "avg_tds_wells", "latest_depth_to_water_wells", "other_things", + "water_well_field_operations", }