From 3499f41431402fb5199bc16ad8e65d37f5fe0043 Mon Sep 17 00:00:00 2001 From: jakeross Date: Mon, 31 Aug 2026 19:23:07 -0700 Subject: [PATCH 1/2] feat(ogc): add the water well field operations internal layer Adds water_well_field_operations, an OGC API - Features collection on /ogcapi-internal only: one feature per water well carrying what a field crew needs to plan and execute a visit. Every other well layer answers a scientific question -- water level, chemistry, trend. This one answers an operational one: may we go to this well, what are we allowed to do when we get there, and is it overdue? 86 columns: identity and location (including latitude and longitude in decimal degrees and how the elevation was obtained), construction detail, the current measuring point, all five status types, the three standing landowner permissions, monitoring frequency and group membership, manual water-level and chemistry-sampling statistics, field-visit history, the installed logger and its continuous record, landowner contacts, and access and directions notes. Two relations rather than one (migration e1f2a3b4c5d6): ogc_internal_water_well_field_operations_stats materialized, nightly refresh Aggregates over observation and transducer_observation. ogc_internal_water_well_field_operations plain view, served by pygeoapi Live join of thing, location, status_history, permission_history, measuring_point_history, monitoring_frequency_history, deployment, sensor, contact and notes, LEFT JOINed to the stats matview. The split is not tidiness. Staleness is dangerous on exactly the columns that are cheap to read: a revoked sampling permission that still reads true until the next nightly refresh sends a crew onto land they are no longer welcome on. The current-record rule is also written against CURRENT_DATE, which inside a materialized view would freeze at refresh time. Three decisions worth review: * Internal-only with no public twin. The layer publishes landowner and operator contact details -- name, organization, role, phone, email -- alongside staff-written access notes. ogc_water_well_field_operations does not exist and must never be created. Every credential the internal mount accepts now reaches personal information, the static desktop-GIS keys included; notices added to CLAUDE.md and docs/internal-ogc-desktop-gis.md. * The three permission columns are three-valued. NULL means no permission is on record, which is not a refusal -- it means nobody has asked the landowner yet. Collapsing it to false would tell a crew a well is off limits when the truth is unknown. * History reads honour end_date, unlike ogc_actively_monitored_wells, which takes the greatest start_date and lets a status closed in 2019 read as current. Written up in docs/ogc_conventions.md. Requested as amp_field_well_operations and renamed before merge per the pre-merge check in docs/ogc_conventions.md: amp is an unexpanded abbreviation whose expansion is recorded nowhere in this codebase, AMP labels the layer's audience rather than filtering its rows (the row set is every water well), and Group B analytic layers prefer a water_well_ prefix. The AMP crews stay named in the collection description. Multi-valued columns are comma-joined text rather than arrays: this layer is pulled into ArcGIS Pro and QGIS and exported to File Geodatabase and GeoPackage for offline field use, and neither format has a list type. Depth to water uses the (value - COALESCE(measuring_point_height, 0)) convention shared with ogc_water_well_summary, ogc_latest_depth_to_water_wells and ogc_well_water_column, and elevation_method reuses that summary's provenance lookup verbatim, so the layers cannot disagree. Twelve supporting indexes come with the migration; none of the foreign keys these lookups traverse was indexed. Design rationale in docs/water-well-field-operations-layer.md, column-by-column sources in docs/water-well-field-operations-columns.md. Co-Authored-By: Claude Opus 5 --- CLAUDE.md | 12 + ...6_add_water_well_field_operations_layer.py | 718 ++++++++++++++++++ core/ogc-field-descriptions.yml | 391 ++++++++++ core/pygeoapi-config-internal.yml | 44 ++ docs/internal-ogc-desktop-gis.md | 10 + docs/ogc_conventions.md | 27 + docs/water-well-field-operations-columns.md | 106 +++ docs/water-well-field-operations-layer.md | 519 +++++++++++++ services/materialized_views.py | 4 + tests/test_cli_commands.py | 9 +- tests/test_ogc.py | 3 + tests/test_ogc_water_well_field_operations.py | 603 +++++++++++++++ tests/test_pygeoapi_mount.py | 5 +- 13 files changed, 2447 insertions(+), 4 deletions(-) create mode 100644 alembic/versions/e1f2a3b4c5d6_add_water_well_field_operations_layer.py create mode 100644 docs/water-well-field-operations-columns.md create mode 100644 docs/water-well-field-operations-layer.md create mode 100644 tests/test_ogc_water_well_field_operations.py diff --git a/CLAUDE.md b/CLAUDE.md index a5f9358c7..bcc472bb1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -204,6 +204,18 @@ ArcGIS Pro cannot send a bearer token at all and neither desktop client can refresh an Authentik token. Read **`docs/internal-ogc-desktop-gis.md`** before changing the credential paths. +**`/ogcapi-internal` carries landowner PII.** The `water_well_field_operations` +collection publishes contact name, organization, role, phone and email for well +owners and operators, plus staff-written access notes. Every credential the +mount accepts reaches it, the static keys included. It is internal-only with +**no public twin** — `ogc_water_well_field_operations` does not exist and must +never be created. The layer also honours `end_date` when reading history +tables, unlike `ogc_actively_monitored_wells`, so "may we sample here" cannot +outlive the permission that granted it. Read +**`docs/water-well-field-operations-layer.md`** before changing it, and +**`docs/water-well-field-operations-columns.md`** for where each column comes +from. + ### OGC field descriptions Per-column `title`/`description`/unit for every collection lives in diff --git a/alembic/versions/e1f2a3b4c5d6_add_water_well_field_operations_layer.py b/alembic/versions/e1f2a3b4c5d6_add_water_well_field_operations_layer.py new file mode 100644 index 000000000..12276cf41 --- /dev/null +++ b/alembic/versions/e1f2a3b4c5d6_add_water_well_field_operations_layer.py @@ -0,0 +1,718 @@ +"""add the water well field operations layer + +An internal-only OGC layer for field crews: one row per water well, carrying +what a crew needs to plan and execute a visit -- where the well is, who owns it +and how to reach them, what the crew is permitted to do there, what is +installed in it, and when it was last measured. Every other well layer answers +a scientific question; this one answers an operational one. + +Two relations, not one: + + ogc_internal_water_well_field_operations_stats MATERIALIZED VIEW + Counts, first/last dates and aggregates over `observation` and + `transducer_observation`. Expensive, refreshed nightly by the pg_cron + job (b6c7d8e9f0a1), which discovers matviews from the catalog and so + needs no schedule change. + + ogc_internal_water_well_field_operations VIEW <- pygeoapi reads this + Live join of thing, location, status_history, permission_history, + measuring_point_history, monitoring_frequency_history, deployment, + sensor, contact and notes, LEFT JOINed to the stats matview. + +The split is not tidiness. Staleness is dangerous on exactly the columns that +are cheap to read: a revoked sampling permission that still reads `true` until +the next nightly refresh sends a crew onto land they are no longer welcome on. +And the current-record rule below is written against CURRENT_DATE, which in a +materialized view would freeze at refresh time -- a permission that expired +this morning would still read current tonight. Those columns have to be in the +plain view. + +Internal only, and deliberately without a public twin: the layer publishes +landowner contact details and staff-written access notes. +`ogc_water_well_field_operations` does not exist and must never be created. The +parity guard in tests/test_migration_view_parity.py reads two named migration +files (f4a5b6c7d8e9 and 2d3c3a268652) and is unaffected by a relation created +here. + +Current-record rule, applied identically to all four history tables +(status_history, permission_history, measuring_point_history, +monitoring_frequency_history): + + start_date <= CURRENT_DATE + AND (end_date IS NULL OR end_date >= CURRENT_DATE) + ORDER BY start_date DESC, id DESC -- via DISTINCT ON + +This diverges from ogc_actively_monitored_wells, which takes the greatest +start_date and ignores end_date entirely, so a status closed in 2019 still +reads as current there. Tolerable on a summary layer; not on one whose job is +to say what is true today. See docs/ogc_conventions.md. + +The three permission columns are three-valued and must stay that way: true = +a current grant says allowed, false = a current grant says not allowed, NULL = +no permission on record. NULL is not "denied" -- it means nobody has asked the +landowner yet. + +Depth to water uses (value - COALESCE(measuring_point_height, 0)) -- the +reading minus the height of the measuring point above ground, a missing height +treated as ground level. Character-for-character the convention in +ogc_water_well_summary, ogc_latest_depth_to_water_wells and +ogc_well_water_column, so the four layers cannot disagree about what a depth to +water is. + +Multi-valued columns are emitted as comma-joined text rather than as arrays +(which ogc_actively_monitored_wells uses). This layer exists to be pulled into +ArcGIS Pro and QGIS and exported to a File Geodatabase or GeoPackage for +offline field use, and neither format has a list type. + +Both relations are new here, so downgrade() drops them rather than restoring a +prior state. The supporting indexes are also created here: none of these +foreign keys was indexed (Postgres does not index foreign keys on its own). + +Revision ID: e1f2a3b4c5d6 +Revises: c9d0e1f2a3b4 +Create Date: 2026-08-31 00:00:00.000000 +""" + +import re +from typing import Sequence, Union + +from alembic import op +from sqlalchemy import inspect, text + +# revision identifiers, used by Alembic. +revision: str = "e1f2a3b4c5d6" +down_revision: Union[str, Sequence[str], None] = "c9d0e1f2a3b4" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +REQUIRED_TABLES = { + "thing", + "location", + "location_thing_association", + "observation", + "sample", + "field_activity", + "field_event", + "data_provenance", + "status_history", + "permission_history", + "measuring_point_history", + "monitoring_frequency_history", + "deployment", + "sensor", + "transducer_observation", + "contact", + "thing_contact_association", + "phone", + "email", + "notes", + "group", + "group_thing_association", + "well_purpose", + "well_casing_material", + "well_screen", + "thing_id_link", +} + +STATS_VIEW = "ogc_internal_water_well_field_operations_stats" +FEATURE_VIEW = "ogc_internal_water_well_field_operations" + +LATEST_LOCATION_CTE = """ +SELECT DISTINCT ON (lta.thing_id) + lta.thing_id, + lta.location_id, + lta.effective_start +FROM location_thing_association AS lta +WHERE lta.effective_end IS NULL +ORDER BY lta.thing_id, lta.effective_start DESC +""".strip() + +# sensor_type values that mean "this well is logging by itself". Anything else +# deployed at a well (a barometer, a camera) is equipment, not a logger, and +# must not make has_datalogger true. +LOGGER_SENSOR_TYPES = ( + "'Data Logger'", + "'Pressure Transducer'", + "'DiverLink'", + "'Diver Cable'", +) + +# Indexes the per-well lookups need. Each is created IF NOT EXISTS: none of +# these existed when this migration was written, but the thing_type one in +# particular is the sort of thing another migration may add first. +SUPPORTING_INDEXES = [ + ( + "ix_status_history_target_type_start", + "status_history (target_table, target_id, status_type, start_date DESC)", + ), + ( + "ix_permission_history_target_type_start", + "permission_history " + "(target_table, target_id, permission_type, start_date DESC)", + ), + ( + "ix_measuring_point_history_thing_start", + "measuring_point_history (thing_id, start_date DESC)", + ), + ( + "ix_monitoring_frequency_history_thing_start", + "monitoring_frequency_history (thing_id, start_date DESC)", + ), + ("ix_deployment_thing_id", "deployment (thing_id)"), + ("ix_thing_contact_association_thing_id", "thing_contact_association (thing_id)"), + ( + "ix_thing_contact_association_contact_id", + "thing_contact_association (contact_id)", + ), + ("ix_well_purpose_thing_id", "well_purpose (thing_id)"), + ("ix_well_casing_material_thing_id", "well_casing_material (thing_id)"), + ("ix_well_screen_thing_id", "well_screen (thing_id)"), + ("ix_thing_id_link_thing_id", "thing_id_link (thing_id)"), + ("ix_thing_thing_type", "thing (thing_type)"), + # No index on transducer_observation (deployment_id, ...): the existing + # uq_transducer_observation_deployment_parameter_datetime leads with + # deployment_id and already serves the per-deployment aggregate. +] + + +def _safe_relation_name(name: str) -> str: + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", name): + raise ValueError(f"Unsafe relation name: {name!r}") + return name + + +def _check_required_tables() -> None: + bind = op.get_bind() + inspector = inspect(bind) + existing_tables = set(inspector.get_table_names(schema="public")) + missing = REQUIRED_TABLES - existing_tables + if missing: + raise RuntimeError( + "Cannot create the water well field operations layer. " + f"Missing required tables: {', '.join(sorted(missing))}" + ) + + +def _create_stats_view() -> str: + """The expensive half: aggregates over the observation-scale tables.""" + safe_name = _safe_relation_name(STATS_VIEW) + return f""" + CREATE MATERIALIZED VIEW {safe_name} AS + WITH wells AS ( + SELECT t.id AS thing_id + FROM thing AS t + WHERE t.thing_type = 'water well' + ), + manual_obs AS ( + SELECT + fe.thing_id, + o.id AS observation_id, + o.observation_datetime, + (o.value - COALESCE(o.measuring_point_height, 0)) AS depth_to_water + FROM observation AS o + JOIN sample AS s ON s.id = o.sample_id + JOIN field_activity AS fa ON fa.id = s.field_activity_id + JOIN field_event AS fe ON fe.id = fa.field_event_id + JOIN wells AS w ON w.thing_id = fe.thing_id + WHERE + fa.activity_type = 'groundwater level' + AND o.value IS NOT NULL + AND o.observation_datetime IS NOT NULL + ), + manual_agg AS ( + SELECT + m.thing_id, + COUNT(*)::integer AS manual_water_level_count, + ( + MIN(m.observation_datetime) AT TIME ZONE 'UTC' + )::date AS manual_water_level_first_date, + ( + MAX(m.observation_datetime) AT TIME ZONE 'UTC' + )::date AS manual_water_level_last_date + FROM manual_obs AS m + GROUP BY m.thing_id + ), + manual_last AS ( + SELECT DISTINCT ON (m.thing_id) + m.thing_id, + m.depth_to_water AS last_depth_to_water_ft + FROM manual_obs AS m + ORDER BY m.thing_id, m.observation_datetime DESC, m.observation_id DESC + ), + chemistry_agg AS ( + SELECT + fe.thing_id, + COUNT(DISTINCT s.id)::integer AS chemistry_sample_count, + ( + MAX(s.sample_date) AT TIME ZONE 'UTC' + )::date AS chemistry_sample_last_date + FROM sample AS s + JOIN field_activity AS fa ON fa.id = s.field_activity_id + JOIN field_event AS fe ON fe.id = fa.field_event_id + JOIN wells AS w ON w.thing_id = fe.thing_id + WHERE fa.activity_type = 'water chemistry' + GROUP BY fe.thing_id + ), + field_event_agg AS ( + SELECT + fe.thing_id, + COUNT(*)::integer AS field_event_count, + ( + MAX(fe.event_date) AT TIME ZONE 'UTC' + )::date AS field_event_last_date + FROM field_event AS fe + JOIN wells AS w ON w.thing_id = fe.thing_id + GROUP BY fe.thing_id + ), + continuous_agg AS ( + SELECT + d.thing_id, + COUNT(*)::bigint AS continuous_reading_count, + MIN(tobs.observation_datetime) AS continuous_first_datetime, + MAX(tobs.observation_datetime) AS continuous_last_datetime + FROM transducer_observation AS tobs + JOIN deployment AS d ON d.id = tobs.deployment_id + JOIN wells AS w ON w.thing_id = d.thing_id + GROUP BY d.thing_id + ) + SELECT + w.thing_id, + COALESCE(ma.manual_water_level_count, 0) AS manual_water_level_count, + ma.manual_water_level_first_date, + ma.manual_water_level_last_date, + ml.last_depth_to_water_ft, + COALESCE(ca.chemistry_sample_count, 0) AS chemistry_sample_count, + ca.chemistry_sample_last_date, + COALESCE(fea.field_event_count, 0) AS field_event_count, + fea.field_event_last_date, + COALESCE(co.continuous_reading_count, 0) AS continuous_reading_count, + co.continuous_first_datetime, + co.continuous_last_datetime + FROM wells AS w + LEFT JOIN manual_agg AS ma ON ma.thing_id = w.thing_id + LEFT JOIN manual_last AS ml ON ml.thing_id = w.thing_id + LEFT JOIN chemistry_agg AS ca ON ca.thing_id = w.thing_id + LEFT JOIN field_event_agg AS fea ON fea.thing_id = w.thing_id + LEFT JOIN continuous_agg AS co ON co.thing_id = w.thing_id + """ + + +def _create_feature_view() -> str: + """The live half: everything a stale answer would misreport.""" + safe_name = _safe_relation_name(FEATURE_VIEW) + safe_stats = _safe_relation_name(STATS_VIEW) + logger_types = ", ".join(LOGGER_SENSOR_TYPES) + return f""" + CREATE VIEW {safe_name} AS + WITH latest_location AS ( +{LATEST_LOCATION_CTE} + ), + current_status AS ( + SELECT DISTINCT ON (sh.target_id, sh.status_type) + sh.target_id AS thing_id, + sh.status_type, + sh.status_value, + sh.start_date, + sh.reason + FROM status_history AS sh + WHERE + sh.target_table = 'thing' + AND sh.start_date <= CURRENT_DATE + AND (sh.end_date IS NULL OR sh.end_date >= CURRENT_DATE) + ORDER BY sh.target_id, sh.status_type, sh.start_date DESC, sh.id DESC + ), + current_permission AS ( + SELECT DISTINCT ON (ph.target_id, ph.permission_type) + ph.target_id AS thing_id, + ph.permission_type, + ph.permission_allowed, + ph.contact_id + FROM permission_history AS ph + WHERE + ph.target_table = 'thing' + AND ph.start_date <= CURRENT_DATE + AND (ph.end_date IS NULL OR ph.end_date >= CURRENT_DATE) + ORDER BY + ph.target_id, ph.permission_type, ph.start_date DESC, ph.id DESC + ), + current_measuring_point AS ( + SELECT DISTINCT ON (mp.thing_id) + mp.thing_id, + mp.measuring_point_height, + mp.measuring_point_description, + mp.start_date + FROM measuring_point_history AS mp + WHERE + mp.start_date <= CURRENT_DATE + AND (mp.end_date IS NULL OR mp.end_date >= CURRENT_DATE) + ORDER BY mp.thing_id, mp.start_date DESC, mp.id DESC + ), + current_monitoring_frequency AS ( + SELECT DISTINCT ON (mf.thing_id) + mf.thing_id, + mf.monitoring_frequency, + mf.start_date + FROM monitoring_frequency_history AS mf + WHERE + mf.start_date <= CURRENT_DATE + AND (mf.end_date IS NULL OR mf.end_date >= CURRENT_DATE) + ORDER BY mf.thing_id, mf.start_date DESC, mf.id DESC + ), + logger_deployments AS ( + SELECT + d.id AS deployment_id, + d.thing_id, + d.installation_date, + d.recording_interval, + d.recording_interval_units, + d.hanging_point_description, + se.sensor_type, + se.model, + se.serial_no, + se.sensor_status + FROM deployment AS d + JOIN sensor AS se ON se.id = d.sensor_id + WHERE + d.installation_date IS NOT NULL + AND d.removal_date IS NULL + AND se.sensor_type IN ({logger_types}) + ), + current_logger AS ( + SELECT DISTINCT ON (ld.thing_id) ld.* + FROM logger_deployments AS ld + ORDER BY ld.thing_id, ld.installation_date DESC, ld.deployment_id DESC + ), + logger_count AS ( + SELECT + ld.thing_id, + COUNT(*)::integer AS datalogger_deployment_count + FROM logger_deployments AS ld + GROUP BY ld.thing_id + ), + thing_contacts AS ( + SELECT + tca.thing_id, + c.id AS contact_id, + c.name, + c.organization, + c.role, + c.contact_type + FROM thing_contact_association AS tca + JOIN contact AS c ON c.id = tca.contact_id + ), + contact_agg AS ( + SELECT + tc.thing_id, + COUNT(DISTINCT tc.contact_id)::integer AS contact_count, + string_agg( + DISTINCT tc.name, ', ' ORDER BY tc.name + ) AS contact_names + FROM thing_contacts AS tc + GROUP BY tc.thing_id + ), + primary_contact AS ( + -- Prefer a contact recorded as Primary, but fall back to any + -- contact on the well rather than publishing a blank name next to + -- a non-zero contact_count. primary_contact_type says which case + -- this is, so the fallback is visible rather than implied. + SELECT DISTINCT ON (tc.thing_id) + tc.thing_id, + tc.contact_id, + tc.name, + tc.organization, + tc.role, + tc.contact_type + FROM thing_contacts AS tc + ORDER BY + tc.thing_id, + (tc.contact_type = 'Primary') DESC NULLS LAST, + tc.contact_id + ) + SELECT + t.id AS id, + t.name, + 'water well'::text AS thing_type, + t.release_status, + t.nma_pk_welldata, + ( + SELECT string_agg( + DISTINCT + COALESCE(tl.alternate_organization, 'unknown') + || ': ' + || tl.alternate_id, + ', ' + ORDER BY + COALESCE(tl.alternate_organization, 'unknown') + || ': ' + || tl.alternate_id + ) + FROM thing_id_link AS tl + WHERE tl.thing_id = t.id + ) AS alternate_ids, + l.county, + l.state, + l.quad_name, + -- Decimal degrees alongside the geometry. A crew types these into + -- a handheld GPS or reads them over the radio; the geometry column + -- is for the map, and a CSV export of this layer drops it. + ST_Y(l.point) AS latitude, + ST_X(l.point) AS longitude, + l.elevation, + dpl.collection_method AS elevation_method, + + -- Construction, same column names as the thing-type views. + t.well_depth, + t.hole_depth, + t.well_casing_diameter, + t.well_casing_depth, + t.well_completion_date, + t.well_driller_name, + t.well_construction_method, + t.well_pump_type, + t.well_pump_depth, + t.formation_completion_code, + t.nma_formation_zone, + ( + SELECT string_agg(DISTINCT wp.purpose, ', ' ORDER BY wp.purpose) + FROM well_purpose AS wp + WHERE wp.thing_id = t.id + ) AS well_purposes, + ( + SELECT string_agg( + DISTINCT wcm.material, ', ' ORDER BY wcm.material + ) + FROM well_casing_material AS wcm + WHERE wcm.thing_id = t.id + ) AS well_casing_materials, + scr.screen_count, + scr.screen_depth_top, + scr.screen_depth_bottom, + + -- Measuring point, current record. + cmp.measuring_point_height, + cmp.measuring_point_description, + cmp.start_date AS measuring_point_start_date, + + -- Status, current record per status type. + well_st.status_value AS well_status, + well_st.start_date AS well_status_since, + mon_st.status_value AS monitoring_status, + mon_st.start_date AS monitoring_status_since, + mon_st.reason AS monitoring_status_reason, + acc_st.status_value AS access_status, + acc_st.start_date AS access_status_since, + open_st.status_value AS open_status, + open_st.start_date AS open_status_since, + dl_st.status_value AS datalogger_suitability_status, + dl_st.start_date AS datalogger_suitability_status_since, + + -- Permission, current grants. Three-valued: NULL means no + -- permission is on record, which is not the same as denied. + wl_perm.permission_allowed AS may_measure_water_level, + chem_perm.permission_allowed AS may_sample_water_chemistry, + dl_perm.permission_allowed AS may_install_datalogger, + granter.name AS permission_granted_by, + + -- Monitoring programme. + cmf.monitoring_frequency, + cmf.start_date AS monitoring_frequency_since, + grp.group_names, + grp.group_types, + + -- Manual water levels (from the stats matview). + COALESCE(st.manual_water_level_count, 0) AS manual_water_level_count, + st.manual_water_level_first_date, + st.manual_water_level_last_date, + ( + CURRENT_DATE - st.manual_water_level_last_date + ) AS days_since_manual_water_level, + st.last_depth_to_water_ft, + + -- Chemistry sampling (from the stats matview). + COALESCE(st.chemistry_sample_count, 0) AS chemistry_sample_count, + st.chemistry_sample_last_date, + ( + CURRENT_DATE - st.chemistry_sample_last_date + ) AS days_since_chemistry_sample, + + -- Field visits (from the stats matview). + COALESCE(st.field_event_count, 0) AS field_event_count, + st.field_event_last_date, + + -- Data logger, current deployment. + (cl.deployment_id IS NOT NULL) AS has_datalogger, + COALESCE(lc.datalogger_deployment_count, 0) + AS datalogger_deployment_count, + cl.sensor_type AS datalogger_sensor_type, + cl.model AS datalogger_model, + cl.serial_no AS datalogger_serial_no, + cl.sensor_status AS datalogger_sensor_status, + cl.installation_date AS datalogger_installed_date, + cl.recording_interval AS datalogger_recording_interval, + cl.recording_interval_units AS datalogger_recording_interval_units, + cl.hanging_point_description AS datalogger_hanging_point_description, + COALESCE(st.continuous_reading_count, 0) AS continuous_reading_count, + st.continuous_first_datetime, + st.continuous_last_datetime, + ( + CURRENT_DATE + - (st.continuous_last_datetime AT TIME ZONE 'UTC')::date + ) AS days_since_continuous_reading, + + -- Contacts. + COALESCE(cag.contact_count, 0) AS contact_count, + pc.name AS primary_contact_name, + pc.organization AS primary_contact_organization, + pc.role AS primary_contact_role, + pc.contact_type AS primary_contact_type, + ( + SELECT p.phone_number + FROM phone AS p + WHERE p.contact_id = pc.contact_id + ORDER BY p.id + LIMIT 1 + ) AS primary_contact_phone, + ( + SELECT e.email + FROM email AS e + WHERE e.contact_id = pc.contact_id + ORDER BY e.id + LIMIT 1 + ) AS primary_contact_email, + cag.contact_names, + + -- Visit instructions. Separator is ' | ' rather than ', ' because + -- the content is free text and routinely contains commas. + ( + SELECT string_agg(n.content, ' | ' ORDER BY n.id DESC) + FROM notes AS n + WHERE + n.target_table = 'thing' + AND n.target_id = t.id + AND n.note_type = 'Access' + ) AS access_notes, + ( + SELECT string_agg(n.content, ' | ' ORDER BY n.id DESC) + FROM notes AS n + WHERE + n.target_table = 'thing' + AND n.target_id = t.id + AND n.note_type = 'Directions' + ) AS directions_notes, + + l.point + FROM thing AS t + JOIN latest_location AS ll ON ll.thing_id = t.id + JOIN location AS l ON l.id = ll.location_id + LEFT JOIN {safe_stats} AS st ON st.thing_id = t.id + LEFT JOIN current_status AS well_st + ON well_st.thing_id = t.id AND well_st.status_type = 'Well Status' + LEFT JOIN current_status AS mon_st + ON mon_st.thing_id = t.id + AND mon_st.status_type = 'Monitoring Status' + LEFT JOIN current_status AS acc_st + ON acc_st.thing_id = t.id AND acc_st.status_type = 'Access Status' + LEFT JOIN current_status AS open_st + ON open_st.thing_id = t.id AND open_st.status_type = 'Open Status' + LEFT JOIN current_status AS dl_st + ON dl_st.thing_id = t.id + AND dl_st.status_type = 'Datalogger Suitability Status' + LEFT JOIN current_permission AS wl_perm + ON wl_perm.thing_id = t.id + AND wl_perm.permission_type = 'Water Level Sample' + LEFT JOIN current_permission AS chem_perm + ON chem_perm.thing_id = t.id + AND chem_perm.permission_type = 'Water Chemistry Sample' + LEFT JOIN current_permission AS dl_perm + ON dl_perm.thing_id = t.id + AND dl_perm.permission_type = 'Datalogger Installation' + LEFT JOIN contact AS granter ON granter.id = wl_perm.contact_id + LEFT JOIN current_measuring_point AS cmp ON cmp.thing_id = t.id + LEFT JOIN current_monitoring_frequency AS cmf ON cmf.thing_id = t.id + LEFT JOIN current_logger AS cl ON cl.thing_id = t.id + LEFT JOIN logger_count AS lc ON lc.thing_id = t.id + LEFT JOIN contact_agg AS cag ON cag.thing_id = t.id + LEFT JOIN primary_contact AS pc ON pc.thing_id = t.id + LEFT JOIN LATERAL ( + -- Character-for-character the elevation-provenance lookup in + -- ogc_water_well_summary (2d3c3a268652), so the two layers cannot + -- disagree about where a well's elevation came from. + SELECT dp.collection_method + FROM data_provenance AS dp + WHERE + dp.target_table = 'location' + AND dp.target_id = l.id + AND dp.field_name = 'elevation' + ORDER BY dp.id DESC + LIMIT 1 + ) AS dpl ON true + LEFT JOIN LATERAL ( + SELECT + COUNT(*)::integer AS screen_count, + MIN(ws.screen_depth_top) AS screen_depth_top, + MAX(ws.screen_depth_bottom) AS screen_depth_bottom + FROM well_screen AS ws + WHERE ws.thing_id = t.id + ) AS scr ON TRUE + LEFT JOIN LATERAL ( + -- group_thing_association has no unique constraint on + -- (group_id, thing_id), so DISTINCT before aggregating. Both + -- strings are ordered by the same group name so they stay + -- index-aligned with each other. + SELECT + string_agg(dm.group_name, ', ' ORDER BY dm.group_name) + AS group_names, + string_agg(dm.group_type, ', ' ORDER BY dm.group_name) + AS group_types + FROM ( + SELECT DISTINCT + g.id AS group_id, + g.name AS group_name, + g.group_type + FROM group_thing_association AS gta + JOIN "group" AS g ON g.id = gta.group_id + WHERE gta.thing_id = t.id + ) AS dm + ) AS grp ON TRUE + WHERE t.thing_type = 'water well' + """ + + +def upgrade() -> None: + _check_required_tables() + + safe_stats = _safe_relation_name(STATS_VIEW) + safe_feature = _safe_relation_name(FEATURE_VIEW) + + # The feature view depends on the stats matview, so it is dropped first and + # created last. + op.execute(text(f"DROP VIEW IF EXISTS {safe_feature}")) + op.execute(text(f"DROP MATERIALIZED VIEW IF EXISTS {safe_stats}")) + + op.execute(text(_create_stats_view())) + # Unique index required for REFRESH MATERIALIZED VIEW CONCURRENTLY. + op.execute( + text( + f"CREATE UNIQUE INDEX ix_{safe_stats}_thing_id " + f"ON {safe_stats} (thing_id)" + ) + ) + op.execute(text(_create_feature_view())) + + for index_name, definition in SUPPORTING_INDEXES: + op.execute( + text( + f"CREATE INDEX IF NOT EXISTS {_safe_relation_name(index_name)} ON {definition}" + ) + ) + + +def downgrade() -> None: + safe_stats = _safe_relation_name(STATS_VIEW) + safe_feature = _safe_relation_name(FEATURE_VIEW) + + op.execute(text(f"DROP VIEW IF EXISTS {safe_feature}")) + op.execute(text(f"DROP MATERIALIZED VIEW IF EXISTS {safe_stats}")) + + for index_name, _definition in SUPPORTING_INDEXES: + op.execute(text(f"DROP INDEX IF EXISTS {_safe_relation_name(index_name)}")) diff --git a/core/ogc-field-descriptions.yml b/core/ogc-field-descriptions.yml index c462b6dd9..244525f83 100644 --- a/core/ogc-field-descriptions.yml +++ b/core/ogc-field-descriptions.yml @@ -2099,6 +2099,397 @@ minor_chemistry_wells: recorded them. +# water well field operations. Internal mount only -- the layer carries landowner +# contact details and staff-written access notes, so it has no public twin. +# See docs/water-well-field-operations-layer.md. +water_well_field_operations: + quad_name: + title: USGS quadrangle + description: >- + Name of the USGS 7.5-minute topographic quadrangle the well falls in. + elevation_method: + title: Elevation method + description: >- + How the ground-surface elevation was determined, such as GPS survey or + read from a digital elevation model. Governs how much precision the + elevation deserves. + enum-lexicon: collection_method + latitude: + title: Latitude + description: >- + Latitude of the well in decimal degrees on the WGS 84 datum, the same + coordinate the feature geometry carries. Published as a plain number so + it survives a spreadsheet or CSV export of this layer, which drops the + geometry, and so a crew can read it into a handheld GPS. + x-ogc-unit: https://qudt.org/vocab/unit/DEG + x-ogc-unitLang: QUDT + longitude: + title: Longitude + description: >- + Longitude of the well in decimal degrees on the WGS 84 datum, negative + west of Greenwich, the same coordinate the feature geometry carries. + Published as a plain number for the same reason as the latitude. + x-ogc-unit: https://qudt.org/vocab/unit/DEG + x-ogc-unitLang: QUDT + alternate_ids: + title: Alternate identifiers + description: >- + Identifiers other organisations use for this well, each written as + organisation followed by the identifier and separated by commas. Empty + where no cross-reference has been recorded. + well_purposes: + title: Well purposes + description: >- + What the well is used for, from the controlled well-purpose vocabulary -- + domestic, irrigation, monitoring, and so on. A well may serve more than + one purpose, in which case the purposes are separated by commas. + well_casing_materials: + title: Casing materials + description: >- + Materials the well casing is made of, from the controlled casing-material + vocabulary. Separated by commas where a well has more than one. + screen_count: + title: Screened intervals + description: >- + Number of discrete screened intervals recorded for the well. Zero means + no screen record exists, which is not the same as an unscreened well. + screen_depth_top: + title: Shallowest screen top + description: >- + Depth to the top of the shallowest screened interval, below ground + surface. + x-ogc-unit: https://qudt.org/vocab/unit/FT + x-ogc-unitLang: QUDT + screen_depth_bottom: + title: Deepest screen bottom + description: >- + Depth to the bottom of the deepest screened interval, below ground + surface. With the shallowest screen top this gives the outer extent of + the screened section, not the length of any one screen. + x-ogc-unit: https://qudt.org/vocab/unit/FT + x-ogc-unitLang: QUDT + measuring_point_height: + title: Measuring point height + description: >- + Height of the current measuring point above the ground surface. Manual + depth-to-water readings are taken from this point, so it is what converts + a raw reading to a depth below ground surface. + x-ogc-unit: https://qudt.org/vocab/unit/FT + x-ogc-unitLang: QUDT + measuring_point_description: + title: Measuring point description + description: >- + Where on the wellhead the measuring point is, as written for the person + standing at the well -- for example the north side of the casing at the + top of the PVC. The single most useful string on this layer for a crew + about to take a reading. + measuring_point_start_date: + title: Measuring point in use since + description: >- + Date the current measuring point configuration took effect. A recent date + means the wellhead was altered and readings before it are referenced to a + different point. + well_status: + title: Well status + description: >- + Current condition of the well itself. Null where no status is on record. + enum: + - Active, pumping well + - Inactive, exists but not used + - Destroyed, exists but not usable + - Abandoned + well_status_since: + title: Well status since + description: Date the current well status took effect. + monitoring_status: + title: Monitoring status + description: >- + Whether the well is currently on a monitoring schedule. Null where no + status is on record. + enum: + - Currently monitored + - Not currently monitored + monitoring_status_since: + title: Monitoring status since + description: Date the current monitoring status took effect. + monitoring_status_reason: + title: Monitoring status reason + description: >- + Why the well carries its current monitoring status, in the words of the + staff member who recorded it. This is where a landowner's request to stop + visiting is written down. + access_status: + title: Access status + description: >- + Current recorded state of physical access to the well. No values are + scoped to this status type in the controlled vocabulary yet, so the + column is expected to be sparse; treat a null as "not recorded" and + confirm access from the access notes and the landowner contact. + access_status_since: + title: Access status since + description: Date the current access status took effect. + open_status: + title: Open status + description: >- + Whether the well is physically open to a sounder or tape. A closed well + cannot be measured by hand however good the access. Null where no status + is on record. + enum: + - Open + - Open (unequipped) + - Closed + open_status_since: + title: Open status since + description: Date the current open status took effect. + datalogger_suitability_status: + title: Datalogger suitability + description: >- + Whether the well is physically suitable for a logger installation. This + is a judgement about the well, separate from whether the landowner has + given permission to install one. Null where no status is on record. + enum: + - Datalogger can be installed + - Datalogger cannot be installed + datalogger_suitability_status_since: + title: Datalogger suitability since + description: Date the current datalogger suitability status took effect. + may_measure_water_level: + title: May measure water level + description: >- + Whether a current permission allows a manual water-level measurement at + this well. True means a permission on record allows it; false means a + permission on record refuses it; null means no permission has been + recorded either way, which is not the same as a refusal -- it means + nobody has asked the landowner yet. + may_sample_water_chemistry: + title: May sample water chemistry + description: >- + Whether a current permission allows a water-chemistry sample to be + collected at this well. True allows, false refuses, null means no + permission is on record -- not a refusal. + may_install_datalogger: + title: May install datalogger + description: >- + Whether a current permission allows a logger to be installed in this + well. True allows, false refuses, null means no permission is on record + -- not a refusal. Separate from datalogger suitability, which says + whether the well could take one. + permission_granted_by: + title: Permission granted by + description: >- + Name of the contact who granted the current water-level permission, so a + crew can say who agreed to the visit. Null where no water-level + permission is on record. + monitoring_frequency: + title: Monitoring frequency + description: >- + How often the well is scheduled to be visited under its current + monitoring plan. Null where no frequency is on record. + enum-lexicon: monitoring_frequency + monitoring_frequency_since: + title: Monitoring frequency since + description: Date the current monitoring frequency took effect. + group_names: + title: Groups + description: >- + Names of the monitoring plans, geographic areas and historical groupings + the well belongs to, separated by commas. A well often sits in several. + group_types: + title: Group types + description: >- + Type of each group named in the groups column, in the same order and + separated by commas -- a monitoring plan, a geographic area, or a + historical grouping. + manual_water_level_count: + title: Manual water level readings + description: >- + Number of hand-measured groundwater level readings on record for the + well. Continuous readings from an installed logger are counted separately. + manual_water_level_first_date: + title: First manual reading + description: >- + Date of the earliest hand-measured groundwater level on record, as a UTC + calendar date. + manual_water_level_last_date: + title: Last manual reading + description: >- + Date of the most recent hand-measured groundwater level on record, as a + UTC calendar date. + days_since_manual_water_level: + title: Days since last manual reading + description: >- + Days elapsed since the most recent hand-measured groundwater level. + Computed against today, not against the last refresh, so it is current on + every request. Null where the well has never been measured by hand. + last_depth_to_water_ft: + title: Latest depth to water + description: >- + Most recent hand-measured depth to water, below ground surface -- the + measured value minus the height of the measuring point above ground, with + a reading that has no recorded measuring-point height treated as taken at + ground level. The same convention as the water-level summary and water + column layers. + x-ogc-unit: https://qudt.org/vocab/unit/FT + x-ogc-unitLang: QUDT + chemistry_sample_count: + title: Chemistry samples + description: >- + Number of water-chemistry samples collected at the well. Counts samples, + not the individual analyte results measured from them. + chemistry_sample_last_date: + title: Last chemistry sample + description: >- + Date the most recent water-chemistry sample was collected, as a UTC + calendar date. + days_since_chemistry_sample: + title: Days since last chemistry sample + description: >- + Days elapsed since the most recent water-chemistry sample. Computed + against today. Null where the well has never been sampled. + field_event_count: + title: Field visits + description: >- + Number of recorded visits to the well. Broader than either measurement + count: a visit that produced no reading, because the gate was locked or + the well was dry, is still a visit. + field_event_last_date: + title: Last field visit + description: >- + Date of the most recent recorded visit, as a UTC calendar date. A gap + between this and the last reading dates means someone went and came back + with nothing. + has_datalogger: + title: Has a datalogger + description: >- + Whether a logger is currently installed in the well -- a deployment with + an installation date and no removal date, of a data logger, pressure + transducer, DiverLink or diver cable. Other equipment deployed at the + well does not count. + datalogger_deployment_count: + title: Current logger deployments + description: >- + Number of logger deployments currently open at the well. Normally zero or + one; a higher number means the single logger columns describe only the + most recently installed of several. + datalogger_sensor_type: + title: Logger type + description: Kind of logger currently installed. + enum: + - Data Logger + - Pressure Transducer + - DiverLink + - Diver Cable + datalogger_model: + title: Logger model + description: Model of the logger currently installed. + datalogger_serial_no: + title: Logger serial number + description: >- + Serial number of the logger currently installed, for matching the + instrument in hand against the record. + datalogger_sensor_status: + title: Logger status + description: >- + Service state of the installed logger. A logger still recorded as + installed but marked retired or lost is a record worth correcting in the + field. + enum-lexicon: sensor_status + datalogger_installed_date: + title: Logger installed + description: Date the current logger deployment began. + datalogger_recording_interval: + title: Logger recording interval + description: >- + How often the installed logger records a reading, in the units given by + the recording interval units column. + datalogger_recording_interval_units: + title: Logger recording interval units + description: Time units the logger recording interval is expressed in. + datalogger_hanging_point_description: + title: Logger hanging point + description: >- + Where the logger cable is secured at the wellhead, as written for the + person about to pull it. Null where the deployment did not record one. + continuous_reading_count: + title: Continuous readings + description: >- + Number of readings logged automatically by instruments deployed in this + well, across all of its deployments. Separate from the hand-measured + count. + continuous_first_datetime: + title: First continuous reading + description: >- + Timestamp of the earliest automatically logged reading held for the well. + continuous_last_datetime: + title: Last continuous reading + description: >- + Timestamp of the most recent automatically logged reading held for the + well. A well whose logger has stopped reporting shows an old value here + while still reading as having a logger installed. + days_since_continuous_reading: + title: Days since last continuous reading + description: >- + Days elapsed since the most recent automatically logged reading, computed + against today. Null where the well has no logged readings at all. Rising + values on an instrumented well mean the logger needs attention. + contact_count: + title: Contacts + description: >- + Number of contacts associated with the well -- landowners, operators, and + anyone else recorded as a point of contact for it. + primary_contact_name: + title: Primary contact + description: >- + Name of the contact to approach about this well. This is the contact + recorded as primary where one exists; where none does, it is another + contact on the well, and the primary contact type column says which case + applies. + primary_contact_organization: + title: Primary contact organisation + description: Organisation the primary contact belongs to, where recorded. + primary_contact_role: + title: Primary contact role + description: >- + Role the primary contact holds in relation to the well, such as owner, + manager, or operator. + enum-lexicon: role + primary_contact_type: + title: Primary contact type + description: >- + Whether the contact shown is the one recorded as primary for the well or + a fallback used because no primary contact is on record. Read it before + relying on the contact details beside it. + enum-lexicon: contact_type + primary_contact_phone: + title: Primary contact phone + description: >- + Phone number for the primary contact. Where several are on record this is + the first one recorded, so a crew that cannot reach it should check the + full contact record. + primary_contact_email: + title: Primary contact email + description: >- + Email address for the primary contact. Where several are on record this + is the first one recorded. + contact_names: + title: All contacts + description: >- + Names of every contact associated with the well, separated by commas, so + a crew can see who else can be approached if the primary contact cannot + be reached. + access_notes: + title: Access notes + description: >- + Staff-written instructions for getting to the well and getting into it -- + gates, keys, locks, livestock, who to call first. Where several notes + exist they are separated by vertical bars, most recently recorded first. + directions_notes: + title: Directions + description: >- + Staff-written directions for finding the well on the ground, which + coordinates alone do not give. Where several notes exist they are + separated by vertical bars, most recently recorded first. + # EDR collections. Keys here are parameter names read out of the data, not # column names -- ogc_waterlevels stamps a single literal, while # ogc_water_chemistry carries the analyte text exactly as the laboratory diff --git a/core/pygeoapi-config-internal.yml b/core/pygeoapi-config-internal.yml index e62723e0c..9b0d080b5 100644 --- a/core/pygeoapi-config-internal.yml +++ b/core/pygeoapi-config-internal.yml @@ -77,6 +77,50 @@ resources: table: ogc_internal_locations geom_field: point + water_well_field_operations: + type: collection + title: Water Well Field Operations + description: >- + One record per water well, assembled for the crews who visit them. Where + every other well layer answers a scientific question, this one answers an + operational one: whether a crew may go to a well, what they are permitted + to do when they get there, and whether it is overdue. It pairs the well's + construction and measuring-point detail with its current well, + monitoring, access, open and datalogger-suitability status, the standing + landowner permissions for water-level measurement, chemistry sampling and + logger installation, the logger installed in it if any, how long it has + been since it was last measured and last sampled, and the landowner + contact and access notes needed to reach it. Published on the internal + mount only, because it carries personal contact details for landowners + and operators alongside staff-written access instructions. + keywords: + [ + water-wells, + field-operations, + landowner-contacts, + access-permissions, + monitoring-status, + data-logger, + internal, + ] + extents: + spatial: + bbox: [-109.05, 31.33, -103.00, 37.00] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + providers: + - type: feature + name: core.feature_provider.DescribedPostgreSQLProvider + data: + host: {postgres_host} + port: {postgres_port} + dbname: {postgres_db} + user: {postgres_user} + password: {postgres_password_env} + search_path: [public] + id_field: id + table: ogc_internal_water_well_field_operations + geom_field: point + latest_depth_to_water_wells: type: collection title: Latest Depth to Water (Water Wells) diff --git a/docs/internal-ogc-desktop-gis.md b/docs/internal-ogc-desktop-gis.md index 7efa66403..045bebd36 100644 --- a/docs/internal-ogc-desktop-gis.md +++ b/docs/internal-ogc-desktop-gis.md @@ -5,6 +5,16 @@ collections. It is gated by `core/internal_ogc_auth.py`, an ASGI middleware that runs in front of the raw Starlette Mount — FastAPI's `Depends()` machinery never sees these requests, so none of the `*_dependency` role parameters apply here. +> **An internal credential dispenses personal information.** The +> `water_well_field_operations` collection publishes landowner and operator +> contact details — name, organisation, role, phone number, email address — +> alongside staff-written access notes that routinely contain gate codes and +> names. Every credential accepted by this mount reaches it: an Authentik JWT +> carrying `OGCInternal`, and the static keys in `INTERNAL_OGC_API_KEYS`, which +> are shared secrets revocable only by redeploy. Any credential path added +> later inherits the same reach. Issue keys with that in mind, and read +> `docs/water-well-field-operations-layer.md` before widening who holds one. + ## Why there are static API keys at all The mount originally accepted only `Authorization: Bearer `. diff --git a/docs/ogc_conventions.md b/docs/ogc_conventions.md index 6ce6c8be3..268f2cc61 100644 --- a/docs/ogc_conventions.md +++ b/docs/ogc_conventions.md @@ -155,6 +155,33 @@ human-readable display name). | `locations` | N/A | Locations | N/A | N/A | Conforms. Hidden from public catalog — scope decision, not a naming fix | | `avg_tds_wells` | `water_well_average_total_dissolved_solids` | Average TDS (Water Wells) | Average Total Dissolved Solids (Water Wells) | 1 and 2 | ⚠️ *needs review*: `avg` and `tds` both unexplained abbreviations; hidden from public catalog but the rule still applies for internal consumers | | `latest_depth_to_water_wells` | N/A | Latest Depth to Water (Water Wells) | N/A | N/A | Conforms. Hidden from public catalog — audit flagged it as redundant with `water_well_summary`, not a naming defect | +| `water_well_field_operations` | N/A | Water Well Field Operations | N/A | N/A | Conforms. Internal-only with no public form at all — it publishes landowner contact details and staff-written access notes | + +## Current-record semantics + +Layers that read a history table (`status_history`, `permission_history`, +`measuring_point_history`, `monitoring_frequency_history`) disagree about what +"current" means, and the disagreement is deliberate. + +`ogc_actively_monitored_wells` takes the row with the greatest `start_date` and +**ignores `end_date`**, so a monitoring status closed in 2019 still reads as +current there. That is tolerable on a summary layer, where the question is +roughly "is this well in the programme". + +`ogc_internal_water_well_field_operations` honours the window: + +```sql +WHERE h.start_date <= CURRENT_DATE + AND (h.end_date IS NULL OR h.end_date >= CURRENT_DATE) +ORDER BY h.start_date DESC, h.id DESC +``` + +Its columns answer "may a crew do this today", and a permission that ran out +last month must not read as a permission. A row whose window has not opened yet +is not current either, and reads null. + +New layers should follow the second form. The first is kept only because +changing it would move rows in a published layer. ## References diff --git a/docs/water-well-field-operations-columns.md b/docs/water-well-field-operations-columns.md new file mode 100644 index 000000000..423ad6eb7 --- /dev/null +++ b/docs/water-well-field-operations-columns.md @@ -0,0 +1,106 @@ +# water well field operations — column sources + +Every column published by `ogc_internal_water_well_field_operations`, in view +order, and where its value comes from. Columns marked *(stats)* are read from +the `ogc_internal_water_well_field_operations_stats` materialized view and are as +fresh as its last refresh; everything else is joined live on each request. + +"Current record" means the history row satisfying +`start_date <= CURRENT_DATE AND (end_date IS NULL OR end_date >= CURRENT_DATE)`, +latest `start_date` first. "Most recent association" means the +`location_thing_association` row with no `effective_end`, latest +`effective_start` first. + +Generated from the view definition in +`alembic/versions/e1f2a3b4c5d6_add_water_well_field_operations_layer.py`. Field +prose lives in `core/ogc-field-descriptions.yml`; the design rationale lives in +`docs/water-well-field-operations-layer.md`. + +| Column | Source | +| --- | --- | +| `id` | `thing.id` | +| `name` | `thing.name` | +| `thing_type` | Literal `'water well'` — the view's row filter | +| `release_status` | `thing.release_status` | +| `nma_pk_welldata` | `thing.nma_pk_welldata` | +| `alternate_ids` | `thing_id_link.alternate_organization` + `.alternate_id`, comma-joined | +| `county` | `location.county`, most recent association | +| `state` | `location.state`, most recent association | +| `quad_name` | `location.quad_name`, most recent association | +| `latitude` | `ST_Y(location.point)` — decimal degrees, WGS 84 | +| `longitude` | `ST_X(location.point)` — decimal degrees, WGS 84 | +| `elevation` | `location.elevation`, most recent association | +| `elevation_method` | `data_provenance.collection_method` where `target_table = 'location'` and `field_name = 'elevation'`, latest `id` | +| `well_depth` | `thing.well_depth` | +| `hole_depth` | `thing.hole_depth` | +| `well_casing_diameter` | `thing.well_casing_diameter` | +| `well_casing_depth` | `thing.well_casing_depth` | +| `well_completion_date` | `thing.well_completion_date` | +| `well_driller_name` | `thing.well_driller_name` | +| `well_construction_method` | `thing.well_construction_method` | +| `well_pump_type` | `thing.well_pump_type` | +| `well_pump_depth` | `thing.well_pump_depth` | +| `formation_completion_code` | `thing.formation_completion_code` | +| `nma_formation_zone` | `thing.nma_formation_zone` | +| `well_purposes` | `well_purpose.purpose`, comma-joined | +| `well_casing_materials` | `well_casing_material.material`, comma-joined | +| `screen_count` | `count(well_screen)` | +| `screen_depth_top` | `min(well_screen.screen_depth_top)` | +| `screen_depth_bottom` | `max(well_screen.screen_depth_bottom)` | +| `measuring_point_height` | `measuring_point_history.measuring_point_height`, current record | +| `measuring_point_description` | `measuring_point_history.measuring_point_description`, current record | +| `measuring_point_start_date` | `measuring_point_history.start_date`, current record | +| `well_status` | `status_history.status_value` where `status_type = 'Well Status'`, current record | +| `well_status_since` | `status_history.start_date`, same record | +| `monitoring_status` | `status_history.status_value` where `status_type = 'Monitoring Status'`, current record | +| `monitoring_status_since` | `status_history.start_date`, same record | +| `monitoring_status_reason` | `status_history.reason`, same record | +| `access_status` | `status_history.status_value` where `status_type = 'Access Status'`, current record | +| `access_status_since` | `status_history.start_date`, same record | +| `open_status` | `status_history.status_value` where `status_type = 'Open Status'`, current record | +| `open_status_since` | `status_history.start_date`, same record | +| `datalogger_suitability_status` | `status_history.status_value` where `status_type = 'Datalogger Suitability Status'`, current record | +| `datalogger_suitability_status_since` | `status_history.start_date`, same record | +| `may_measure_water_level` | `permission_history.permission_allowed` where `permission_type = 'Water Level Sample'`, current record | +| `may_sample_water_chemistry` | `permission_history.permission_allowed` where `permission_type = 'Water Chemistry Sample'`, current record | +| `may_install_datalogger` | `permission_history.permission_allowed` where `permission_type = 'Datalogger Installation'`, current record | +| `permission_granted_by` | `contact.name` via `permission_history.contact_id` on the current water-level grant | +| `monitoring_frequency` | `monitoring_frequency_history.monitoring_frequency`, current record | +| `monitoring_frequency_since` | `monitoring_frequency_history.start_date`, current record | +| `group_names` | `group.name` via `group_thing_association`, comma-joined | +| `group_types` | `group.group_type`, same order as `group_names` | +| `manual_water_level_count` | `count(observation)` via `sample` → `field_activity` → `field_event`, `activity_type = 'groundwater level'` *(stats)* | +| `manual_water_level_first_date` | `min(observation.observation_datetime)`, UTC date, same chain *(stats)* | +| `manual_water_level_last_date` | `max(observation.observation_datetime)`, UTC date, same chain *(stats)* | +| `days_since_manual_water_level` | `CURRENT_DATE - manual_water_level_last_date` | +| `last_depth_to_water_ft` | `observation.value - COALESCE(observation.measuring_point_height, 0)` on the latest reading *(stats)* | +| `chemistry_sample_count` | `count(DISTINCT sample.id)`, `activity_type = 'water chemistry'` *(stats)* | +| `chemistry_sample_last_date` | `max(sample.sample_date)`, UTC date, same filter *(stats)* | +| `days_since_chemistry_sample` | `CURRENT_DATE - chemistry_sample_last_date` | +| `field_event_count` | `count(field_event)` for the well *(stats)* | +| `field_event_last_date` | `max(field_event.event_date)`, UTC date *(stats)* | +| `has_datalogger` | `true` when an open logger deployment exists — see `datalogger_sensor_type` | +| `datalogger_deployment_count` | `count(deployment)` open logger deployments | +| `datalogger_sensor_type` | `sensor.sensor_type` on the current deployment, restricted to Data Logger / Pressure Transducer / DiverLink / Diver Cable | +| `datalogger_model` | `sensor.model`, same deployment | +| `datalogger_serial_no` | `sensor.serial_no`, same deployment | +| `datalogger_sensor_status` | `sensor.sensor_status`, same deployment | +| `datalogger_installed_date` | `deployment.installation_date`, same deployment | +| `datalogger_recording_interval` | `deployment.recording_interval`, same deployment | +| `datalogger_recording_interval_units` | `deployment.recording_interval_units`, same deployment | +| `datalogger_hanging_point_description` | `deployment.hanging_point_description`, same deployment | +| `continuous_reading_count` | `count(transducer_observation)` via `deployment` *(stats)* | +| `continuous_first_datetime` | `min(transducer_observation.observation_datetime)` *(stats)* | +| `continuous_last_datetime` | `max(transducer_observation.observation_datetime)` *(stats)* | +| `days_since_continuous_reading` | `CURRENT_DATE - continuous_last_datetime::date` | +| `contact_count` | `count(DISTINCT contact)` via `thing_contact_association` | +| `primary_contact_name` | `contact.name`, `contact_type = 'Primary'` preferred, else lowest `contact.id` | +| `primary_contact_organization` | `contact.organization`, same contact | +| `primary_contact_role` | `contact.role`, same contact | +| `primary_contact_type` | `contact.contact_type`, same contact — says whether the row above is a real primary or a fallback | +| `primary_contact_phone` | `phone.phone_number`, lowest `phone.id` for that contact | +| `primary_contact_email` | `email.email`, lowest `email.id` for that contact | +| `contact_names` | `contact.name` for every associated contact, comma-joined | +| `access_notes` | `notes.content` where `note_type = 'Access'`, newest first, joined with ` | ` | +| `directions_notes` | `notes.content` where `note_type = 'Directions'`, newest first, joined with ` | ` | +| `point` | `location.point`, most recent association (PostGIS Point, EPSG:4326) | diff --git a/docs/water-well-field-operations-layer.md b/docs/water-well-field-operations-layer.md new file mode 100644 index 000000000..9e2c61308 --- /dev/null +++ b/docs/water-well-field-operations-layer.md @@ -0,0 +1,519 @@ +# Water well field operations layer — design + +Status: **implemented** on `feat/amp-field-operations-ogc-layer`. Migration +`e1f2a3b4c5d6`; column-by-column source table in +`docs/water-well-field-operations-columns.md`. + +Two questions this document opened were settled before implementation: "AMP" is +a label for the crews' view of the register, not a subset of it, so the layer +covers every water well; and the contact detail in §4 is published in full +(option A). + +## 0. Why the name does not say AMP + +The layer was requested as `amp_field_well_operations` and renamed before it +merged, per the pre-merge check in `docs/ogc_conventions.md`. Three rules +pushed the same way: + +- `amp` is an unexpanded abbreviation, which the do/don't rules forbid + outright ("Spell abbreviations out in full ... If the accurate name is long, + that's fine"). Its expansion is recorded nowhere in this codebase, so a + consumer could not resolve it even in principle. +- AMP is a label for who uses the layer, not a filter on what is in it — the + row set is every water well, with no group predicate. A name carrying `amp` + would claim a scope the data does not have, which is its own Don't. +- Group B analytic layers prefer a `water_well_` prefix over a `_wells` + suffix, and this is one. + +The AMP crews remain the layer's audience; that belongs in the description, +which says so, rather than in an identifier that is a long-term commitment. + +A new OGC API - Features collection, `water_well_field_operations`, published on +`/ogcapi-internal` only. One feature per water well, carrying everything a +field crew needs to plan and execute a visit: where the well is, who owns it +and how to reach them, what the crew is allowed to do there, what is installed +in it, and when it was last measured. + +Every other well layer in the catalogue answers a scientific question — what is +the water level, what is the chemistry, what is the trend. This one answers an +operational one: **can we go to this well, what are we allowed to do when we +get there, and is it overdue?** + +## 1. Why a new layer rather than a column pass on an existing one + +`water_wells` is the construction register and is public. `water_well_summary` +and `actively_monitored_wells` summarise the measurement record and are public. +The operational fields — landowner phone numbers, access notes, sampling +permissions — cannot go on any of those, because those layers are published +anonymously. The split is a publication boundary, not a modelling preference. + +## 2. Scope + +**In:** water wells. `thing_type = 'water well'`, the same predicate every +other well layer in this catalogue uses. Springs, streams, and met stations are +out even though crews visit them — this is the wells layer, per the request. + +`thing_type` in the lexicon also carries `observation well`, `piezometer`, +`monitoring well`, `production well`, `injection well`, `exploration well`, +`test well`, `abandoned well`, `dry hole`, `artesian well`, and `dug well`. +None of them are included. That matches `ogc_water_wells`, +`ogc_water_well_summary`, and `ogc_well_water_column`, so the layers cannot +disagree about which rows are "wells" — see §11, open question 1. + +**Row set:** every water well with a current location, including wells with no +measurement record at all. A well nobody has ever measured is exactly the well +a field crew needs to find, so this layer must not inherit +`water_well_summary`'s `total_water_levels > 0` restriction. + +**Release status:** unfiltered. The internal mount is unfiltered by design and +this layer has no public twin, so there is no `release_status = 'public'` +predicate anywhere in it. `release_status` is published as a column so a +consumer can see what they are looking at. + +## 3. Placement: internal-only, no public counterpart + +Every other `ogc_internal_*` relation is the unfiltered twin of a public +`ogc_*` one. This layer deliberately breaks that: `ogc_internal_water_well_field_operations` +exists and `ogc_water_well_field_operations` does not, and must never be created. + +This does not disturb the existing parity guard. +`tests/test_migration_view_parity.py::test_internal_migration_mirrors_every_public_relation` +reads two named migration files (`f4a5b6c7d8e9` and `2d3c3a268652`) and asserts +that internal file mirrors that public file's 24 relations. A relation created +by a *new* migration is outside what that test reads, so the count assertion +stays at 24 and stays true. + +Precedent for an internal-only collection: `locations`, `avg_tds_wells`, +`latest_depth_to_water_wells` (static entries in the internal config template +only) and `other_things` (`internal_only: True` in `THING_COLLECTIONS`). + +Because this layer is not a thing-type register, it is wired the same way +`locations` is: a static block in `core/pygeoapi-config-internal.yml`, not an +entry in `THING_COLLECTIONS`. + +## 4. The PII decision — read this before implementing + +This layer publishes landowner and operator contact details: name, +organization, role, phone number, email address. That is what "contact info" +means for a field crew, and it is why the layer is internal-only. + +The consequence is not confined to this layer. `/ogcapi-internal` is reachable +two ways on this branch (`core/internal_ogc_auth.py`): an Authentik JWT +carrying `OGCInternal`, or a static API key from `INTERNAL_OGC_API_KEYS`. +**Shipping this layer means every one of those credentials becomes a credential +that dispenses landowner personal information**, and any credential path added +later -- the user-issued keys on `feat/api-key-management` among them -- +inherits the same reach. The static desktop-GIS keys in particular are shared +secrets that can only be revoked by redeploy. + +pygeoapi has no per-field authorization — a column is either in the view or it +is not — so this is all-or-nothing per layer. Three options: + +| Option | What crews get | Cost | +| --- | --- | --- | +| **A. Full contact detail** (recommended) | Name, organization, role, phone, email | Every internal credential now dispenses PII | +| B. Presence only | `contact_count`, `primary_contact_organization`, `primary_contact_role` — no name, phone, or email | Crews must look the owner up in the UI before every visit; layer half-fails its purpose | +| C. Second layer | `water_well_field_operations` without PII, `water_well_field_contacts` with it | Two layers, two configs, two sets of field docs, same credentials still reach both — the split buys nothing without per-layer auth | + +**Decision: A.** The request was explicit, the mount is authenticated, and B +makes the layer not worth building. Shipped with: + +- A notice at the top of `docs/internal-ogc-desktop-gis.md` stating that an + internal credential now conveys landowner PII, so key issuance is a decision + made with that in mind. **`docs/api-key-management.md` still needs the same + notice** — that file lives on `feat/api-key-management` and does not exist on + this branch, so it could not be edited here. +- The collection `description` says so too, so it is visible in the catalogue + itself. +- The layer does not filter contacts on `release_status`. `Contact` carries one, + but the internal mount is unfiltered by design and this layer has no public + twin to keep in step with. + +## 5. Relations: one matview, one view + +Two relations, not one, because the columns divide cleanly by cost *and* by how +badly staleness hurts: + +``` +ogc_internal_water_well_field_operations_stats MATERIALIZED VIEW, keyed thing_id + counts, first/last dates, aggregates over observation and + transducer_observation. Expensive. Refreshed nightly. + +ogc_internal_water_well_field_operations VIEW <- pygeoapi points here + live join of thing, location, status_history, permission_history, + deployment, sensor, contact, notes + LEFT JOIN the stats matview +``` + +Reasons for the split: + +1. **Staleness is dangerous on exactly the cheap columns.** "May we sample this + well" and "is access currently granted" must not be up to 24 hours old. A + revoked permission that still reads `true` until the next nightly refresh + sends a crew onto land they are no longer welcome on. `permission_history` + and `status_history` are small and indexed by `(target_id, target_table)`; + reading them live costs nothing. +2. **`CURRENT_DATE` freezes in a matview.** The current-record rule in §7 is + date-relative. Evaluated inside a materialized view it would be pinned to + the refresh time, and a permission that expired this morning would still + read as current tonight. It has to be in the plain view. +3. **The expensive columns don't need to be fresh.** A reading count and a + last-measured date are fine at 24 hours old. + +Plain views already back the eleven thing-type collections, so pygeoapi serving +a non-materialized view is established. Filtering and bbox pushdown resolve +against the base tables' indexes. + +`_stats` carries a unique index on `thing_id` so it can be refreshed +`CONCURRENTLY` (`oco refresh-matview --concurrently`). The nightly pg_cron job +discovers every matview in the `public` schema by name, so it is picked up with +no schedule change. Add it to `MATERIALIZED_VIEWS` in +`services/materialized_views.py` for the CLI path. + +## 6. Columns + +`id` is `thing.id` and is unique — pygeoapi's `id_field: id` requires exactly +one row per id for `/items/{id}`. Every one-to-many is therefore aggregated. + +Multi-valued columns are emitted as **`text`, comma-space joined**, not as +Postgres arrays. `ogc_actively_monitored_wells` uses `text[]` and that is fine +for a layer read in a browser, but this one exists to be pulled into ArcGIS Pro +and QGIS and exported to a File Geodatabase or GeoPackage for offline field +use, and neither format has a list type. A joined string survives the round +trip; an array does not. + +### Identity and location + +| Column | Source | +| --- | --- | +| `id` | `thing.id` | +| `name` | `thing.name` | +| `thing_type` | `'water well'::text` | +| `release_status` | `thing.release_status` | +| `nma_pk_welldata` | `thing.nma_pk_welldata` — legacy NM_Aquifer key, still what crews say out loud | +| `alternate_ids` | `thing_id_link`, joined as `organization:alternate_id` pairs | +| `county`, `quad_name`, `state` | `location` | +| `latitude`, `longitude` | `ST_Y`/`ST_X` of the same point, decimal degrees on WGS 84 -- a plain number survives a CSV export, which drops the geometry, and a crew can read it into a handheld GPS | +| `elevation` | `location.elevation` | +| `elevation_method` | `data_provenance.collection_method` for that location's elevation -- the same lookup `water_well_summary` uses, so the two layers cannot disagree about how an elevation was obtained | +| `point` | `location.point`, most recent association (the shared `LATEST_LOCATION_CTE`) | + +### Well construction + +`well_depth`, `hole_depth`, `well_casing_diameter`, `well_casing_depth`, +`well_completion_date`, `well_driller_name`, `well_construction_method`, +`well_pump_type`, `well_pump_depth`, `formation_completion_code`, +`nma_formation_zone` — all straight from `thing`, same columns and same names +as the thing-view template, so a crew reading both layers sees one vocabulary. + +Plus, aggregated: + +| Column | Source | +| --- | --- | +| `well_purposes` | `well_purpose.purpose`, joined | +| `well_casing_materials` | `well_casing_material.material`, joined | +| `screen_count` | `count(well_screen)` | +| `screen_depth_top`, `screen_depth_bottom` | `min`/`max` across that well's screens, ft below ground surface | + +### Measuring point (current record only) + +| Column | Source | +| --- | --- | +| `measuring_point_height` | `measuring_point_history.measuring_point_height`, ft above ground surface | +| `measuring_point_description` | e.g. "North side of casing, top of PVC" | +| `measuring_point_start_date` | when the current configuration took effect | + +The description is the single most useful string on the layer for a crew +standing at a wellhead. Current record per §7. + +### Status (current record per status type) + +`status_history.status_type` has five values in the lexicon, and all five are +published, each as `` plus `_since`: + +| Column | `status_type` | Values | +| --- | --- | --- | +| `well_status` | Well Status | Abandoned; Active, pumping well; Destroyed, exists but not usable; Inactive, exists but not used | +| `monitoring_status` | Monitoring Status | Currently monitored; Not currently monitored | +| `access_status` | Access Status | (lexicon does not yet scope values to type — open question 3) | +| `open_status` | Open Status | Open; Open (unequipped); Closed | +| `datalogger_suitability_status` | Datalogger Suitability Status | Datalogger can be installed; Datalogger cannot be installed | + +`monitoring_status_reason` is also published — `status_history.reason`, which +is where "landowner asked us to stop" is written down. + +### Permission (current grants) + +From `permission_history`, whose `permission_type` lexicon is exactly the three +things a crew does: + +| Column | `permission_type` | +| --- | --- | +| `may_measure_water_level` | Water Level Sample | +| `may_sample_water_chemistry` | Water Chemistry Sample | +| `may_install_datalogger` | Datalogger Installation | + +**These are three-valued and must stay that way.** `true` = a current grant +says allowed. `false` = a current grant says *not* allowed. `NULL` = **no +permission on record**, which is not the same as denied and must not be +rendered as "no". Collapsing NULL to false would tell a crew a well is off +limits when the truth is that nobody has asked yet; collapsing it to true is +worse. Every consumer-facing rendering of these columns has to carry the third +state. + +`permission_granted_by` — the granting contact's name from the current +water-level grant — is published alongside, so a crew can name who said yes. + +### Monitoring programme + +| Column | Source | +| --- | --- | +| `monitoring_frequency` | current `monitoring_frequency_history` record | +| `monitoring_frequency_since` | its `start_date` | +| `group_names` | `group` via `group_thing_association`, joined, de-duplicated | +| `group_types` | Monitoring Plan / Geographic Area / Historical, joined, index-aligned with `group_names` | + +### Manual measurement statistics + +Manual groundwater levels, reached the way every other water-level view in this +schema reaches them: `observation -> sample -> field_activity -> field_event`, +`field_activity.activity_type = 'groundwater level'`. + +| Column | Definition | +| --- | --- | +| `manual_water_level_count` | count of readings with a value and a timestamp | +| `manual_water_level_first_date` | earliest `observation_datetime`, UTC calendar date | +| `manual_water_level_last_date` | latest, UTC calendar date | +| `days_since_manual_water_level` | `CURRENT_DATE - manual_water_level_last_date`, computed in the plain view so it does not freeze at refresh | +| `last_depth_to_water_ft` | the latest reading, below ground surface | + +`last_depth_to_water_ft` uses `(o.value - COALESCE(o.measuring_point_height, 0))` +— the reading minus the height of the measuring point above ground, with a +missing height treated as taken at ground level. This is the exact convention +in `ogc_water_well_summary`, `ogc_latest_depth_to_water_wells`, and +`ogc_well_water_column`, and it must stay identical or the four layers will +disagree about what a depth to water is. See +`docs/measuring-point-height-null-handling.md`. + +### Chemistry sampling statistics + +Same chain, `field_activity.activity_type = 'water chemistry'`: +`chemistry_sample_count`, `chemistry_sample_last_date`, +`days_since_chemistry_sample`. Sample-level, not analyte-level — a crew wants +"when was this well last sampled", not "what was the sulfate". + +### Field visit statistics + +`field_event_count`, `field_event_last_date` from `field_event`. Broader than +either measurement chain: a visit that produced no reading is still a visit, +and the gap between `field_event_last_date` and +`manual_water_level_last_date` is itself a signal. + +### Data logger / continuous record + +"Has a logger" is a deployment question, not a sensor-inventory question. A +deployment is current when `installation_date IS NOT NULL` and `removal_date IS +NULL`, and the sensor is one of `Data Logger`, `Pressure Transducer`, +`DiverLink`, `Diver Cable` (`sensor.sensor_type`). + +| Column | Definition | +| --- | --- | +| `has_datalogger` | boolean, true when a current logger deployment exists | +| `datalogger_sensor_type`, `datalogger_model`, `datalogger_serial_no` | from `sensor` on the current deployment | +| `datalogger_sensor_status` | In Service / In Repair / Retired / Lost | +| `datalogger_installed_date` | `deployment.installation_date` | +| `datalogger_recording_interval`, `datalogger_recording_interval_units` | how often it logs | +| `datalogger_hanging_point_description` | where the cable hangs from — a field instruction | +| `continuous_reading_count` | rows in `transducer_observation` for this well's deployments | +| `continuous_first_datetime`, `continuous_last_datetime` | series extent | +| `days_since_continuous_reading` | live, in the plain view | + +Where a well has more than one current logger deployment, the singular +`datalogger_*` columns take the most recently installed one, and +`datalogger_deployment_count` says how many there are so nothing is silently +hidden. + +`continuous_reading_count` aggregates the largest table in the schema. It is +the reason `_stats` is materialized. If the refresh proves too slow, the +fallback is to source the three continuous columns from `transducer_daily_data` +instead — cheaper, one row per day, and adequate for "is the logger still +reporting". + +### Contacts + +Subject to §4. + +| Column | Definition | +| --- | --- | +| `contact_count` | contacts associated with the well | +| `primary_contact_name`, `_organization`, `_role` | prefer the `contact_type = 'Primary'` contact, else any contact on the well, lowest `contact.id` as tie-break | +| `primary_contact_type` | which of those two cases produced the row above | +| `primary_contact_phone` | lowest-id `phone.phone_number` for that contact | +| `primary_contact_email` | lowest-id `email.email` for that contact | +| `contact_names` | every associated contact, joined | + +The fallback is a change from this document's first draft, which took the +primary contact strictly. A well whose only contact is recorded as Secondary +would then show `contact_count = 1` beside a blank name and phone — the exact +failure mode that ruled out option B in §4. `primary_contact_type` keeps the +fallback visible rather than implied. + +Deliberately **not** published: `address`. A mailing address is not how a crew +reaches a landowner, and it is the most sensitive field on the record. + +### Notes + +`notes` is polymorphic on `(target_id, target_table = 'thing')` with a +`note_type` lexicon. Two types are operational and are published as their own +columns, most recent first, joined: + +| Column | `note_type` | +| --- | --- | +| `access_notes` | Access | +| `directions_notes` | Directions | + +The rest (Construction, Maintenance, Historical, General, Water, Water Quality, +Sampling Procedure, Coordinate, OwnerComment, Site Notes (legacy)) stay out. +They are a well's history, not its visit instructions, and free text is where +PII leaks — see open question 4. + +## 7. The "current record" rule + +Four histories feed this layer — `status_history`, `permission_history`, +`measuring_point_history`, `monitoring_frequency_history` — and all four have +the same `(start_date, end_date)` shape. One rule for all of them: + +```sql +WHERE h.start_date <= CURRENT_DATE + AND (h.end_date IS NULL OR h.end_date >= CURRENT_DATE) +ORDER BY h.start_date DESC, h.id DESC +LIMIT 1 -- via DISTINCT ON per thing +``` + +**This diverges from the existing catalogue on purpose.** +`ogc_actively_monitored_wells` takes the row with the greatest `start_date` and +ignores `end_date` entirely, so a monitoring status that was closed in 2019 +still reads as current there. That is tolerable on a summary layer. It is not +tolerable on a layer whose whole job is to tell a crew what is true today — +"you may install a logger here" must not survive the expiry of the permission +that said so. + +The divergence has to be written into `docs/ogc_conventions.md`, or the next +person will read the two views side by side and assume one is a mistake. + +Rows whose window has not opened yet (`start_date > CURRENT_DATE`) are not +current and read NULL. + +## 8. Field-level documentation + +`core/ogc_field_metadata.py` keys `core/ogc-field-descriptions.yml` by backing +relation with the `ogc_internal_` prefix stripped, so the entry is +`water_well_field_operations`. This is not optional: +`tests/test_ogc_field_descriptions.py::test_every_published_column_has_an_entry` +walks both mounts and fails on any published column without a `description`. + +Columns already in `_defaults` (`id`, `name`, `thing_type`, `release_status`, +`elevation`, `well_depth`, `hole_depth`, `well_casing_diameter`, …) are +inherited and must not be restated. Everything new to this layer needs an +entry, and the entry says what the value *means*, never how the view is built. + +Controlled vocabularies use `enum-lexicon` rather than a duplicated `enum` +list: `status_value` for the five status columns, `monitoring_frequency`, +`sensor_type`, `sensor_status`, `well_purpose`, `casing_material`, `role`, +`group_type`. Depth and height columns carry +`x-ogc-unit: https://qudt.org/vocab/unit/FT` with +`x-ogc-unitLang: QUDT`, matching the rest of the file. + +The three permission booleans need their NULL meaning spelled out in the +`description` — that is the only place a consumer will ever read it. + +## 9. Wiring + +1. **Migration** — new Alembic revision on the current head (`c9d0e1f2a3b4`), + creating the matview, its unique index, and the view, plus the supporting + indexes in §10. `downgrade()` drops all three; nothing pre-existing is + modified, so there is no prior state to restore. Follow the house style in + `2d3c3a268652`: self-contained, no cross-migration imports, `REQUIRED_TABLES` + guard up front. +2. **`core/pygeoapi-config-internal.yml`** — one static resource block, next to + `locations`. `name: core.feature_provider.DescribedPostgreSQLProvider`, + `id_field: id`, `table: ogc_internal_water_well_field_operations`, + `geom_field: point`. No `time_field`: no single column is the feature's + time, and picking one would make `datetime=` filtering quietly mean + something arbitrary. +3. **`core/pygeoapi-config.yml`** — no change. Ever. +4. **`core/ogc-field-descriptions.yml`** — the `water_well_field_operations` block. +5. **`services/materialized_views.py`** — append + `ogc_internal_water_well_field_operations_stats` to `MATERIALIZED_VIEWS`. +6. **`core/gis-curated-layers.yml`** — *no* entry. Every layer in that file + ships as a `.qlr` and a `.lyrx` artifact, and this one is PII-bearing and + internal; it should be reached by browsing the authenticated connection, not + handed out as a downloadable file. Revisit only if crews ask. +7. **Docs** — `docs/api-key-management.md` and `docs/internal-ogc-desktop-gis.md` + gain the PII notice from §4; `docs/ogc_conventions.md` gains the §7 + divergence. + +Description text has to clear +`tests/test_pygeoapi_mount.py::test_every_collection_description_explains_the_layer`: +at least 200 characters, ends in a full stop, no placeholder words, no +hyphenated word split across a YAML line break, and at least four unique +lowercase-hyphenated keywords. + +## 10. Indexes + +The per-thing lookups this layer adds are foreign-key joins, and Postgres does +not index foreign keys on its own. Check each before creating it — `b8c9d0e1f2a3` +already added four on the observation chain. + +- `status_history (target_table, target_id, status_type, start_date DESC)` +- `permission_history (target_table, target_id, permission_type, start_date DESC)` +- `measuring_point_history (thing_id, start_date DESC)` +- `monitoring_frequency_history (thing_id, start_date DESC)` +- `notes (target_id, target_table)` — already exists as `ix_notes_polymorphic_link` +- `deployment (thing_id)` +- `thing_contact_association (thing_id)`, `(contact_id)` +- `transducer_observation (deployment_id, observation_datetime)` — for the + continuous aggregates; the largest table in the schema and the one that will + decide whether the nightly refresh is acceptable + +## 11. Settled, and still open + +Settled before implementation: + +- **"AMP" is a label, not a subset.** The layer covers every water well; there + is no group predicate. +- **Contact detail is published in full** (§4, option A). + +Still open: + +1. **Which `thing_type` values count as a well?** Shipped as `'water well'` + only, for consistency with every existing well layer. If AMP crews also + visit piezometers and monitoring wells, the predicate widens to a list — and + then it should widen in the other four layers too, as its own ticket, rather + than this layer quietly disagreeing with them. +2. **Access Status has no values.** `status_type` carries `Access Status` but + the `status_value` lexicon has no access-related terms — the eleven values + map to Well, Monitoring, Open, and Datalogger Suitability status. The column + is published and will read null until the vocabulary gains terms scoped to + it; the field description says so. Either access statuses are recorded with + values from another category, or the column is empty in practice, and the + answer needs a look at production data. +3. **Free-text PII.** `access_notes` and `directions_notes` are staff-written + and will contain phone numbers, gate codes, and names. A further reason the + layer is internal-only, and the §4 decision covers them too. +4. **Refresh cadence.** Nightly, via the existing pg_cron job. If crews plan + the next day's route the evening before, nightly is fine; if they re-plan in + the field, the stats columns are the ones that go stale. +5. **`continuous_reading_count` cost at production scale.** It aggregates the + largest table in the schema. If the nightly refresh proves too slow, the + fallback is to source the three continuous columns from + `transducer_daily_data` instead. + +## 12. Out of scope + +- Any public counterpart. +- Write access. This is a read layer; edits happen in the Ocotillo UI. +- A curated `.qlr` / `.lyrx` artifact (§9.6). +- Springs and other non-well monitoring points. +- Per-field authorization. pygeoapi has no such hook; the layer boundary is the + authorization boundary. diff --git a/services/materialized_views.py b/services/materialized_views.py index 9b7e3740e..75c5239d4 100644 --- a/services/materialized_views.py +++ b/services/materialized_views.py @@ -19,5 +19,9 @@ # two pivot views above, at per-result grain. "ogc_water_chemistry", "ogc_internal_water_chemistry", + # Field-operations aggregates for the internal-only + # ogc_internal_water_well_field_operations layer (e1f2a3b4c5d6). The + # feature view itself is a plain view and needs no refresh. + "ogc_internal_water_well_field_operations_stats", "transducer_daily_data", ) diff --git a/tests/test_cli_commands.py b/tests/test_cli_commands.py index 97534a601..257e4c320 100644 --- a/tests/test_cli_commands.py +++ b/tests/test_cli_commands.py @@ -72,10 +72,11 @@ def __exit__(self, exc_type, exc, tb): "REFRESH MATERIALIZED VIEW ogc_minor_chemistry_wells", "REFRESH MATERIALIZED VIEW ogc_water_chemistry", "REFRESH MATERIALIZED VIEW ogc_internal_water_chemistry", + "REFRESH MATERIALIZED VIEW ogc_internal_water_well_field_operations_stats", "REFRESH MATERIALIZED VIEW transducer_daily_data", ] assert commit_called["value"] is True - assert "Refreshed 10 materialized view(s)." in result.output + assert "Refreshed 11 materialized view(s)." in result.output def test_refresh_materialized_views_custom_and_concurrently( @@ -704,10 +705,12 @@ def _write_csv(path: Path, *, well_name: str, notes: str): "Water level accurate to within two hundreths of a foot," f"{notes}" ) - csv_text = textwrap.dedent(f"""\ + csv_text = textwrap.dedent( + f"""\ {header} {row} - """) + """ + ) path.write_text(csv_text) unique_notes = f"pytest-{uuid.uuid4()}" diff --git a/tests/test_ogc.py b/tests/test_ogc.py index 18c5fea74..5d6d3c83f 100644 --- a/tests/test_ogc.py +++ b/tests/test_ogc.py @@ -818,12 +818,15 @@ def test_ogc_collections(ogc_client): # and latest_depth_to_water_wells repeats water_well_summary # (BDMS-977), and other_things is internal vocabulary (BDMS-979). The # backing relations are retained and still served on /ogcapi-internal. + # water_well_field_operations is internal-only for a stronger reason than + # the other four: it publishes landowner contact details. assert ids.isdisjoint( { "locations", "avg_tds_wells", "latest_depth_to_water_wells", "other_things", + "water_well_field_operations", } ) diff --git a/tests/test_ogc_water_well_field_operations.py b/tests/test_ogc_water_well_field_operations.py new file mode 100644 index 000000000..a2b65541b --- /dev/null +++ b/tests/test_ogc_water_well_field_operations.py @@ -0,0 +1,603 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""The internal-only water well field operations layer (e1f2a3b4c5d6). + +What is worth testing here is not that the columns exist but that the layer's +three load-bearing rules hold: a well with no measurements still appears, a +permission with no record reads NULL rather than false, and a history record +whose window has closed is not treated as current. + +See docs/water-well-field-operations-layer.md. +""" + +from datetime import date, datetime, timedelta +from importlib.util import find_spec + +import pytest +from fastapi.testclient import TestClient +from sqlalchemy import text + +from core.dependencies import ( + admin_function, + amp_admin_function, + amp_editor_function, + amp_viewer_function, + editor_function, + viewer_function, +) +from core.factory import create_api_app +from db import ( + DataProvenance, + Deployment, + Notes, + PermissionHistory, + Sensor, + StatusHistory, +) +from db.engine import session_ctx +from tests import override_authentication + +pytestmark = pytest.mark.skipif( + find_spec("pygeoapi") is None, + reason="pygeoapi is not installed in this environment", +) + +VIEW = "ogc_internal_water_well_field_operations" +STATS_VIEW = "ogc_internal_water_well_field_operations_stats" + + +@pytest.fixture(scope="module") +def today(): + """The database's idea of today, not Python's. + + The view computes its `days_since_*` columns against CURRENT_DATE, which is + evaluated in the database session's timezone. That is UTC here while the + developer running the tests may not be, so a date read from Python is off + by one for part of every day. Read it from the same place the view does. + """ + with session_ctx() as session: + return session.execute(text("SELECT CURRENT_DATE")).scalar() + + +@pytest.fixture(scope="module") +def yesterday(today): + return today - timedelta(days=1) + + +@pytest.fixture(scope="module") +def last_year(today): + return today - timedelta(days=365) + + +@pytest.fixture(scope="module") +def ogc_client(): + app = create_api_app() + for dependency in ( + admin_function, + editor_function, + amp_admin_function, + amp_editor_function, + ): + app.dependency_overrides[dependency] = override_authentication( + default={"name": "foobar", "sub": "1234567890"} + ) + for dependency in (viewer_function, amp_viewer_function): + app.dependency_overrides[dependency] = override_authentication() + + with TestClient(app) as client: + yield client + + app.dependency_overrides = {} + + +def _row(session, thing_id, columns): + return session.execute( + text(f"SELECT {columns} FROM {VIEW} WHERE id = :thing_id"), + {"thing_id": thing_id}, + ).one() + + +def _refresh_stats(session): + session.execute(text(f"REFRESH MATERIALIZED VIEW {STATS_VIEW}")) + session.commit() + + +# ------------------------------------------------------------------ row set + + +def test_a_well_with_no_measurements_still_appears(water_well_thing): + # water_well_summary drops wells with no readings, because a summary of + # nothing says nothing. This layer must not: a well nobody has measured is + # exactly the well a crew needs to find. + with session_ctx() as session: + _refresh_stats(session) + row = _row( + session, + water_well_thing.id, + "name, manual_water_level_count, chemistry_sample_count, " + "continuous_reading_count, days_since_manual_water_level", + ) + + assert row.name == "Test Well" + assert row.manual_water_level_count == 0 + assert row.chemistry_sample_count == 0 + assert row.continuous_reading_count == 0 + # No reading means no elapsed time to report, not zero days since one. + assert row.days_since_manual_water_level is None + + +def test_latitude_and_longitude_match_the_geometry(water_well_thing): + with session_ctx() as session: + row = session.execute( + text( + "SELECT latitude, longitude, ST_Y(point) AS geom_y, " + f"ST_X(point) AS geom_x FROM {VIEW} WHERE id = :thing_id" + ), + {"thing_id": water_well_thing.id}, + ).one() + + # Decimal degrees on WGS 84, read off the same point the geometry carries. + assert row.latitude == row.geom_y + assert row.longitude == row.geom_x + assert -180 <= row.longitude <= 180 + assert -90 <= row.latitude <= 90 + + +def test_elevation_method_reads_the_location_provenance(water_well_thing, location): + with session_ctx() as session: + row = _row(session, water_well_thing.id, "elevation_method") + # No provenance record is "not recorded", not a method of "unknown". + assert row.elevation_method is None + + provenance = DataProvenance( + target_id=location.id, + target_table="location", + field_name="elevation", + collection_method="Survey-grade GPS", + ) + session.add(provenance) + session.commit() + + row = _row(session, water_well_thing.id, "elevation_method") + assert row.elevation_method == "Survey-grade GPS" + + session.delete(provenance) + session.commit() + + +def test_elevation_method_ignores_provenance_for_other_fields( + water_well_thing, location +): + with session_ctx() as session: + provenance = DataProvenance( + target_id=location.id, + target_table="location", + field_name="point", + collection_method="Survey-grade GPS", + ) + session.add(provenance) + session.commit() + + row = _row(session, water_well_thing.id, "elevation_method") + # How the coordinates were obtained says nothing about the elevation. + assert row.elevation_method is None + + session.delete(provenance) + session.commit() + + +# --------------------------------------------------------------- permissions + + +def test_permission_with_no_record_is_null_not_false(water_well_thing): + with session_ctx() as session: + row = _row( + session, + water_well_thing.id, + "may_measure_water_level, may_sample_water_chemistry, " + "may_install_datalogger", + ) + + # NULL means nobody has asked the landowner. Rendering it as "no" would + # tell a crew the well is off limits when the truth is unknown. + assert row.may_measure_water_level is None + assert row.may_sample_water_chemistry is None + assert row.may_install_datalogger is None + + +def test_permission_distinguishes_granted_from_refused( + water_well_thing, contact, last_year +): + with session_ctx() as session: + granted = PermissionHistory( + contact_id=contact.id, + target_id=water_well_thing.id, + target_table="thing", + permission_type="Water Level Sample", + permission_allowed=True, + start_date=last_year, + ) + refused = PermissionHistory( + contact_id=contact.id, + target_id=water_well_thing.id, + target_table="thing", + permission_type="Datalogger Installation", + permission_allowed=False, + start_date=last_year, + ) + session.add_all([granted, refused]) + session.commit() + + row = _row( + session, + water_well_thing.id, + "may_measure_water_level, may_install_datalogger, " + "may_sample_water_chemistry, permission_granted_by", + ) + + assert row.may_measure_water_level is True + assert row.may_install_datalogger is False + # Untouched permission type stays unknown rather than inheriting either. + assert row.may_sample_water_chemistry is None + assert row.permission_granted_by == contact.name + + session.delete(granted) + session.delete(refused) + session.commit() + + +def test_expired_permission_is_not_current( + water_well_thing, contact, last_year, yesterday +): + with session_ctx() as session: + expired = PermissionHistory( + contact_id=contact.id, + target_id=water_well_thing.id, + target_table="thing", + permission_type="Water Level Sample", + permission_allowed=True, + start_date=last_year, + end_date=yesterday, + ) + session.add(expired) + session.commit() + + row = _row(session, water_well_thing.id, "may_measure_water_level") + # A permission that ran out yesterday is not a permission today. This + # is the divergence from ogc_actively_monitored_wells, which ignores + # end_date entirely. + assert row.may_measure_water_level is None + + session.delete(expired) + session.commit() + + +# -------------------------------------------------------------------- status + + +def test_status_reads_the_current_record_not_the_latest( + water_well_thing, last_year, yesterday +): + with session_ctx() as session: + closed = StatusHistory( + target_id=water_well_thing.id, + target_table="thing", + status_type="Monitoring Status", + status_value="Currently monitored", + start_date=last_year, + end_date=yesterday, + reason="programme ended", + ) + open_status = StatusHistory( + target_id=water_well_thing.id, + target_table="thing", + status_type="Well Status", + status_value="Active, pumping well", + start_date=last_year, + ) + session.add_all([closed, open_status]) + session.commit() + + row = _row( + session, + water_well_thing.id, + "monitoring_status, monitoring_status_since, well_status, " + "well_status_since", + ) + + assert row.monitoring_status is None + assert row.monitoring_status_since is None + assert row.well_status == "Active, pumping well" + assert row.well_status_since == last_year + + session.delete(closed) + session.delete(open_status) + session.commit() + + +def test_status_types_do_not_bleed_into_each_other(water_well_thing, last_year): + with session_ctx() as session: + monitoring = StatusHistory( + target_id=water_well_thing.id, + target_table="thing", + status_type="Monitoring Status", + status_value="Not currently monitored", + start_date=last_year, + reason="landowner asked us to stop", + ) + session.add(monitoring) + session.commit() + + row = _row( + session, + water_well_thing.id, + "monitoring_status, monitoring_status_reason, well_status, " + "open_status, access_status, datalogger_suitability_status", + ) + + assert row.monitoring_status == "Not currently monitored" + assert row.monitoring_status_reason == "landowner asked us to stop" + assert row.well_status is None + assert row.open_status is None + assert row.access_status is None + assert row.datalogger_suitability_status is None + + session.delete(monitoring) + session.commit() + + +# ---------------------------------------------------------------- datalogger + + +def test_has_datalogger_counts_only_logger_equipment( + water_well_thing, sensor_to_water_well_thing_deployment +): + with session_ctx() as session: + row = _row( + session, + water_well_thing.id, + "has_datalogger, datalogger_deployment_count, " + "datalogger_sensor_type, datalogger_serial_no, " + "datalogger_recording_interval", + ) + + assert row.has_datalogger is True + assert row.datalogger_deployment_count == 1 + assert row.datalogger_sensor_type == "Pressure Transducer" + assert row.datalogger_serial_no == "123456" + assert row.datalogger_recording_interval == 24 + + +def test_non_logger_equipment_does_not_make_a_well_instrumented( + water_well_thing, last_year +): + with session_ctx() as session: + barometer = Sensor( + name="Test Barometer", + sensor_type="Barometer", + sensor_status="In Service", + release_status="draft", + ) + session.add(barometer) + session.commit() + deployment = Deployment( + sensor_id=barometer.id, + thing_id=water_well_thing.id, + installation_date=last_year, + removal_date=None, + ) + session.add(deployment) + session.commit() + + row = _row( + session, + water_well_thing.id, + "has_datalogger, datalogger_deployment_count", + ) + # A barometer at the well is equipment, not a logger in the well. + assert row.has_datalogger is False + assert row.datalogger_deployment_count == 0 + + session.delete(deployment) + session.delete(barometer) + session.commit() + + +def test_a_removed_logger_leaves_the_well_uninstrumented( + water_well_thing, sensor, sensor_to_water_well_thing_deployment, yesterday +): + with session_ctx() as session: + deployment = session.get(Deployment, sensor_to_water_well_thing_deployment.id) + deployment.removal_date = yesterday + session.commit() + + row = _row(session, water_well_thing.id, "has_datalogger") + assert row.has_datalogger is False + + deployment.removal_date = None + session.commit() + + +# ------------------------------------------------------------- measurements + + +def test_last_depth_to_water_uses_the_measuring_point_convention( + water_well_thing, groundwater_level_sample, today +): + from db import Observation + from tests import get_parameter_id + + with session_ctx() as session: + readings = [] + for day, value, measuring_point_height in ((1, 6.0, 1.0), (2, 9.0, 2.0)): + observation = Observation( + observation_datetime=datetime(2025, 1, day, 12, 0, 0), + sample_id=groundwater_level_sample.id, + parameter_id=get_parameter_id("groundwater level", "Field Parameter"), + release_status="public", + value=value, + unit="ft", + measuring_point_height=measuring_point_height, + groundwater_level_reason="Water level not affected", + ) + session.add(observation) + readings.append(observation) + session.commit() + _refresh_stats(session) + + row = _row( + session, + water_well_thing.id, + "manual_water_level_count, manual_water_level_first_date, " + "manual_water_level_last_date, last_depth_to_water_ft, " + "days_since_manual_water_level", + ) + + assert row.manual_water_level_count == 2 + assert row.manual_water_level_first_date == date(2025, 1, 1) + assert row.manual_water_level_last_date == date(2025, 1, 2) + # Latest reading is 9 ft from a measuring point 2 ft above ground. + assert abs(float(row.last_depth_to_water_ft) - 7.0) < 1e-9 + # Computed against today, not against the last refresh. + assert row.days_since_manual_water_level == (today - date(2025, 1, 2)).days + + for observation in readings: + session.delete(observation) + session.commit() + _refresh_stats(session) + + +def test_private_readings_are_counted_on_this_internal_layer( + water_well_thing, groundwater_level_sample +): + from db import Observation + from tests import get_parameter_id + + with session_ctx() as session: + observation = Observation( + observation_datetime=datetime(2025, 2, 1, 12, 0, 0), + sample_id=groundwater_level_sample.id, + parameter_id=get_parameter_id("groundwater level", "Field Parameter"), + release_status="private", + value=4.0, + unit="ft", + measuring_point_height=1.0, + groundwater_level_reason="Water level not affected", + ) + session.add(observation) + session.commit() + _refresh_stats(session) + + row = _row(session, water_well_thing.id, "manual_water_level_count") + # The internal mount is unfiltered by design; this layer has no public + # twin to keep in step with. + assert row.manual_water_level_count == 1 + + session.delete(observation) + session.commit() + _refresh_stats(session) + + +# ------------------------------------------------------ multi-valued columns + + +def test_multi_valued_columns_are_comma_joined_text( + water_well_thing, domestic_well_purpose, irrigation_well_purpose +): + with session_ctx() as session: + row = _row(session, water_well_thing.id, "well_purposes") + + # Text, not an array: this layer is exported to File Geodatabase and + # GeoPackage for offline field use, and neither format has a list type. + assert isinstance(row.well_purposes, str) + assert row.well_purposes == "Domestic, Irrigation" + + +def test_contacts_and_access_notes_are_published(water_well_thing, contact, phone): + with session_ctx() as session: + note = Notes( + target_id=water_well_thing.id, + target_table="thing", + note_type="Access", + content="Gate is locked, call ahead", + ) + session.add(note) + session.commit() + + row = _row( + session, + water_well_thing.id, + "contact_count, primary_contact_name, primary_contact_type, " + "primary_contact_role, primary_contact_phone, access_notes", + ) + + assert row.contact_count == 1 + assert row.primary_contact_name == contact.name + assert row.primary_contact_type == "Primary" + assert row.primary_contact_role == "Owner" + assert row.primary_contact_phone == phone.phone_number + assert row.access_notes == "Gate is locked, call ahead" + + session.delete(note) + session.commit() + + +# ------------------------------------------------------------------- mounts + + +def test_layer_is_served_on_the_internal_mount_only(ogc_client): + internal = ogc_client.get( + "/ogcapi-internal/collections/water_well_field_operations/items?limit=1" + ) + assert internal.status_code == 200 + assert internal.json()["type"] == "FeatureCollection" + + public = ogc_client.get( + "/ogcapi/collections/water_well_field_operations/items?limit=1" + ) + # The layer publishes landowner contact details; it must not exist at all + # on the anonymous mount. + assert public.status_code == 404 + + +def test_every_column_is_documented_on_the_internal_mount(ogc_client): + response = ogc_client.get( + "/ogcapi-internal/collections/water_well_field_operations/schema" + ) + assert response.status_code == 200 + + properties = response.json()["properties"] + gaps = [ + name + for name, prop in properties.items() + if name != "geometry" and not prop.get("description") + ] + assert not gaps, f"columns with no YAML entry: {gaps}" + + +def test_permission_columns_explain_their_null_meaning(ogc_client): + response = ogc_client.get( + "/ogcapi-internal/collections/water_well_field_operations/schema" + ) + properties = response.json()["properties"] + + # The three-valued meaning only reaches a consumer through this prose. + for column in ( + "may_measure_water_level", + "may_sample_water_chemistry", + "may_install_datalogger", + ): + assert "null" in properties[column]["description"].lower() diff --git a/tests/test_pygeoapi_mount.py b/tests/test_pygeoapi_mount.py index e267d1803..2572874ed 100644 --- a/tests/test_pygeoapi_mount.py +++ b/tests/test_pygeoapi_mount.py @@ -115,12 +115,15 @@ def test_loading_a_mount_restores_config_env_vars(): # Layers hidden from the public catalog but still served to staff GIS # clients on /ogcapi-internal: locations duplicates the thing-type layers # (BDMS-978), avg_tds_wells and latest_depth_to_water_wells are misleading -# or redundant (BDMS-977), other_things is internal vocabulary (BDMS-979). +# or redundant (BDMS-977), other_things is internal vocabulary (BDMS-979), +# and water_well_field_operations carries landowner contact details and +# staff-written access notes, so it has no public form at all. INTERNAL_ONLY_COLLECTIONS = { "locations", "avg_tds_wells", "latest_depth_to_water_wells", "other_things", + "water_well_field_operations", } From c96f717247dbac46b2a58cfda7e1e16113d0cec3 Mon Sep 17 00:00:00 2001 From: jirhiker <2035568+jirhiker@users.noreply.github.com> Date: Tue, 1 Sep 2026 02:23:37 +0000 Subject: [PATCH 2/2] Formatting changes --- tests/test_cli_commands.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/test_cli_commands.py b/tests/test_cli_commands.py index 257e4c320..9923fb38c 100644 --- a/tests/test_cli_commands.py +++ b/tests/test_cli_commands.py @@ -705,12 +705,10 @@ def _write_csv(path: Path, *, well_name: str, notes: str): "Water level accurate to within two hundreths of a foot," f"{notes}" ) - csv_text = textwrap.dedent( - f"""\ + csv_text = textwrap.dedent(f"""\ {header} {row} - """ - ) + """) path.write_text(csv_text) unique_notes = f"pytest-{uuid.uuid4()}"