From 9fb2410af513b220e5e5a82c84c7a9219b541f1e Mon Sep 17 00:00:00 2001 From: jakeross Date: Mon, 31 Aug 2026 19:23:07 -0700 Subject: [PATCH 1/5] feat(ogc): add the water well field operations internal layer Adds water_well_field_operations, an OGC API - Features collection on /ogcapi-internal only: one feature per water well carrying what a field crew needs to plan and execute a visit. Every other well layer answers a scientific question -- water level, chemistry, trend. This one answers an operational one: may we go to this well, what are we allowed to do when we get there, and is it overdue? 86 columns: identity and location (including latitude and longitude in decimal degrees and how the elevation was obtained), construction detail, the current measuring point, all five status types, the three standing landowner permissions, monitoring frequency and group membership, manual water-level and chemistry-sampling statistics, field-visit history, the installed logger and its continuous record, landowner contacts, and access and directions notes. Two relations rather than one (migration e1f2a3b4c5d6): ogc_internal_water_well_field_operations_stats materialized, nightly refresh Aggregates over observation and transducer_observation. ogc_internal_water_well_field_operations plain view, served by pygeoapi Live join of thing, location, status_history, permission_history, measuring_point_history, monitoring_frequency_history, deployment, sensor, contact and notes, LEFT JOINed to the stats matview. The split is not tidiness. Staleness is dangerous on exactly the columns that are cheap to read: a revoked sampling permission that still reads true until the next nightly refresh sends a crew onto land they are no longer welcome on. The current-record rule is also written against CURRENT_DATE, which inside a materialized view would freeze at refresh time. Three decisions worth review: * Internal-only with no public twin. The layer publishes landowner and operator contact details -- name, organization, role, phone, email -- alongside staff-written access notes. ogc_water_well_field_operations does not exist and must never be created. Every credential the internal mount accepts now reaches personal information, the static desktop-GIS keys included; notices added to CLAUDE.md and docs/internal-ogc-desktop-gis.md. * The three permission columns are three-valued. NULL means no permission is on record, which is not a refusal -- it means nobody has asked the landowner yet. Collapsing it to false would tell a crew a well is off limits when the truth is unknown. * History reads honour end_date, unlike ogc_actively_monitored_wells, which takes the greatest start_date and lets a status closed in 2019 read as current. Written up in docs/ogc_conventions.md. Requested as amp_field_well_operations and renamed before merge per the pre-merge check in docs/ogc_conventions.md: amp is an unexpanded abbreviation whose expansion is recorded nowhere in this codebase, AMP labels the layer's audience rather than filtering its rows (the row set is every water well), and Group B analytic layers prefer a water_well_ prefix. The AMP crews stay named in the collection description. Multi-valued columns are comma-joined text rather than arrays: this layer is pulled into ArcGIS Pro and QGIS and exported to File Geodatabase and GeoPackage for offline field use, and neither format has a list type. Depth to water uses the (value - COALESCE(measuring_point_height, 0)) convention shared with ogc_water_well_summary, ogc_latest_depth_to_water_wells and ogc_well_water_column, and elevation_method reuses that summary's provenance lookup verbatim, so the layers cannot disagree. Twelve supporting indexes come with the migration; none of the foreign keys these lookups traverse was indexed. Design rationale in docs/water-well-field-operations-layer.md, column-by-column sources in docs/water-well-field-operations-columns.md. Co-Authored-By: Claude Opus 5 --- CLAUDE.md | 12 + ...6_add_water_well_field_operations_layer.py | 718 ++++++++++++++++++ core/ogc-field-descriptions.yml | 391 ++++++++++ core/pygeoapi-config-internal.yml | 44 ++ docs/internal-ogc-desktop-gis.md | 10 + docs/ogc_conventions.md | 27 + docs/water-well-field-operations-columns.md | 106 +++ docs/water-well-field-operations-layer.md | 519 +++++++++++++ services/materialized_views.py | 4 + tests/test_cli_commands.py | 9 +- tests/test_ogc.py | 3 + tests/test_ogc_water_well_field_operations.py | 603 +++++++++++++++ tests/test_pygeoapi_mount.py | 5 +- 13 files changed, 2447 insertions(+), 4 deletions(-) create mode 100644 alembic/versions/e1f2a3b4c5d6_add_water_well_field_operations_layer.py create mode 100644 docs/water-well-field-operations-columns.md create mode 100644 docs/water-well-field-operations-layer.md create mode 100644 tests/test_ogc_water_well_field_operations.py diff --git a/CLAUDE.md b/CLAUDE.md index a5f9358c7..bcc472bb1 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -204,6 +204,18 @@ ArcGIS Pro cannot send a bearer token at all and neither desktop client can refresh an Authentik token. Read **`docs/internal-ogc-desktop-gis.md`** before changing the credential paths. +**`/ogcapi-internal` carries landowner PII.** The `water_well_field_operations` +collection publishes contact name, organization, role, phone and email for well +owners and operators, plus staff-written access notes. Every credential the +mount accepts reaches it, the static keys included. It is internal-only with +**no public twin** — `ogc_water_well_field_operations` does not exist and must +never be created. The layer also honours `end_date` when reading history +tables, unlike `ogc_actively_monitored_wells`, so "may we sample here" cannot +outlive the permission that granted it. Read +**`docs/water-well-field-operations-layer.md`** before changing it, and +**`docs/water-well-field-operations-columns.md`** for where each column comes +from. + ### OGC field descriptions Per-column `title`/`description`/unit for every collection lives in diff --git a/alembic/versions/e1f2a3b4c5d6_add_water_well_field_operations_layer.py b/alembic/versions/e1f2a3b4c5d6_add_water_well_field_operations_layer.py new file mode 100644 index 000000000..12276cf41 --- /dev/null +++ b/alembic/versions/e1f2a3b4c5d6_add_water_well_field_operations_layer.py @@ -0,0 +1,718 @@ +"""add the water well field operations layer + +An internal-only OGC layer for field crews: one row per water well, carrying +what a crew needs to plan and execute a visit -- where the well is, who owns it +and how to reach them, what the crew is permitted to do there, what is +installed in it, and when it was last measured. Every other well layer answers +a scientific question; this one answers an operational one. + +Two relations, not one: + + ogc_internal_water_well_field_operations_stats MATERIALIZED VIEW + Counts, first/last dates and aggregates over `observation` and + `transducer_observation`. Expensive, refreshed nightly by the pg_cron + job (b6c7d8e9f0a1), which discovers matviews from the catalog and so + needs no schedule change. + + ogc_internal_water_well_field_operations VIEW <- pygeoapi reads this + Live join of thing, location, status_history, permission_history, + measuring_point_history, monitoring_frequency_history, deployment, + sensor, contact and notes, LEFT JOINed to the stats matview. + +The split is not tidiness. Staleness is dangerous on exactly the columns that +are cheap to read: a revoked sampling permission that still reads `true` until +the next nightly refresh sends a crew onto land they are no longer welcome on. +And the current-record rule below is written against CURRENT_DATE, which in a +materialized view would freeze at refresh time -- a permission that expired +this morning would still read current tonight. Those columns have to be in the +plain view. + +Internal only, and deliberately without a public twin: the layer publishes +landowner contact details and staff-written access notes. +`ogc_water_well_field_operations` does not exist and must never be created. The +parity guard in tests/test_migration_view_parity.py reads two named migration +files (f4a5b6c7d8e9 and 2d3c3a268652) and is unaffected by a relation created +here. + +Current-record rule, applied identically to all four history tables +(status_history, permission_history, measuring_point_history, +monitoring_frequency_history): + + start_date <= CURRENT_DATE + AND (end_date IS NULL OR end_date >= CURRENT_DATE) + ORDER BY start_date DESC, id DESC -- via DISTINCT ON + +This diverges from ogc_actively_monitored_wells, which takes the greatest +start_date and ignores end_date entirely, so a status closed in 2019 still +reads as current there. Tolerable on a summary layer; not on one whose job is +to say what is true today. See docs/ogc_conventions.md. + +The three permission columns are three-valued and must stay that way: true = +a current grant says allowed, false = a current grant says not allowed, NULL = +no permission on record. NULL is not "denied" -- it means nobody has asked the +landowner yet. + +Depth to water uses (value - COALESCE(measuring_point_height, 0)) -- the +reading minus the height of the measuring point above ground, a missing height +treated as ground level. Character-for-character the convention in +ogc_water_well_summary, ogc_latest_depth_to_water_wells and +ogc_well_water_column, so the four layers cannot disagree about what a depth to +water is. + +Multi-valued columns are emitted as comma-joined text rather than as arrays +(which ogc_actively_monitored_wells uses). This layer exists to be pulled into +ArcGIS Pro and QGIS and exported to a File Geodatabase or GeoPackage for +offline field use, and neither format has a list type. + +Both relations are new here, so downgrade() drops them rather than restoring a +prior state. The supporting indexes are also created here: none of these +foreign keys was indexed (Postgres does not index foreign keys on its own). + +Revision ID: e1f2a3b4c5d6 +Revises: c9d0e1f2a3b4 +Create Date: 2026-08-31 00:00:00.000000 +""" + +import re +from typing import Sequence, Union + +from alembic import op +from sqlalchemy import inspect, text + +# revision identifiers, used by Alembic. +revision: str = "e1f2a3b4c5d6" +down_revision: Union[str, Sequence[str], None] = "c9d0e1f2a3b4" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +REQUIRED_TABLES = { + "thing", + "location", + "location_thing_association", + "observation", + "sample", + "field_activity", + "field_event", + "data_provenance", + "status_history", + "permission_history", + "measuring_point_history", + "monitoring_frequency_history", + "deployment", + "sensor", + "transducer_observation", + "contact", + "thing_contact_association", + "phone", + "email", + "notes", + "group", + "group_thing_association", + "well_purpose", + "well_casing_material", + "well_screen", + "thing_id_link", +} + +STATS_VIEW = "ogc_internal_water_well_field_operations_stats" +FEATURE_VIEW = "ogc_internal_water_well_field_operations" + +LATEST_LOCATION_CTE = """ +SELECT DISTINCT ON (lta.thing_id) + lta.thing_id, + lta.location_id, + lta.effective_start +FROM location_thing_association AS lta +WHERE lta.effective_end IS NULL +ORDER BY lta.thing_id, lta.effective_start DESC +""".strip() + +# sensor_type values that mean "this well is logging by itself". Anything else +# deployed at a well (a barometer, a camera) is equipment, not a logger, and +# must not make has_datalogger true. +LOGGER_SENSOR_TYPES = ( + "'Data Logger'", + "'Pressure Transducer'", + "'DiverLink'", + "'Diver Cable'", +) + +# Indexes the per-well lookups need. Each is created IF NOT EXISTS: none of +# these existed when this migration was written, but the thing_type one in +# particular is the sort of thing another migration may add first. +SUPPORTING_INDEXES = [ + ( + "ix_status_history_target_type_start", + "status_history (target_table, target_id, status_type, start_date DESC)", + ), + ( + "ix_permission_history_target_type_start", + "permission_history " + "(target_table, target_id, permission_type, start_date DESC)", + ), + ( + "ix_measuring_point_history_thing_start", + "measuring_point_history (thing_id, start_date DESC)", + ), + ( + "ix_monitoring_frequency_history_thing_start", + "monitoring_frequency_history (thing_id, start_date DESC)", + ), + ("ix_deployment_thing_id", "deployment (thing_id)"), + ("ix_thing_contact_association_thing_id", "thing_contact_association (thing_id)"), + ( + "ix_thing_contact_association_contact_id", + "thing_contact_association (contact_id)", + ), + ("ix_well_purpose_thing_id", "well_purpose (thing_id)"), + ("ix_well_casing_material_thing_id", "well_casing_material (thing_id)"), + ("ix_well_screen_thing_id", "well_screen (thing_id)"), + ("ix_thing_id_link_thing_id", "thing_id_link (thing_id)"), + ("ix_thing_thing_type", "thing (thing_type)"), + # No index on transducer_observation (deployment_id, ...): the existing + # uq_transducer_observation_deployment_parameter_datetime leads with + # deployment_id and already serves the per-deployment aggregate. +] + + +def _safe_relation_name(name: str) -> str: + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", name): + raise ValueError(f"Unsafe relation name: {name!r}") + return name + + +def _check_required_tables() -> None: + bind = op.get_bind() + inspector = inspect(bind) + existing_tables = set(inspector.get_table_names(schema="public")) + missing = REQUIRED_TABLES - existing_tables + if missing: + raise RuntimeError( + "Cannot create the water well field operations layer. " + f"Missing required tables: {', '.join(sorted(missing))}" + ) + + +def _create_stats_view() -> str: + """The expensive half: aggregates over the observation-scale tables.""" + safe_name = _safe_relation_name(STATS_VIEW) + return f""" + CREATE MATERIALIZED VIEW {safe_name} AS + WITH wells AS ( + SELECT t.id AS thing_id + FROM thing AS t + WHERE t.thing_type = 'water well' + ), + manual_obs AS ( + SELECT + fe.thing_id, + o.id AS observation_id, + o.observation_datetime, + (o.value - COALESCE(o.measuring_point_height, 0)) AS depth_to_water + FROM observation AS o + JOIN sample AS s ON s.id = o.sample_id + JOIN field_activity AS fa ON fa.id = s.field_activity_id + JOIN field_event AS fe ON fe.id = fa.field_event_id + JOIN wells AS w ON w.thing_id = fe.thing_id + WHERE + fa.activity_type = 'groundwater level' + AND o.value IS NOT NULL + AND o.observation_datetime IS NOT NULL + ), + manual_agg AS ( + SELECT + m.thing_id, + COUNT(*)::integer AS manual_water_level_count, + ( + MIN(m.observation_datetime) AT TIME ZONE 'UTC' + )::date AS manual_water_level_first_date, + ( + MAX(m.observation_datetime) AT TIME ZONE 'UTC' + )::date AS manual_water_level_last_date + FROM manual_obs AS m + GROUP BY m.thing_id + ), + manual_last AS ( + SELECT DISTINCT ON (m.thing_id) + m.thing_id, + m.depth_to_water AS last_depth_to_water_ft + FROM manual_obs AS m + ORDER BY m.thing_id, m.observation_datetime DESC, m.observation_id DESC + ), + chemistry_agg AS ( + SELECT + fe.thing_id, + COUNT(DISTINCT s.id)::integer AS chemistry_sample_count, + ( + MAX(s.sample_date) AT TIME ZONE 'UTC' + )::date AS chemistry_sample_last_date + FROM sample AS s + JOIN field_activity AS fa ON fa.id = s.field_activity_id + JOIN field_event AS fe ON fe.id = fa.field_event_id + JOIN wells AS w ON w.thing_id = fe.thing_id + WHERE fa.activity_type = 'water chemistry' + GROUP BY fe.thing_id + ), + field_event_agg AS ( + SELECT + fe.thing_id, + COUNT(*)::integer AS field_event_count, + ( + MAX(fe.event_date) AT TIME ZONE 'UTC' + )::date AS field_event_last_date + FROM field_event AS fe + JOIN wells AS w ON w.thing_id = fe.thing_id + GROUP BY fe.thing_id + ), + continuous_agg AS ( + SELECT + d.thing_id, + COUNT(*)::bigint AS continuous_reading_count, + MIN(tobs.observation_datetime) AS continuous_first_datetime, + MAX(tobs.observation_datetime) AS continuous_last_datetime + FROM transducer_observation AS tobs + JOIN deployment AS d ON d.id = tobs.deployment_id + JOIN wells AS w ON w.thing_id = d.thing_id + GROUP BY d.thing_id + ) + SELECT + w.thing_id, + COALESCE(ma.manual_water_level_count, 0) AS manual_water_level_count, + ma.manual_water_level_first_date, + ma.manual_water_level_last_date, + ml.last_depth_to_water_ft, + COALESCE(ca.chemistry_sample_count, 0) AS chemistry_sample_count, + ca.chemistry_sample_last_date, + COALESCE(fea.field_event_count, 0) AS field_event_count, + fea.field_event_last_date, + COALESCE(co.continuous_reading_count, 0) AS continuous_reading_count, + co.continuous_first_datetime, + co.continuous_last_datetime + FROM wells AS w + LEFT JOIN manual_agg AS ma ON ma.thing_id = w.thing_id + LEFT JOIN manual_last AS ml ON ml.thing_id = w.thing_id + LEFT JOIN chemistry_agg AS ca ON ca.thing_id = w.thing_id + LEFT JOIN field_event_agg AS fea ON fea.thing_id = w.thing_id + LEFT JOIN continuous_agg AS co ON co.thing_id = w.thing_id + """ + + +def _create_feature_view() -> str: + """The live half: everything a stale answer would misreport.""" + safe_name = _safe_relation_name(FEATURE_VIEW) + safe_stats = _safe_relation_name(STATS_VIEW) + logger_types = ", ".join(LOGGER_SENSOR_TYPES) + return f""" + CREATE VIEW {safe_name} AS + WITH latest_location AS ( +{LATEST_LOCATION_CTE} + ), + current_status AS ( + SELECT DISTINCT ON (sh.target_id, sh.status_type) + sh.target_id AS thing_id, + sh.status_type, + sh.status_value, + sh.start_date, + sh.reason + FROM status_history AS sh + WHERE + sh.target_table = 'thing' + AND sh.start_date <= CURRENT_DATE + AND (sh.end_date IS NULL OR sh.end_date >= CURRENT_DATE) + ORDER BY sh.target_id, sh.status_type, sh.start_date DESC, sh.id DESC + ), + current_permission AS ( + SELECT DISTINCT ON (ph.target_id, ph.permission_type) + ph.target_id AS thing_id, + ph.permission_type, + ph.permission_allowed, + ph.contact_id + FROM permission_history AS ph + WHERE + ph.target_table = 'thing' + AND ph.start_date <= CURRENT_DATE + AND (ph.end_date IS NULL OR ph.end_date >= CURRENT_DATE) + ORDER BY + ph.target_id, ph.permission_type, ph.start_date DESC, ph.id DESC + ), + current_measuring_point AS ( + SELECT DISTINCT ON (mp.thing_id) + mp.thing_id, + mp.measuring_point_height, + mp.measuring_point_description, + mp.start_date + FROM measuring_point_history AS mp + WHERE + mp.start_date <= CURRENT_DATE + AND (mp.end_date IS NULL OR mp.end_date >= CURRENT_DATE) + ORDER BY mp.thing_id, mp.start_date DESC, mp.id DESC + ), + current_monitoring_frequency AS ( + SELECT DISTINCT ON (mf.thing_id) + mf.thing_id, + mf.monitoring_frequency, + mf.start_date + FROM monitoring_frequency_history AS mf + WHERE + mf.start_date <= CURRENT_DATE + AND (mf.end_date IS NULL OR mf.end_date >= CURRENT_DATE) + ORDER BY mf.thing_id, mf.start_date DESC, mf.id DESC + ), + logger_deployments AS ( + SELECT + d.id AS deployment_id, + d.thing_id, + d.installation_date, + d.recording_interval, + d.recording_interval_units, + d.hanging_point_description, + se.sensor_type, + se.model, + se.serial_no, + se.sensor_status + FROM deployment AS d + JOIN sensor AS se ON se.id = d.sensor_id + WHERE + d.installation_date IS NOT NULL + AND d.removal_date IS NULL + AND se.sensor_type IN ({logger_types}) + ), + current_logger AS ( + SELECT DISTINCT ON (ld.thing_id) ld.* + FROM logger_deployments AS ld + ORDER BY ld.thing_id, ld.installation_date DESC, ld.deployment_id DESC + ), + logger_count AS ( + SELECT + ld.thing_id, + COUNT(*)::integer AS datalogger_deployment_count + FROM logger_deployments AS ld + GROUP BY ld.thing_id + ), + thing_contacts AS ( + SELECT + tca.thing_id, + c.id AS contact_id, + c.name, + c.organization, + c.role, + c.contact_type + FROM thing_contact_association AS tca + JOIN contact AS c ON c.id = tca.contact_id + ), + contact_agg AS ( + SELECT + tc.thing_id, + COUNT(DISTINCT tc.contact_id)::integer AS contact_count, + string_agg( + DISTINCT tc.name, ', ' ORDER BY tc.name + ) AS contact_names + FROM thing_contacts AS tc + GROUP BY tc.thing_id + ), + primary_contact AS ( + -- Prefer a contact recorded as Primary, but fall back to any + -- contact on the well rather than publishing a blank name next to + -- a non-zero contact_count. primary_contact_type says which case + -- this is, so the fallback is visible rather than implied. + SELECT DISTINCT ON (tc.thing_id) + tc.thing_id, + tc.contact_id, + tc.name, + tc.organization, + tc.role, + tc.contact_type + FROM thing_contacts AS tc + ORDER BY + tc.thing_id, + (tc.contact_type = 'Primary') DESC NULLS LAST, + tc.contact_id + ) + SELECT + t.id AS id, + t.name, + 'water well'::text AS thing_type, + t.release_status, + t.nma_pk_welldata, + ( + SELECT string_agg( + DISTINCT + COALESCE(tl.alternate_organization, 'unknown') + || ': ' + || tl.alternate_id, + ', ' + ORDER BY + COALESCE(tl.alternate_organization, 'unknown') + || ': ' + || tl.alternate_id + ) + FROM thing_id_link AS tl + WHERE tl.thing_id = t.id + ) AS alternate_ids, + l.county, + l.state, + l.quad_name, + -- Decimal degrees alongside the geometry. A crew types these into + -- a handheld GPS or reads them over the radio; the geometry column + -- is for the map, and a CSV export of this layer drops it. + ST_Y(l.point) AS latitude, + ST_X(l.point) AS longitude, + l.elevation, + dpl.collection_method AS elevation_method, + + -- Construction, same column names as the thing-type views. + t.well_depth, + t.hole_depth, + t.well_casing_diameter, + t.well_casing_depth, + t.well_completion_date, + t.well_driller_name, + t.well_construction_method, + t.well_pump_type, + t.well_pump_depth, + t.formation_completion_code, + t.nma_formation_zone, + ( + SELECT string_agg(DISTINCT wp.purpose, ', ' ORDER BY wp.purpose) + FROM well_purpose AS wp + WHERE wp.thing_id = t.id + ) AS well_purposes, + ( + SELECT string_agg( + DISTINCT wcm.material, ', ' ORDER BY wcm.material + ) + FROM well_casing_material AS wcm + WHERE wcm.thing_id = t.id + ) AS well_casing_materials, + scr.screen_count, + scr.screen_depth_top, + scr.screen_depth_bottom, + + -- Measuring point, current record. + cmp.measuring_point_height, + cmp.measuring_point_description, + cmp.start_date AS measuring_point_start_date, + + -- Status, current record per status type. + well_st.status_value AS well_status, + well_st.start_date AS well_status_since, + mon_st.status_value AS monitoring_status, + mon_st.start_date AS monitoring_status_since, + mon_st.reason AS monitoring_status_reason, + acc_st.status_value AS access_status, + acc_st.start_date AS access_status_since, + open_st.status_value AS open_status, + open_st.start_date AS open_status_since, + dl_st.status_value AS datalogger_suitability_status, + dl_st.start_date AS datalogger_suitability_status_since, + + -- Permission, current grants. Three-valued: NULL means no + -- permission is on record, which is not the same as denied. + wl_perm.permission_allowed AS may_measure_water_level, + chem_perm.permission_allowed AS may_sample_water_chemistry, + dl_perm.permission_allowed AS may_install_datalogger, + granter.name AS permission_granted_by, + + -- Monitoring programme. + cmf.monitoring_frequency, + cmf.start_date AS monitoring_frequency_since, + grp.group_names, + grp.group_types, + + -- Manual water levels (from the stats matview). + COALESCE(st.manual_water_level_count, 0) AS manual_water_level_count, + st.manual_water_level_first_date, + st.manual_water_level_last_date, + ( + CURRENT_DATE - st.manual_water_level_last_date + ) AS days_since_manual_water_level, + st.last_depth_to_water_ft, + + -- Chemistry sampling (from the stats matview). + COALESCE(st.chemistry_sample_count, 0) AS chemistry_sample_count, + st.chemistry_sample_last_date, + ( + CURRENT_DATE - st.chemistry_sample_last_date + ) AS days_since_chemistry_sample, + + -- Field visits (from the stats matview). + COALESCE(st.field_event_count, 0) AS field_event_count, + st.field_event_last_date, + + -- Data logger, current deployment. + (cl.deployment_id IS NOT NULL) AS has_datalogger, + COALESCE(lc.datalogger_deployment_count, 0) + AS datalogger_deployment_count, + cl.sensor_type AS datalogger_sensor_type, + cl.model AS datalogger_model, + cl.serial_no AS datalogger_serial_no, + cl.sensor_status AS datalogger_sensor_status, + cl.installation_date AS datalogger_installed_date, + cl.recording_interval AS datalogger_recording_interval, + cl.recording_interval_units AS datalogger_recording_interval_units, + cl.hanging_point_description AS datalogger_hanging_point_description, + COALESCE(st.continuous_reading_count, 0) AS continuous_reading_count, + st.continuous_first_datetime, + st.continuous_last_datetime, + ( + CURRENT_DATE + - (st.continuous_last_datetime AT TIME ZONE 'UTC')::date + ) AS days_since_continuous_reading, + + -- Contacts. + COALESCE(cag.contact_count, 0) AS contact_count, + pc.name AS primary_contact_name, + pc.organization AS primary_contact_organization, + pc.role AS primary_contact_role, + pc.contact_type AS primary_contact_type, + ( + SELECT p.phone_number + FROM phone AS p + WHERE p.contact_id = pc.contact_id + ORDER BY p.id + LIMIT 1 + ) AS primary_contact_phone, + ( + SELECT e.email + FROM email AS e + WHERE e.contact_id = pc.contact_id + ORDER BY e.id + LIMIT 1 + ) AS primary_contact_email, + cag.contact_names, + + -- Visit instructions. Separator is ' | ' rather than ', ' because + -- the content is free text and routinely contains commas. + ( + SELECT string_agg(n.content, ' | ' ORDER BY n.id DESC) + FROM notes AS n + WHERE + n.target_table = 'thing' + AND n.target_id = t.id + AND n.note_type = 'Access' + ) AS access_notes, + ( + SELECT string_agg(n.content, ' | ' ORDER BY n.id DESC) + FROM notes AS n + WHERE + n.target_table = 'thing' + AND n.target_id = t.id + AND n.note_type = 'Directions' + ) AS directions_notes, + + l.point + FROM thing AS t + JOIN latest_location AS ll ON ll.thing_id = t.id + JOIN location AS l ON l.id = ll.location_id + LEFT JOIN {safe_stats} AS st ON st.thing_id = t.id + LEFT JOIN current_status AS well_st + ON well_st.thing_id = t.id AND well_st.status_type = 'Well Status' + LEFT JOIN current_status AS mon_st + ON mon_st.thing_id = t.id + AND mon_st.status_type = 'Monitoring Status' + LEFT JOIN current_status AS acc_st + ON acc_st.thing_id = t.id AND acc_st.status_type = 'Access Status' + LEFT JOIN current_status AS open_st + ON open_st.thing_id = t.id AND open_st.status_type = 'Open Status' + LEFT JOIN current_status AS dl_st + ON dl_st.thing_id = t.id + AND dl_st.status_type = 'Datalogger Suitability Status' + LEFT JOIN current_permission AS wl_perm + ON wl_perm.thing_id = t.id + AND wl_perm.permission_type = 'Water Level Sample' + LEFT JOIN current_permission AS chem_perm + ON chem_perm.thing_id = t.id + AND chem_perm.permission_type = 'Water Chemistry Sample' + LEFT JOIN current_permission AS dl_perm + ON dl_perm.thing_id = t.id + AND dl_perm.permission_type = 'Datalogger Installation' + LEFT JOIN contact AS granter ON granter.id = wl_perm.contact_id + LEFT JOIN current_measuring_point AS cmp ON cmp.thing_id = t.id + LEFT JOIN current_monitoring_frequency AS cmf ON cmf.thing_id = t.id + LEFT JOIN current_logger AS cl ON cl.thing_id = t.id + LEFT JOIN logger_count AS lc ON lc.thing_id = t.id + LEFT JOIN contact_agg AS cag ON cag.thing_id = t.id + LEFT JOIN primary_contact AS pc ON pc.thing_id = t.id + LEFT JOIN LATERAL ( + -- Character-for-character the elevation-provenance lookup in + -- ogc_water_well_summary (2d3c3a268652), so the two layers cannot + -- disagree about where a well's elevation came from. + SELECT dp.collection_method + FROM data_provenance AS dp + WHERE + dp.target_table = 'location' + AND dp.target_id = l.id + AND dp.field_name = 'elevation' + ORDER BY dp.id DESC + LIMIT 1 + ) AS dpl ON true + LEFT JOIN LATERAL ( + SELECT + COUNT(*)::integer AS screen_count, + MIN(ws.screen_depth_top) AS screen_depth_top, + MAX(ws.screen_depth_bottom) AS screen_depth_bottom + FROM well_screen AS ws + WHERE ws.thing_id = t.id + ) AS scr ON TRUE + LEFT JOIN LATERAL ( + -- group_thing_association has no unique constraint on + -- (group_id, thing_id), so DISTINCT before aggregating. Both + -- strings are ordered by the same group name so they stay + -- index-aligned with each other. + SELECT + string_agg(dm.group_name, ', ' ORDER BY dm.group_name) + AS group_names, + string_agg(dm.group_type, ', ' ORDER BY dm.group_name) + AS group_types + FROM ( + SELECT DISTINCT + g.id AS group_id, + g.name AS group_name, + g.group_type + FROM group_thing_association AS gta + JOIN "group" AS g ON g.id = gta.group_id + WHERE gta.thing_id = t.id + ) AS dm + ) AS grp ON TRUE + WHERE t.thing_type = 'water well' + """ + + +def upgrade() -> None: + _check_required_tables() + + safe_stats = _safe_relation_name(STATS_VIEW) + safe_feature = _safe_relation_name(FEATURE_VIEW) + + # The feature view depends on the stats matview, so it is dropped first and + # created last. + op.execute(text(f"DROP VIEW IF EXISTS {safe_feature}")) + op.execute(text(f"DROP MATERIALIZED VIEW IF EXISTS {safe_stats}")) + + op.execute(text(_create_stats_view())) + # Unique index required for REFRESH MATERIALIZED VIEW CONCURRENTLY. + op.execute( + text( + f"CREATE UNIQUE INDEX ix_{safe_stats}_thing_id " + f"ON {safe_stats} (thing_id)" + ) + ) + op.execute(text(_create_feature_view())) + + for index_name, definition in SUPPORTING_INDEXES: + op.execute( + text( + f"CREATE INDEX IF NOT EXISTS {_safe_relation_name(index_name)} ON {definition}" + ) + ) + + +def downgrade() -> None: + safe_stats = _safe_relation_name(STATS_VIEW) + safe_feature = _safe_relation_name(FEATURE_VIEW) + + op.execute(text(f"DROP VIEW IF EXISTS {safe_feature}")) + op.execute(text(f"DROP MATERIALIZED VIEW IF EXISTS {safe_stats}")) + + for index_name, _definition in SUPPORTING_INDEXES: + op.execute(text(f"DROP INDEX IF EXISTS {_safe_relation_name(index_name)}")) diff --git a/core/ogc-field-descriptions.yml b/core/ogc-field-descriptions.yml index c462b6dd9..244525f83 100644 --- a/core/ogc-field-descriptions.yml +++ b/core/ogc-field-descriptions.yml @@ -2099,6 +2099,397 @@ minor_chemistry_wells: recorded them. +# water well field operations. Internal mount only -- the layer carries landowner +# contact details and staff-written access notes, so it has no public twin. +# See docs/water-well-field-operations-layer.md. +water_well_field_operations: + quad_name: + title: USGS quadrangle + description: >- + Name of the USGS 7.5-minute topographic quadrangle the well falls in. + elevation_method: + title: Elevation method + description: >- + How the ground-surface elevation was determined, such as GPS survey or + read from a digital elevation model. Governs how much precision the + elevation deserves. + enum-lexicon: collection_method + latitude: + title: Latitude + description: >- + Latitude of the well in decimal degrees on the WGS 84 datum, the same + coordinate the feature geometry carries. Published as a plain number so + it survives a spreadsheet or CSV export of this layer, which drops the + geometry, and so a crew can read it into a handheld GPS. + x-ogc-unit: https://qudt.org/vocab/unit/DEG + x-ogc-unitLang: QUDT + longitude: + title: Longitude + description: >- + Longitude of the well in decimal degrees on the WGS 84 datum, negative + west of Greenwich, the same coordinate the feature geometry carries. + Published as a plain number for the same reason as the latitude. + x-ogc-unit: https://qudt.org/vocab/unit/DEG + x-ogc-unitLang: QUDT + alternate_ids: + title: Alternate identifiers + description: >- + Identifiers other organisations use for this well, each written as + organisation followed by the identifier and separated by commas. Empty + where no cross-reference has been recorded. + well_purposes: + title: Well purposes + description: >- + What the well is used for, from the controlled well-purpose vocabulary -- + domestic, irrigation, monitoring, and so on. A well may serve more than + one purpose, in which case the purposes are separated by commas. + well_casing_materials: + title: Casing materials + description: >- + Materials the well casing is made of, from the controlled casing-material + vocabulary. Separated by commas where a well has more than one. + screen_count: + title: Screened intervals + description: >- + Number of discrete screened intervals recorded for the well. Zero means + no screen record exists, which is not the same as an unscreened well. + screen_depth_top: + title: Shallowest screen top + description: >- + Depth to the top of the shallowest screened interval, below ground + surface. + x-ogc-unit: https://qudt.org/vocab/unit/FT + x-ogc-unitLang: QUDT + screen_depth_bottom: + title: Deepest screen bottom + description: >- + Depth to the bottom of the deepest screened interval, below ground + surface. With the shallowest screen top this gives the outer extent of + the screened section, not the length of any one screen. + x-ogc-unit: https://qudt.org/vocab/unit/FT + x-ogc-unitLang: QUDT + measuring_point_height: + title: Measuring point height + description: >- + Height of the current measuring point above the ground surface. Manual + depth-to-water readings are taken from this point, so it is what converts + a raw reading to a depth below ground surface. + x-ogc-unit: https://qudt.org/vocab/unit/FT + x-ogc-unitLang: QUDT + measuring_point_description: + title: Measuring point description + description: >- + Where on the wellhead the measuring point is, as written for the person + standing at the well -- for example the north side of the casing at the + top of the PVC. The single most useful string on this layer for a crew + about to take a reading. + measuring_point_start_date: + title: Measuring point in use since + description: >- + Date the current measuring point configuration took effect. A recent date + means the wellhead was altered and readings before it are referenced to a + different point. + well_status: + title: Well status + description: >- + Current condition of the well itself. Null where no status is on record. + enum: + - Active, pumping well + - Inactive, exists but not used + - Destroyed, exists but not usable + - Abandoned + well_status_since: + title: Well status since + description: Date the current well status took effect. + monitoring_status: + title: Monitoring status + description: >- + Whether the well is currently on a monitoring schedule. Null where no + status is on record. + enum: + - Currently monitored + - Not currently monitored + monitoring_status_since: + title: Monitoring status since + description: Date the current monitoring status took effect. + monitoring_status_reason: + title: Monitoring status reason + description: >- + Why the well carries its current monitoring status, in the words of the + staff member who recorded it. This is where a landowner's request to stop + visiting is written down. + access_status: + title: Access status + description: >- + Current recorded state of physical access to the well. No values are + scoped to this status type in the controlled vocabulary yet, so the + column is expected to be sparse; treat a null as "not recorded" and + confirm access from the access notes and the landowner contact. + access_status_since: + title: Access status since + description: Date the current access status took effect. + open_status: + title: Open status + description: >- + Whether the well is physically open to a sounder or tape. A closed well + cannot be measured by hand however good the access. Null where no status + is on record. + enum: + - Open + - Open (unequipped) + - Closed + open_status_since: + title: Open status since + description: Date the current open status took effect. + datalogger_suitability_status: + title: Datalogger suitability + description: >- + Whether the well is physically suitable for a logger installation. This + is a judgement about the well, separate from whether the landowner has + given permission to install one. Null where no status is on record. + enum: + - Datalogger can be installed + - Datalogger cannot be installed + datalogger_suitability_status_since: + title: Datalogger suitability since + description: Date the current datalogger suitability status took effect. + may_measure_water_level: + title: May measure water level + description: >- + Whether a current permission allows a manual water-level measurement at + this well. True means a permission on record allows it; false means a + permission on record refuses it; null means no permission has been + recorded either way, which is not the same as a refusal -- it means + nobody has asked the landowner yet. + may_sample_water_chemistry: + title: May sample water chemistry + description: >- + Whether a current permission allows a water-chemistry sample to be + collected at this well. True allows, false refuses, null means no + permission is on record -- not a refusal. + may_install_datalogger: + title: May install datalogger + description: >- + Whether a current permission allows a logger to be installed in this + well. True allows, false refuses, null means no permission is on record + -- not a refusal. Separate from datalogger suitability, which says + whether the well could take one. + permission_granted_by: + title: Permission granted by + description: >- + Name of the contact who granted the current water-level permission, so a + crew can say who agreed to the visit. Null where no water-level + permission is on record. + monitoring_frequency: + title: Monitoring frequency + description: >- + How often the well is scheduled to be visited under its current + monitoring plan. Null where no frequency is on record. + enum-lexicon: monitoring_frequency + monitoring_frequency_since: + title: Monitoring frequency since + description: Date the current monitoring frequency took effect. + group_names: + title: Groups + description: >- + Names of the monitoring plans, geographic areas and historical groupings + the well belongs to, separated by commas. A well often sits in several. + group_types: + title: Group types + description: >- + Type of each group named in the groups column, in the same order and + separated by commas -- a monitoring plan, a geographic area, or a + historical grouping. + manual_water_level_count: + title: Manual water level readings + description: >- + Number of hand-measured groundwater level readings on record for the + well. Continuous readings from an installed logger are counted separately. + manual_water_level_first_date: + title: First manual reading + description: >- + Date of the earliest hand-measured groundwater level on record, as a UTC + calendar date. + manual_water_level_last_date: + title: Last manual reading + description: >- + Date of the most recent hand-measured groundwater level on record, as a + UTC calendar date. + days_since_manual_water_level: + title: Days since last manual reading + description: >- + Days elapsed since the most recent hand-measured groundwater level. + Computed against today, not against the last refresh, so it is current on + every request. Null where the well has never been measured by hand. + last_depth_to_water_ft: + title: Latest depth to water + description: >- + Most recent hand-measured depth to water, below ground surface -- the + measured value minus the height of the measuring point above ground, with + a reading that has no recorded measuring-point height treated as taken at + ground level. The same convention as the water-level summary and water + column layers. + x-ogc-unit: https://qudt.org/vocab/unit/FT + x-ogc-unitLang: QUDT + chemistry_sample_count: + title: Chemistry samples + description: >- + Number of water-chemistry samples collected at the well. Counts samples, + not the individual analyte results measured from them. + chemistry_sample_last_date: + title: Last chemistry sample + description: >- + Date the most recent water-chemistry sample was collected, as a UTC + calendar date. + days_since_chemistry_sample: + title: Days since last chemistry sample + description: >- + Days elapsed since the most recent water-chemistry sample. Computed + against today. Null where the well has never been sampled. + field_event_count: + title: Field visits + description: >- + Number of recorded visits to the well. Broader than either measurement + count: a visit that produced no reading, because the gate was locked or + the well was dry, is still a visit. + field_event_last_date: + title: Last field visit + description: >- + Date of the most recent recorded visit, as a UTC calendar date. A gap + between this and the last reading dates means someone went and came back + with nothing. + has_datalogger: + title: Has a datalogger + description: >- + Whether a logger is currently installed in the well -- a deployment with + an installation date and no removal date, of a data logger, pressure + transducer, DiverLink or diver cable. Other equipment deployed at the + well does not count. + datalogger_deployment_count: + title: Current logger deployments + description: >- + Number of logger deployments currently open at the well. Normally zero or + one; a higher number means the single logger columns describe only the + most recently installed of several. + datalogger_sensor_type: + title: Logger type + description: Kind of logger currently installed. + enum: + - Data Logger + - Pressure Transducer + - DiverLink + - Diver Cable + datalogger_model: + title: Logger model + description: Model of the logger currently installed. + datalogger_serial_no: + title: Logger serial number + description: >- + Serial number of the logger currently installed, for matching the + instrument in hand against the record. + datalogger_sensor_status: + title: Logger status + description: >- + Service state of the installed logger. A logger still recorded as + installed but marked retired or lost is a record worth correcting in the + field. + enum-lexicon: sensor_status + datalogger_installed_date: + title: Logger installed + description: Date the current logger deployment began. + datalogger_recording_interval: + title: Logger recording interval + description: >- + How often the installed logger records a reading, in the units given by + the recording interval units column. + datalogger_recording_interval_units: + title: Logger recording interval units + description: Time units the logger recording interval is expressed in. + datalogger_hanging_point_description: + title: Logger hanging point + description: >- + Where the logger cable is secured at the wellhead, as written for the + person about to pull it. Null where the deployment did not record one. + continuous_reading_count: + title: Continuous readings + description: >- + Number of readings logged automatically by instruments deployed in this + well, across all of its deployments. Separate from the hand-measured + count. + continuous_first_datetime: + title: First continuous reading + description: >- + Timestamp of the earliest automatically logged reading held for the well. + continuous_last_datetime: + title: Last continuous reading + description: >- + Timestamp of the most recent automatically logged reading held for the + well. A well whose logger has stopped reporting shows an old value here + while still reading as having a logger installed. + days_since_continuous_reading: + title: Days since last continuous reading + description: >- + Days elapsed since the most recent automatically logged reading, computed + against today. Null where the well has no logged readings at all. Rising + values on an instrumented well mean the logger needs attention. + contact_count: + title: Contacts + description: >- + Number of contacts associated with the well -- landowners, operators, and + anyone else recorded as a point of contact for it. + primary_contact_name: + title: Primary contact + description: >- + Name of the contact to approach about this well. This is the contact + recorded as primary where one exists; where none does, it is another + contact on the well, and the primary contact type column says which case + applies. + primary_contact_organization: + title: Primary contact organisation + description: Organisation the primary contact belongs to, where recorded. + primary_contact_role: + title: Primary contact role + description: >- + Role the primary contact holds in relation to the well, such as owner, + manager, or operator. + enum-lexicon: role + primary_contact_type: + title: Primary contact type + description: >- + Whether the contact shown is the one recorded as primary for the well or + a fallback used because no primary contact is on record. Read it before + relying on the contact details beside it. + enum-lexicon: contact_type + primary_contact_phone: + title: Primary contact phone + description: >- + Phone number for the primary contact. Where several are on record this is + the first one recorded, so a crew that cannot reach it should check the + full contact record. + primary_contact_email: + title: Primary contact email + description: >- + Email address for the primary contact. Where several are on record this + is the first one recorded. + contact_names: + title: All contacts + description: >- + Names of every contact associated with the well, separated by commas, so + a crew can see who else can be approached if the primary contact cannot + be reached. + access_notes: + title: Access notes + description: >- + Staff-written instructions for getting to the well and getting into it -- + gates, keys, locks, livestock, who to call first. Where several notes + exist they are separated by vertical bars, most recently recorded first. + directions_notes: + title: Directions + description: >- + Staff-written directions for finding the well on the ground, which + coordinates alone do not give. Where several notes exist they are + separated by vertical bars, most recently recorded first. + # EDR collections. Keys here are parameter names read out of the data, not # column names -- ogc_waterlevels stamps a single literal, while # ogc_water_chemistry carries the analyte text exactly as the laboratory diff --git a/core/pygeoapi-config-internal.yml b/core/pygeoapi-config-internal.yml index ca318979d..428fb81f5 100644 --- a/core/pygeoapi-config-internal.yml +++ b/core/pygeoapi-config-internal.yml @@ -77,6 +77,50 @@ resources: table: ogc_internal_locations geom_field: point + water_well_field_operations: + type: collection + title: Water Well Field Operations + description: >- + One record per water well, assembled for the crews who visit them. Where + every other well layer answers a scientific question, this one answers an + operational one: whether a crew may go to a well, what they are permitted + to do when they get there, and whether it is overdue. It pairs the well's + construction and measuring-point detail with its current well, + monitoring, access, open and datalogger-suitability status, the standing + landowner permissions for water-level measurement, chemistry sampling and + logger installation, the logger installed in it if any, how long it has + been since it was last measured and last sampled, and the landowner + contact and access notes needed to reach it. Published on the internal + mount only, because it carries personal contact details for landowners + and operators alongside staff-written access instructions. + keywords: + [ + water-wells, + field-operations, + landowner-contacts, + access-permissions, + monitoring-status, + data-logger, + internal, + ] + extents: + spatial: + bbox: [-109.05, 31.33, -103.00, 37.00] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + providers: + - type: feature + name: core.feature_provider.DescribedPostgreSQLProvider + data: + host: {postgres_host} + port: {postgres_port} + dbname: {postgres_db} + user: {postgres_user} + password: {postgres_password_env} + search_path: [public] + id_field: id + table: ogc_internal_water_well_field_operations + geom_field: point + latest_depth_to_water_wells: type: collection title: Latest Depth to Water (Water Wells) diff --git a/docs/internal-ogc-desktop-gis.md b/docs/internal-ogc-desktop-gis.md index 7efa66403..045bebd36 100644 --- a/docs/internal-ogc-desktop-gis.md +++ b/docs/internal-ogc-desktop-gis.md @@ -5,6 +5,16 @@ collections. It is gated by `core/internal_ogc_auth.py`, an ASGI middleware that runs in front of the raw Starlette Mount — FastAPI's `Depends()` machinery never sees these requests, so none of the `*_dependency` role parameters apply here. +> **An internal credential dispenses personal information.** The +> `water_well_field_operations` collection publishes landowner and operator +> contact details — name, organisation, role, phone number, email address — +> alongside staff-written access notes that routinely contain gate codes and +> names. Every credential accepted by this mount reaches it: an Authentik JWT +> carrying `OGCInternal`, and the static keys in `INTERNAL_OGC_API_KEYS`, which +> are shared secrets revocable only by redeploy. Any credential path added +> later inherits the same reach. Issue keys with that in mind, and read +> `docs/water-well-field-operations-layer.md` before widening who holds one. + ## Why there are static API keys at all The mount originally accepted only `Authorization: Bearer `. diff --git a/docs/ogc_conventions.md b/docs/ogc_conventions.md index 7a0d45691..d929279b9 100644 --- a/docs/ogc_conventions.md +++ b/docs/ogc_conventions.md @@ -156,6 +156,33 @@ human-readable display name). | `locations` | N/A | Locations | N/A | N/A | Conforms. Hidden from public catalog — scope decision, not a naming fix | | `avg_tds_wells` | `water_well_average_total_dissolved_solids` | Average Total Dissolved Solids (Water Wells) | N/A | 2 | Title implemented. `avg` abbreviation remains in the ID; hidden from public catalog but the rule still applies for internal consumers | | `latest_depth_to_water_wells` | N/A | Latest Depth to Water (Water Wells) | N/A | N/A | Conforms. Hidden from public catalog — audit flagged it as redundant with `water_well_summary`, not a naming defect | +| `water_well_field_operations` | N/A | Water Well Field Operations | N/A | N/A | Conforms. Internal-only with no public form at all — it publishes landowner contact details and staff-written access notes | + +## Current-record semantics + +Layers that read a history table (`status_history`, `permission_history`, +`measuring_point_history`, `monitoring_frequency_history`) disagree about what +"current" means, and the disagreement is deliberate. + +`ogc_actively_monitored_wells` takes the row with the greatest `start_date` and +**ignores `end_date`**, so a monitoring status closed in 2019 still reads as +current there. That is tolerable on a summary layer, where the question is +roughly "is this well in the programme". + +`ogc_internal_water_well_field_operations` honours the window: + +```sql +WHERE h.start_date <= CURRENT_DATE + AND (h.end_date IS NULL OR h.end_date >= CURRENT_DATE) +ORDER BY h.start_date DESC, h.id DESC +``` + +Its columns answer "may a crew do this today", and a permission that ran out +last month must not read as a permission. A row whose window has not opened yet +is not current either, and reads null. + +New layers should follow the second form. The first is kept only because +changing it would move rows in a published layer. ## References diff --git a/docs/water-well-field-operations-columns.md b/docs/water-well-field-operations-columns.md new file mode 100644 index 000000000..423ad6eb7 --- /dev/null +++ b/docs/water-well-field-operations-columns.md @@ -0,0 +1,106 @@ +# water well field operations — column sources + +Every column published by `ogc_internal_water_well_field_operations`, in view +order, and where its value comes from. Columns marked *(stats)* are read from +the `ogc_internal_water_well_field_operations_stats` materialized view and are as +fresh as its last refresh; everything else is joined live on each request. + +"Current record" means the history row satisfying +`start_date <= CURRENT_DATE AND (end_date IS NULL OR end_date >= CURRENT_DATE)`, +latest `start_date` first. "Most recent association" means the +`location_thing_association` row with no `effective_end`, latest +`effective_start` first. + +Generated from the view definition in +`alembic/versions/e1f2a3b4c5d6_add_water_well_field_operations_layer.py`. Field +prose lives in `core/ogc-field-descriptions.yml`; the design rationale lives in +`docs/water-well-field-operations-layer.md`. + +| Column | Source | +| --- | --- | +| `id` | `thing.id` | +| `name` | `thing.name` | +| `thing_type` | Literal `'water well'` — the view's row filter | +| `release_status` | `thing.release_status` | +| `nma_pk_welldata` | `thing.nma_pk_welldata` | +| `alternate_ids` | `thing_id_link.alternate_organization` + `.alternate_id`, comma-joined | +| `county` | `location.county`, most recent association | +| `state` | `location.state`, most recent association | +| `quad_name` | `location.quad_name`, most recent association | +| `latitude` | `ST_Y(location.point)` — decimal degrees, WGS 84 | +| `longitude` | `ST_X(location.point)` — decimal degrees, WGS 84 | +| `elevation` | `location.elevation`, most recent association | +| `elevation_method` | `data_provenance.collection_method` where `target_table = 'location'` and `field_name = 'elevation'`, latest `id` | +| `well_depth` | `thing.well_depth` | +| `hole_depth` | `thing.hole_depth` | +| `well_casing_diameter` | `thing.well_casing_diameter` | +| `well_casing_depth` | `thing.well_casing_depth` | +| `well_completion_date` | `thing.well_completion_date` | +| `well_driller_name` | `thing.well_driller_name` | +| `well_construction_method` | `thing.well_construction_method` | +| `well_pump_type` | `thing.well_pump_type` | +| `well_pump_depth` | `thing.well_pump_depth` | +| `formation_completion_code` | `thing.formation_completion_code` | +| `nma_formation_zone` | `thing.nma_formation_zone` | +| `well_purposes` | `well_purpose.purpose`, comma-joined | +| `well_casing_materials` | `well_casing_material.material`, comma-joined | +| `screen_count` | `count(well_screen)` | +| `screen_depth_top` | `min(well_screen.screen_depth_top)` | +| `screen_depth_bottom` | `max(well_screen.screen_depth_bottom)` | +| `measuring_point_height` | `measuring_point_history.measuring_point_height`, current record | +| `measuring_point_description` | `measuring_point_history.measuring_point_description`, current record | +| `measuring_point_start_date` | `measuring_point_history.start_date`, current record | +| `well_status` | `status_history.status_value` where `status_type = 'Well Status'`, current record | +| `well_status_since` | `status_history.start_date`, same record | +| `monitoring_status` | `status_history.status_value` where `status_type = 'Monitoring Status'`, current record | +| `monitoring_status_since` | `status_history.start_date`, same record | +| `monitoring_status_reason` | `status_history.reason`, same record | +| `access_status` | `status_history.status_value` where `status_type = 'Access Status'`, current record | +| `access_status_since` | `status_history.start_date`, same record | +| `open_status` | `status_history.status_value` where `status_type = 'Open Status'`, current record | +| `open_status_since` | `status_history.start_date`, same record | +| `datalogger_suitability_status` | `status_history.status_value` where `status_type = 'Datalogger Suitability Status'`, current record | +| `datalogger_suitability_status_since` | `status_history.start_date`, same record | +| `may_measure_water_level` | `permission_history.permission_allowed` where `permission_type = 'Water Level Sample'`, current record | +| `may_sample_water_chemistry` | `permission_history.permission_allowed` where `permission_type = 'Water Chemistry Sample'`, current record | +| `may_install_datalogger` | `permission_history.permission_allowed` where `permission_type = 'Datalogger Installation'`, current record | +| `permission_granted_by` | `contact.name` via `permission_history.contact_id` on the current water-level grant | +| `monitoring_frequency` | `monitoring_frequency_history.monitoring_frequency`, current record | +| `monitoring_frequency_since` | `monitoring_frequency_history.start_date`, current record | +| `group_names` | `group.name` via `group_thing_association`, comma-joined | +| `group_types` | `group.group_type`, same order as `group_names` | +| `manual_water_level_count` | `count(observation)` via `sample` → `field_activity` → `field_event`, `activity_type = 'groundwater level'` *(stats)* | +| `manual_water_level_first_date` | `min(observation.observation_datetime)`, UTC date, same chain *(stats)* | +| `manual_water_level_last_date` | `max(observation.observation_datetime)`, UTC date, same chain *(stats)* | +| `days_since_manual_water_level` | `CURRENT_DATE - manual_water_level_last_date` | +| `last_depth_to_water_ft` | `observation.value - COALESCE(observation.measuring_point_height, 0)` on the latest reading *(stats)* | +| `chemistry_sample_count` | `count(DISTINCT sample.id)`, `activity_type = 'water chemistry'` *(stats)* | +| `chemistry_sample_last_date` | `max(sample.sample_date)`, UTC date, same filter *(stats)* | +| `days_since_chemistry_sample` | `CURRENT_DATE - chemistry_sample_last_date` | +| `field_event_count` | `count(field_event)` for the well *(stats)* | +| `field_event_last_date` | `max(field_event.event_date)`, UTC date *(stats)* | +| `has_datalogger` | `true` when an open logger deployment exists — see `datalogger_sensor_type` | +| `datalogger_deployment_count` | `count(deployment)` open logger deployments | +| `datalogger_sensor_type` | `sensor.sensor_type` on the current deployment, restricted to Data Logger / Pressure Transducer / DiverLink / Diver Cable | +| `datalogger_model` | `sensor.model`, same deployment | +| `datalogger_serial_no` | `sensor.serial_no`, same deployment | +| `datalogger_sensor_status` | `sensor.sensor_status`, same deployment | +| `datalogger_installed_date` | `deployment.installation_date`, same deployment | +| `datalogger_recording_interval` | `deployment.recording_interval`, same deployment | +| `datalogger_recording_interval_units` | `deployment.recording_interval_units`, same deployment | +| `datalogger_hanging_point_description` | `deployment.hanging_point_description`, same deployment | +| `continuous_reading_count` | `count(transducer_observation)` via `deployment` *(stats)* | +| `continuous_first_datetime` | `min(transducer_observation.observation_datetime)` *(stats)* | +| `continuous_last_datetime` | `max(transducer_observation.observation_datetime)` *(stats)* | +| `days_since_continuous_reading` | `CURRENT_DATE - continuous_last_datetime::date` | +| `contact_count` | `count(DISTINCT contact)` via `thing_contact_association` | +| `primary_contact_name` | `contact.name`, `contact_type = 'Primary'` preferred, else lowest `contact.id` | +| `primary_contact_organization` | `contact.organization`, same contact | +| `primary_contact_role` | `contact.role`, same contact | +| `primary_contact_type` | `contact.contact_type`, same contact — says whether the row above is a real primary or a fallback | +| `primary_contact_phone` | `phone.phone_number`, lowest `phone.id` for that contact | +| `primary_contact_email` | `email.email`, lowest `email.id` for that contact | +| `contact_names` | `contact.name` for every associated contact, comma-joined | +| `access_notes` | `notes.content` where `note_type = 'Access'`, newest first, joined with ` | ` | +| `directions_notes` | `notes.content` where `note_type = 'Directions'`, newest first, joined with ` | ` | +| `point` | `location.point`, most recent association (PostGIS Point, EPSG:4326) | diff --git a/docs/water-well-field-operations-layer.md b/docs/water-well-field-operations-layer.md new file mode 100644 index 000000000..9e2c61308 --- /dev/null +++ b/docs/water-well-field-operations-layer.md @@ -0,0 +1,519 @@ +# Water well field operations layer — design + +Status: **implemented** on `feat/amp-field-operations-ogc-layer`. Migration +`e1f2a3b4c5d6`; column-by-column source table in +`docs/water-well-field-operations-columns.md`. + +Two questions this document opened were settled before implementation: "AMP" is +a label for the crews' view of the register, not a subset of it, so the layer +covers every water well; and the contact detail in §4 is published in full +(option A). + +## 0. Why the name does not say AMP + +The layer was requested as `amp_field_well_operations` and renamed before it +merged, per the pre-merge check in `docs/ogc_conventions.md`. Three rules +pushed the same way: + +- `amp` is an unexpanded abbreviation, which the do/don't rules forbid + outright ("Spell abbreviations out in full ... If the accurate name is long, + that's fine"). Its expansion is recorded nowhere in this codebase, so a + consumer could not resolve it even in principle. +- AMP is a label for who uses the layer, not a filter on what is in it — the + row set is every water well, with no group predicate. A name carrying `amp` + would claim a scope the data does not have, which is its own Don't. +- Group B analytic layers prefer a `water_well_` prefix over a `_wells` + suffix, and this is one. + +The AMP crews remain the layer's audience; that belongs in the description, +which says so, rather than in an identifier that is a long-term commitment. + +A new OGC API - Features collection, `water_well_field_operations`, published on +`/ogcapi-internal` only. One feature per water well, carrying everything a +field crew needs to plan and execute a visit: where the well is, who owns it +and how to reach them, what the crew is allowed to do there, what is installed +in it, and when it was last measured. + +Every other well layer in the catalogue answers a scientific question — what is +the water level, what is the chemistry, what is the trend. This one answers an +operational one: **can we go to this well, what are we allowed to do when we +get there, and is it overdue?** + +## 1. Why a new layer rather than a column pass on an existing one + +`water_wells` is the construction register and is public. `water_well_summary` +and `actively_monitored_wells` summarise the measurement record and are public. +The operational fields — landowner phone numbers, access notes, sampling +permissions — cannot go on any of those, because those layers are published +anonymously. The split is a publication boundary, not a modelling preference. + +## 2. Scope + +**In:** water wells. `thing_type = 'water well'`, the same predicate every +other well layer in this catalogue uses. Springs, streams, and met stations are +out even though crews visit them — this is the wells layer, per the request. + +`thing_type` in the lexicon also carries `observation well`, `piezometer`, +`monitoring well`, `production well`, `injection well`, `exploration well`, +`test well`, `abandoned well`, `dry hole`, `artesian well`, and `dug well`. +None of them are included. That matches `ogc_water_wells`, +`ogc_water_well_summary`, and `ogc_well_water_column`, so the layers cannot +disagree about which rows are "wells" — see §11, open question 1. + +**Row set:** every water well with a current location, including wells with no +measurement record at all. A well nobody has ever measured is exactly the well +a field crew needs to find, so this layer must not inherit +`water_well_summary`'s `total_water_levels > 0` restriction. + +**Release status:** unfiltered. The internal mount is unfiltered by design and +this layer has no public twin, so there is no `release_status = 'public'` +predicate anywhere in it. `release_status` is published as a column so a +consumer can see what they are looking at. + +## 3. Placement: internal-only, no public counterpart + +Every other `ogc_internal_*` relation is the unfiltered twin of a public +`ogc_*` one. This layer deliberately breaks that: `ogc_internal_water_well_field_operations` +exists and `ogc_water_well_field_operations` does not, and must never be created. + +This does not disturb the existing parity guard. +`tests/test_migration_view_parity.py::test_internal_migration_mirrors_every_public_relation` +reads two named migration files (`f4a5b6c7d8e9` and `2d3c3a268652`) and asserts +that internal file mirrors that public file's 24 relations. A relation created +by a *new* migration is outside what that test reads, so the count assertion +stays at 24 and stays true. + +Precedent for an internal-only collection: `locations`, `avg_tds_wells`, +`latest_depth_to_water_wells` (static entries in the internal config template +only) and `other_things` (`internal_only: True` in `THING_COLLECTIONS`). + +Because this layer is not a thing-type register, it is wired the same way +`locations` is: a static block in `core/pygeoapi-config-internal.yml`, not an +entry in `THING_COLLECTIONS`. + +## 4. The PII decision — read this before implementing + +This layer publishes landowner and operator contact details: name, +organization, role, phone number, email address. That is what "contact info" +means for a field crew, and it is why the layer is internal-only. + +The consequence is not confined to this layer. `/ogcapi-internal` is reachable +two ways on this branch (`core/internal_ogc_auth.py`): an Authentik JWT +carrying `OGCInternal`, or a static API key from `INTERNAL_OGC_API_KEYS`. +**Shipping this layer means every one of those credentials becomes a credential +that dispenses landowner personal information**, and any credential path added +later -- the user-issued keys on `feat/api-key-management` among them -- +inherits the same reach. The static desktop-GIS keys in particular are shared +secrets that can only be revoked by redeploy. + +pygeoapi has no per-field authorization — a column is either in the view or it +is not — so this is all-or-nothing per layer. Three options: + +| Option | What crews get | Cost | +| --- | --- | --- | +| **A. Full contact detail** (recommended) | Name, organization, role, phone, email | Every internal credential now dispenses PII | +| B. Presence only | `contact_count`, `primary_contact_organization`, `primary_contact_role` — no name, phone, or email | Crews must look the owner up in the UI before every visit; layer half-fails its purpose | +| C. Second layer | `water_well_field_operations` without PII, `water_well_field_contacts` with it | Two layers, two configs, two sets of field docs, same credentials still reach both — the split buys nothing without per-layer auth | + +**Decision: A.** The request was explicit, the mount is authenticated, and B +makes the layer not worth building. Shipped with: + +- A notice at the top of `docs/internal-ogc-desktop-gis.md` stating that an + internal credential now conveys landowner PII, so key issuance is a decision + made with that in mind. **`docs/api-key-management.md` still needs the same + notice** — that file lives on `feat/api-key-management` and does not exist on + this branch, so it could not be edited here. +- The collection `description` says so too, so it is visible in the catalogue + itself. +- The layer does not filter contacts on `release_status`. `Contact` carries one, + but the internal mount is unfiltered by design and this layer has no public + twin to keep in step with. + +## 5. Relations: one matview, one view + +Two relations, not one, because the columns divide cleanly by cost *and* by how +badly staleness hurts: + +``` +ogc_internal_water_well_field_operations_stats MATERIALIZED VIEW, keyed thing_id + counts, first/last dates, aggregates over observation and + transducer_observation. Expensive. Refreshed nightly. + +ogc_internal_water_well_field_operations VIEW <- pygeoapi points here + live join of thing, location, status_history, permission_history, + deployment, sensor, contact, notes + LEFT JOIN the stats matview +``` + +Reasons for the split: + +1. **Staleness is dangerous on exactly the cheap columns.** "May we sample this + well" and "is access currently granted" must not be up to 24 hours old. A + revoked permission that still reads `true` until the next nightly refresh + sends a crew onto land they are no longer welcome on. `permission_history` + and `status_history` are small and indexed by `(target_id, target_table)`; + reading them live costs nothing. +2. **`CURRENT_DATE` freezes in a matview.** The current-record rule in §7 is + date-relative. Evaluated inside a materialized view it would be pinned to + the refresh time, and a permission that expired this morning would still + read as current tonight. It has to be in the plain view. +3. **The expensive columns don't need to be fresh.** A reading count and a + last-measured date are fine at 24 hours old. + +Plain views already back the eleven thing-type collections, so pygeoapi serving +a non-materialized view is established. Filtering and bbox pushdown resolve +against the base tables' indexes. + +`_stats` carries a unique index on `thing_id` so it can be refreshed +`CONCURRENTLY` (`oco refresh-matview --concurrently`). The nightly pg_cron job +discovers every matview in the `public` schema by name, so it is picked up with +no schedule change. Add it to `MATERIALIZED_VIEWS` in +`services/materialized_views.py` for the CLI path. + +## 6. Columns + +`id` is `thing.id` and is unique — pygeoapi's `id_field: id` requires exactly +one row per id for `/items/{id}`. Every one-to-many is therefore aggregated. + +Multi-valued columns are emitted as **`text`, comma-space joined**, not as +Postgres arrays. `ogc_actively_monitored_wells` uses `text[]` and that is fine +for a layer read in a browser, but this one exists to be pulled into ArcGIS Pro +and QGIS and exported to a File Geodatabase or GeoPackage for offline field +use, and neither format has a list type. A joined string survives the round +trip; an array does not. + +### Identity and location + +| Column | Source | +| --- | --- | +| `id` | `thing.id` | +| `name` | `thing.name` | +| `thing_type` | `'water well'::text` | +| `release_status` | `thing.release_status` | +| `nma_pk_welldata` | `thing.nma_pk_welldata` — legacy NM_Aquifer key, still what crews say out loud | +| `alternate_ids` | `thing_id_link`, joined as `organization:alternate_id` pairs | +| `county`, `quad_name`, `state` | `location` | +| `latitude`, `longitude` | `ST_Y`/`ST_X` of the same point, decimal degrees on WGS 84 -- a plain number survives a CSV export, which drops the geometry, and a crew can read it into a handheld GPS | +| `elevation` | `location.elevation` | +| `elevation_method` | `data_provenance.collection_method` for that location's elevation -- the same lookup `water_well_summary` uses, so the two layers cannot disagree about how an elevation was obtained | +| `point` | `location.point`, most recent association (the shared `LATEST_LOCATION_CTE`) | + +### Well construction + +`well_depth`, `hole_depth`, `well_casing_diameter`, `well_casing_depth`, +`well_completion_date`, `well_driller_name`, `well_construction_method`, +`well_pump_type`, `well_pump_depth`, `formation_completion_code`, +`nma_formation_zone` — all straight from `thing`, same columns and same names +as the thing-view template, so a crew reading both layers sees one vocabulary. + +Plus, aggregated: + +| Column | Source | +| --- | --- | +| `well_purposes` | `well_purpose.purpose`, joined | +| `well_casing_materials` | `well_casing_material.material`, joined | +| `screen_count` | `count(well_screen)` | +| `screen_depth_top`, `screen_depth_bottom` | `min`/`max` across that well's screens, ft below ground surface | + +### Measuring point (current record only) + +| Column | Source | +| --- | --- | +| `measuring_point_height` | `measuring_point_history.measuring_point_height`, ft above ground surface | +| `measuring_point_description` | e.g. "North side of casing, top of PVC" | +| `measuring_point_start_date` | when the current configuration took effect | + +The description is the single most useful string on the layer for a crew +standing at a wellhead. Current record per §7. + +### Status (current record per status type) + +`status_history.status_type` has five values in the lexicon, and all five are +published, each as `` plus `_since`: + +| Column | `status_type` | Values | +| --- | --- | --- | +| `well_status` | Well Status | Abandoned; Active, pumping well; Destroyed, exists but not usable; Inactive, exists but not used | +| `monitoring_status` | Monitoring Status | Currently monitored; Not currently monitored | +| `access_status` | Access Status | (lexicon does not yet scope values to type — open question 3) | +| `open_status` | Open Status | Open; Open (unequipped); Closed | +| `datalogger_suitability_status` | Datalogger Suitability Status | Datalogger can be installed; Datalogger cannot be installed | + +`monitoring_status_reason` is also published — `status_history.reason`, which +is where "landowner asked us to stop" is written down. + +### Permission (current grants) + +From `permission_history`, whose `permission_type` lexicon is exactly the three +things a crew does: + +| Column | `permission_type` | +| --- | --- | +| `may_measure_water_level` | Water Level Sample | +| `may_sample_water_chemistry` | Water Chemistry Sample | +| `may_install_datalogger` | Datalogger Installation | + +**These are three-valued and must stay that way.** `true` = a current grant +says allowed. `false` = a current grant says *not* allowed. `NULL` = **no +permission on record**, which is not the same as denied and must not be +rendered as "no". Collapsing NULL to false would tell a crew a well is off +limits when the truth is that nobody has asked yet; collapsing it to true is +worse. Every consumer-facing rendering of these columns has to carry the third +state. + +`permission_granted_by` — the granting contact's name from the current +water-level grant — is published alongside, so a crew can name who said yes. + +### Monitoring programme + +| Column | Source | +| --- | --- | +| `monitoring_frequency` | current `monitoring_frequency_history` record | +| `monitoring_frequency_since` | its `start_date` | +| `group_names` | `group` via `group_thing_association`, joined, de-duplicated | +| `group_types` | Monitoring Plan / Geographic Area / Historical, joined, index-aligned with `group_names` | + +### Manual measurement statistics + +Manual groundwater levels, reached the way every other water-level view in this +schema reaches them: `observation -> sample -> field_activity -> field_event`, +`field_activity.activity_type = 'groundwater level'`. + +| Column | Definition | +| --- | --- | +| `manual_water_level_count` | count of readings with a value and a timestamp | +| `manual_water_level_first_date` | earliest `observation_datetime`, UTC calendar date | +| `manual_water_level_last_date` | latest, UTC calendar date | +| `days_since_manual_water_level` | `CURRENT_DATE - manual_water_level_last_date`, computed in the plain view so it does not freeze at refresh | +| `last_depth_to_water_ft` | the latest reading, below ground surface | + +`last_depth_to_water_ft` uses `(o.value - COALESCE(o.measuring_point_height, 0))` +— the reading minus the height of the measuring point above ground, with a +missing height treated as taken at ground level. This is the exact convention +in `ogc_water_well_summary`, `ogc_latest_depth_to_water_wells`, and +`ogc_well_water_column`, and it must stay identical or the four layers will +disagree about what a depth to water is. See +`docs/measuring-point-height-null-handling.md`. + +### Chemistry sampling statistics + +Same chain, `field_activity.activity_type = 'water chemistry'`: +`chemistry_sample_count`, `chemistry_sample_last_date`, +`days_since_chemistry_sample`. Sample-level, not analyte-level — a crew wants +"when was this well last sampled", not "what was the sulfate". + +### Field visit statistics + +`field_event_count`, `field_event_last_date` from `field_event`. Broader than +either measurement chain: a visit that produced no reading is still a visit, +and the gap between `field_event_last_date` and +`manual_water_level_last_date` is itself a signal. + +### Data logger / continuous record + +"Has a logger" is a deployment question, not a sensor-inventory question. A +deployment is current when `installation_date IS NOT NULL` and `removal_date IS +NULL`, and the sensor is one of `Data Logger`, `Pressure Transducer`, +`DiverLink`, `Diver Cable` (`sensor.sensor_type`). + +| Column | Definition | +| --- | --- | +| `has_datalogger` | boolean, true when a current logger deployment exists | +| `datalogger_sensor_type`, `datalogger_model`, `datalogger_serial_no` | from `sensor` on the current deployment | +| `datalogger_sensor_status` | In Service / In Repair / Retired / Lost | +| `datalogger_installed_date` | `deployment.installation_date` | +| `datalogger_recording_interval`, `datalogger_recording_interval_units` | how often it logs | +| `datalogger_hanging_point_description` | where the cable hangs from — a field instruction | +| `continuous_reading_count` | rows in `transducer_observation` for this well's deployments | +| `continuous_first_datetime`, `continuous_last_datetime` | series extent | +| `days_since_continuous_reading` | live, in the plain view | + +Where a well has more than one current logger deployment, the singular +`datalogger_*` columns take the most recently installed one, and +`datalogger_deployment_count` says how many there are so nothing is silently +hidden. + +`continuous_reading_count` aggregates the largest table in the schema. It is +the reason `_stats` is materialized. If the refresh proves too slow, the +fallback is to source the three continuous columns from `transducer_daily_data` +instead — cheaper, one row per day, and adequate for "is the logger still +reporting". + +### Contacts + +Subject to §4. + +| Column | Definition | +| --- | --- | +| `contact_count` | contacts associated with the well | +| `primary_contact_name`, `_organization`, `_role` | prefer the `contact_type = 'Primary'` contact, else any contact on the well, lowest `contact.id` as tie-break | +| `primary_contact_type` | which of those two cases produced the row above | +| `primary_contact_phone` | lowest-id `phone.phone_number` for that contact | +| `primary_contact_email` | lowest-id `email.email` for that contact | +| `contact_names` | every associated contact, joined | + +The fallback is a change from this document's first draft, which took the +primary contact strictly. A well whose only contact is recorded as Secondary +would then show `contact_count = 1` beside a blank name and phone — the exact +failure mode that ruled out option B in §4. `primary_contact_type` keeps the +fallback visible rather than implied. + +Deliberately **not** published: `address`. A mailing address is not how a crew +reaches a landowner, and it is the most sensitive field on the record. + +### Notes + +`notes` is polymorphic on `(target_id, target_table = 'thing')` with a +`note_type` lexicon. Two types are operational and are published as their own +columns, most recent first, joined: + +| Column | `note_type` | +| --- | --- | +| `access_notes` | Access | +| `directions_notes` | Directions | + +The rest (Construction, Maintenance, Historical, General, Water, Water Quality, +Sampling Procedure, Coordinate, OwnerComment, Site Notes (legacy)) stay out. +They are a well's history, not its visit instructions, and free text is where +PII leaks — see open question 4. + +## 7. The "current record" rule + +Four histories feed this layer — `status_history`, `permission_history`, +`measuring_point_history`, `monitoring_frequency_history` — and all four have +the same `(start_date, end_date)` shape. One rule for all of them: + +```sql +WHERE h.start_date <= CURRENT_DATE + AND (h.end_date IS NULL OR h.end_date >= CURRENT_DATE) +ORDER BY h.start_date DESC, h.id DESC +LIMIT 1 -- via DISTINCT ON per thing +``` + +**This diverges from the existing catalogue on purpose.** +`ogc_actively_monitored_wells` takes the row with the greatest `start_date` and +ignores `end_date` entirely, so a monitoring status that was closed in 2019 +still reads as current there. That is tolerable on a summary layer. It is not +tolerable on a layer whose whole job is to tell a crew what is true today — +"you may install a logger here" must not survive the expiry of the permission +that said so. + +The divergence has to be written into `docs/ogc_conventions.md`, or the next +person will read the two views side by side and assume one is a mistake. + +Rows whose window has not opened yet (`start_date > CURRENT_DATE`) are not +current and read NULL. + +## 8. Field-level documentation + +`core/ogc_field_metadata.py` keys `core/ogc-field-descriptions.yml` by backing +relation with the `ogc_internal_` prefix stripped, so the entry is +`water_well_field_operations`. This is not optional: +`tests/test_ogc_field_descriptions.py::test_every_published_column_has_an_entry` +walks both mounts and fails on any published column without a `description`. + +Columns already in `_defaults` (`id`, `name`, `thing_type`, `release_status`, +`elevation`, `well_depth`, `hole_depth`, `well_casing_diameter`, …) are +inherited and must not be restated. Everything new to this layer needs an +entry, and the entry says what the value *means*, never how the view is built. + +Controlled vocabularies use `enum-lexicon` rather than a duplicated `enum` +list: `status_value` for the five status columns, `monitoring_frequency`, +`sensor_type`, `sensor_status`, `well_purpose`, `casing_material`, `role`, +`group_type`. Depth and height columns carry +`x-ogc-unit: https://qudt.org/vocab/unit/FT` with +`x-ogc-unitLang: QUDT`, matching the rest of the file. + +The three permission booleans need their NULL meaning spelled out in the +`description` — that is the only place a consumer will ever read it. + +## 9. Wiring + +1. **Migration** — new Alembic revision on the current head (`c9d0e1f2a3b4`), + creating the matview, its unique index, and the view, plus the supporting + indexes in §10. `downgrade()` drops all three; nothing pre-existing is + modified, so there is no prior state to restore. Follow the house style in + `2d3c3a268652`: self-contained, no cross-migration imports, `REQUIRED_TABLES` + guard up front. +2. **`core/pygeoapi-config-internal.yml`** — one static resource block, next to + `locations`. `name: core.feature_provider.DescribedPostgreSQLProvider`, + `id_field: id`, `table: ogc_internal_water_well_field_operations`, + `geom_field: point`. No `time_field`: no single column is the feature's + time, and picking one would make `datetime=` filtering quietly mean + something arbitrary. +3. **`core/pygeoapi-config.yml`** — no change. Ever. +4. **`core/ogc-field-descriptions.yml`** — the `water_well_field_operations` block. +5. **`services/materialized_views.py`** — append + `ogc_internal_water_well_field_operations_stats` to `MATERIALIZED_VIEWS`. +6. **`core/gis-curated-layers.yml`** — *no* entry. Every layer in that file + ships as a `.qlr` and a `.lyrx` artifact, and this one is PII-bearing and + internal; it should be reached by browsing the authenticated connection, not + handed out as a downloadable file. Revisit only if crews ask. +7. **Docs** — `docs/api-key-management.md` and `docs/internal-ogc-desktop-gis.md` + gain the PII notice from §4; `docs/ogc_conventions.md` gains the §7 + divergence. + +Description text has to clear +`tests/test_pygeoapi_mount.py::test_every_collection_description_explains_the_layer`: +at least 200 characters, ends in a full stop, no placeholder words, no +hyphenated word split across a YAML line break, and at least four unique +lowercase-hyphenated keywords. + +## 10. Indexes + +The per-thing lookups this layer adds are foreign-key joins, and Postgres does +not index foreign keys on its own. Check each before creating it — `b8c9d0e1f2a3` +already added four on the observation chain. + +- `status_history (target_table, target_id, status_type, start_date DESC)` +- `permission_history (target_table, target_id, permission_type, start_date DESC)` +- `measuring_point_history (thing_id, start_date DESC)` +- `monitoring_frequency_history (thing_id, start_date DESC)` +- `notes (target_id, target_table)` — already exists as `ix_notes_polymorphic_link` +- `deployment (thing_id)` +- `thing_contact_association (thing_id)`, `(contact_id)` +- `transducer_observation (deployment_id, observation_datetime)` — for the + continuous aggregates; the largest table in the schema and the one that will + decide whether the nightly refresh is acceptable + +## 11. Settled, and still open + +Settled before implementation: + +- **"AMP" is a label, not a subset.** The layer covers every water well; there + is no group predicate. +- **Contact detail is published in full** (§4, option A). + +Still open: + +1. **Which `thing_type` values count as a well?** Shipped as `'water well'` + only, for consistency with every existing well layer. If AMP crews also + visit piezometers and monitoring wells, the predicate widens to a list — and + then it should widen in the other four layers too, as its own ticket, rather + than this layer quietly disagreeing with them. +2. **Access Status has no values.** `status_type` carries `Access Status` but + the `status_value` lexicon has no access-related terms — the eleven values + map to Well, Monitoring, Open, and Datalogger Suitability status. The column + is published and will read null until the vocabulary gains terms scoped to + it; the field description says so. Either access statuses are recorded with + values from another category, or the column is empty in practice, and the + answer needs a look at production data. +3. **Free-text PII.** `access_notes` and `directions_notes` are staff-written + and will contain phone numbers, gate codes, and names. A further reason the + layer is internal-only, and the §4 decision covers them too. +4. **Refresh cadence.** Nightly, via the existing pg_cron job. If crews plan + the next day's route the evening before, nightly is fine; if they re-plan in + the field, the stats columns are the ones that go stale. +5. **`continuous_reading_count` cost at production scale.** It aggregates the + largest table in the schema. If the nightly refresh proves too slow, the + fallback is to source the three continuous columns from + `transducer_daily_data` instead. + +## 12. Out of scope + +- Any public counterpart. +- Write access. This is a read layer; edits happen in the Ocotillo UI. +- A curated `.qlr` / `.lyrx` artifact (§9.6). +- Springs and other non-well monitoring points. +- Per-field authorization. pygeoapi has no such hook; the layer boundary is the + authorization boundary. diff --git a/services/materialized_views.py b/services/materialized_views.py index 9b7e3740e..75c5239d4 100644 --- a/services/materialized_views.py +++ b/services/materialized_views.py @@ -19,5 +19,9 @@ # two pivot views above, at per-result grain. "ogc_water_chemistry", "ogc_internal_water_chemistry", + # Field-operations aggregates for the internal-only + # ogc_internal_water_well_field_operations layer (e1f2a3b4c5d6). The + # feature view itself is a plain view and needs no refresh. + "ogc_internal_water_well_field_operations_stats", "transducer_daily_data", ) diff --git a/tests/test_cli_commands.py b/tests/test_cli_commands.py index 97534a601..257e4c320 100644 --- a/tests/test_cli_commands.py +++ b/tests/test_cli_commands.py @@ -72,10 +72,11 @@ def __exit__(self, exc_type, exc, tb): "REFRESH MATERIALIZED VIEW ogc_minor_chemistry_wells", "REFRESH MATERIALIZED VIEW ogc_water_chemistry", "REFRESH MATERIALIZED VIEW ogc_internal_water_chemistry", + "REFRESH MATERIALIZED VIEW ogc_internal_water_well_field_operations_stats", "REFRESH MATERIALIZED VIEW transducer_daily_data", ] assert commit_called["value"] is True - assert "Refreshed 10 materialized view(s)." in result.output + assert "Refreshed 11 materialized view(s)." in result.output def test_refresh_materialized_views_custom_and_concurrently( @@ -704,10 +705,12 @@ def _write_csv(path: Path, *, well_name: str, notes: str): "Water level accurate to within two hundreths of a foot," f"{notes}" ) - csv_text = textwrap.dedent(f"""\ + csv_text = textwrap.dedent( + f"""\ {header} {row} - """) + """ + ) path.write_text(csv_text) unique_notes = f"pytest-{uuid.uuid4()}" diff --git a/tests/test_ogc.py b/tests/test_ogc.py index 18c5fea74..5d6d3c83f 100644 --- a/tests/test_ogc.py +++ b/tests/test_ogc.py @@ -818,12 +818,15 @@ def test_ogc_collections(ogc_client): # and latest_depth_to_water_wells repeats water_well_summary # (BDMS-977), and other_things is internal vocabulary (BDMS-979). The # backing relations are retained and still served on /ogcapi-internal. + # water_well_field_operations is internal-only for a stronger reason than + # the other four: it publishes landowner contact details. assert ids.isdisjoint( { "locations", "avg_tds_wells", "latest_depth_to_water_wells", "other_things", + "water_well_field_operations", } ) diff --git a/tests/test_ogc_water_well_field_operations.py b/tests/test_ogc_water_well_field_operations.py new file mode 100644 index 000000000..a2b65541b --- /dev/null +++ b/tests/test_ogc_water_well_field_operations.py @@ -0,0 +1,603 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""The internal-only water well field operations layer (e1f2a3b4c5d6). + +What is worth testing here is not that the columns exist but that the layer's +three load-bearing rules hold: a well with no measurements still appears, a +permission with no record reads NULL rather than false, and a history record +whose window has closed is not treated as current. + +See docs/water-well-field-operations-layer.md. +""" + +from datetime import date, datetime, timedelta +from importlib.util import find_spec + +import pytest +from fastapi.testclient import TestClient +from sqlalchemy import text + +from core.dependencies import ( + admin_function, + amp_admin_function, + amp_editor_function, + amp_viewer_function, + editor_function, + viewer_function, +) +from core.factory import create_api_app +from db import ( + DataProvenance, + Deployment, + Notes, + PermissionHistory, + Sensor, + StatusHistory, +) +from db.engine import session_ctx +from tests import override_authentication + +pytestmark = pytest.mark.skipif( + find_spec("pygeoapi") is None, + reason="pygeoapi is not installed in this environment", +) + +VIEW = "ogc_internal_water_well_field_operations" +STATS_VIEW = "ogc_internal_water_well_field_operations_stats" + + +@pytest.fixture(scope="module") +def today(): + """The database's idea of today, not Python's. + + The view computes its `days_since_*` columns against CURRENT_DATE, which is + evaluated in the database session's timezone. That is UTC here while the + developer running the tests may not be, so a date read from Python is off + by one for part of every day. Read it from the same place the view does. + """ + with session_ctx() as session: + return session.execute(text("SELECT CURRENT_DATE")).scalar() + + +@pytest.fixture(scope="module") +def yesterday(today): + return today - timedelta(days=1) + + +@pytest.fixture(scope="module") +def last_year(today): + return today - timedelta(days=365) + + +@pytest.fixture(scope="module") +def ogc_client(): + app = create_api_app() + for dependency in ( + admin_function, + editor_function, + amp_admin_function, + amp_editor_function, + ): + app.dependency_overrides[dependency] = override_authentication( + default={"name": "foobar", "sub": "1234567890"} + ) + for dependency in (viewer_function, amp_viewer_function): + app.dependency_overrides[dependency] = override_authentication() + + with TestClient(app) as client: + yield client + + app.dependency_overrides = {} + + +def _row(session, thing_id, columns): + return session.execute( + text(f"SELECT {columns} FROM {VIEW} WHERE id = :thing_id"), + {"thing_id": thing_id}, + ).one() + + +def _refresh_stats(session): + session.execute(text(f"REFRESH MATERIALIZED VIEW {STATS_VIEW}")) + session.commit() + + +# ------------------------------------------------------------------ row set + + +def test_a_well_with_no_measurements_still_appears(water_well_thing): + # water_well_summary drops wells with no readings, because a summary of + # nothing says nothing. This layer must not: a well nobody has measured is + # exactly the well a crew needs to find. + with session_ctx() as session: + _refresh_stats(session) + row = _row( + session, + water_well_thing.id, + "name, manual_water_level_count, chemistry_sample_count, " + "continuous_reading_count, days_since_manual_water_level", + ) + + assert row.name == "Test Well" + assert row.manual_water_level_count == 0 + assert row.chemistry_sample_count == 0 + assert row.continuous_reading_count == 0 + # No reading means no elapsed time to report, not zero days since one. + assert row.days_since_manual_water_level is None + + +def test_latitude_and_longitude_match_the_geometry(water_well_thing): + with session_ctx() as session: + row = session.execute( + text( + "SELECT latitude, longitude, ST_Y(point) AS geom_y, " + f"ST_X(point) AS geom_x FROM {VIEW} WHERE id = :thing_id" + ), + {"thing_id": water_well_thing.id}, + ).one() + + # Decimal degrees on WGS 84, read off the same point the geometry carries. + assert row.latitude == row.geom_y + assert row.longitude == row.geom_x + assert -180 <= row.longitude <= 180 + assert -90 <= row.latitude <= 90 + + +def test_elevation_method_reads_the_location_provenance(water_well_thing, location): + with session_ctx() as session: + row = _row(session, water_well_thing.id, "elevation_method") + # No provenance record is "not recorded", not a method of "unknown". + assert row.elevation_method is None + + provenance = DataProvenance( + target_id=location.id, + target_table="location", + field_name="elevation", + collection_method="Survey-grade GPS", + ) + session.add(provenance) + session.commit() + + row = _row(session, water_well_thing.id, "elevation_method") + assert row.elevation_method == "Survey-grade GPS" + + session.delete(provenance) + session.commit() + + +def test_elevation_method_ignores_provenance_for_other_fields( + water_well_thing, location +): + with session_ctx() as session: + provenance = DataProvenance( + target_id=location.id, + target_table="location", + field_name="point", + collection_method="Survey-grade GPS", + ) + session.add(provenance) + session.commit() + + row = _row(session, water_well_thing.id, "elevation_method") + # How the coordinates were obtained says nothing about the elevation. + assert row.elevation_method is None + + session.delete(provenance) + session.commit() + + +# --------------------------------------------------------------- permissions + + +def test_permission_with_no_record_is_null_not_false(water_well_thing): + with session_ctx() as session: + row = _row( + session, + water_well_thing.id, + "may_measure_water_level, may_sample_water_chemistry, " + "may_install_datalogger", + ) + + # NULL means nobody has asked the landowner. Rendering it as "no" would + # tell a crew the well is off limits when the truth is unknown. + assert row.may_measure_water_level is None + assert row.may_sample_water_chemistry is None + assert row.may_install_datalogger is None + + +def test_permission_distinguishes_granted_from_refused( + water_well_thing, contact, last_year +): + with session_ctx() as session: + granted = PermissionHistory( + contact_id=contact.id, + target_id=water_well_thing.id, + target_table="thing", + permission_type="Water Level Sample", + permission_allowed=True, + start_date=last_year, + ) + refused = PermissionHistory( + contact_id=contact.id, + target_id=water_well_thing.id, + target_table="thing", + permission_type="Datalogger Installation", + permission_allowed=False, + start_date=last_year, + ) + session.add_all([granted, refused]) + session.commit() + + row = _row( + session, + water_well_thing.id, + "may_measure_water_level, may_install_datalogger, " + "may_sample_water_chemistry, permission_granted_by", + ) + + assert row.may_measure_water_level is True + assert row.may_install_datalogger is False + # Untouched permission type stays unknown rather than inheriting either. + assert row.may_sample_water_chemistry is None + assert row.permission_granted_by == contact.name + + session.delete(granted) + session.delete(refused) + session.commit() + + +def test_expired_permission_is_not_current( + water_well_thing, contact, last_year, yesterday +): + with session_ctx() as session: + expired = PermissionHistory( + contact_id=contact.id, + target_id=water_well_thing.id, + target_table="thing", + permission_type="Water Level Sample", + permission_allowed=True, + start_date=last_year, + end_date=yesterday, + ) + session.add(expired) + session.commit() + + row = _row(session, water_well_thing.id, "may_measure_water_level") + # A permission that ran out yesterday is not a permission today. This + # is the divergence from ogc_actively_monitored_wells, which ignores + # end_date entirely. + assert row.may_measure_water_level is None + + session.delete(expired) + session.commit() + + +# -------------------------------------------------------------------- status + + +def test_status_reads_the_current_record_not_the_latest( + water_well_thing, last_year, yesterday +): + with session_ctx() as session: + closed = StatusHistory( + target_id=water_well_thing.id, + target_table="thing", + status_type="Monitoring Status", + status_value="Currently monitored", + start_date=last_year, + end_date=yesterday, + reason="programme ended", + ) + open_status = StatusHistory( + target_id=water_well_thing.id, + target_table="thing", + status_type="Well Status", + status_value="Active, pumping well", + start_date=last_year, + ) + session.add_all([closed, open_status]) + session.commit() + + row = _row( + session, + water_well_thing.id, + "monitoring_status, monitoring_status_since, well_status, " + "well_status_since", + ) + + assert row.monitoring_status is None + assert row.monitoring_status_since is None + assert row.well_status == "Active, pumping well" + assert row.well_status_since == last_year + + session.delete(closed) + session.delete(open_status) + session.commit() + + +def test_status_types_do_not_bleed_into_each_other(water_well_thing, last_year): + with session_ctx() as session: + monitoring = StatusHistory( + target_id=water_well_thing.id, + target_table="thing", + status_type="Monitoring Status", + status_value="Not currently monitored", + start_date=last_year, + reason="landowner asked us to stop", + ) + session.add(monitoring) + session.commit() + + row = _row( + session, + water_well_thing.id, + "monitoring_status, monitoring_status_reason, well_status, " + "open_status, access_status, datalogger_suitability_status", + ) + + assert row.monitoring_status == "Not currently monitored" + assert row.monitoring_status_reason == "landowner asked us to stop" + assert row.well_status is None + assert row.open_status is None + assert row.access_status is None + assert row.datalogger_suitability_status is None + + session.delete(monitoring) + session.commit() + + +# ---------------------------------------------------------------- datalogger + + +def test_has_datalogger_counts_only_logger_equipment( + water_well_thing, sensor_to_water_well_thing_deployment +): + with session_ctx() as session: + row = _row( + session, + water_well_thing.id, + "has_datalogger, datalogger_deployment_count, " + "datalogger_sensor_type, datalogger_serial_no, " + "datalogger_recording_interval", + ) + + assert row.has_datalogger is True + assert row.datalogger_deployment_count == 1 + assert row.datalogger_sensor_type == "Pressure Transducer" + assert row.datalogger_serial_no == "123456" + assert row.datalogger_recording_interval == 24 + + +def test_non_logger_equipment_does_not_make_a_well_instrumented( + water_well_thing, last_year +): + with session_ctx() as session: + barometer = Sensor( + name="Test Barometer", + sensor_type="Barometer", + sensor_status="In Service", + release_status="draft", + ) + session.add(barometer) + session.commit() + deployment = Deployment( + sensor_id=barometer.id, + thing_id=water_well_thing.id, + installation_date=last_year, + removal_date=None, + ) + session.add(deployment) + session.commit() + + row = _row( + session, + water_well_thing.id, + "has_datalogger, datalogger_deployment_count", + ) + # A barometer at the well is equipment, not a logger in the well. + assert row.has_datalogger is False + assert row.datalogger_deployment_count == 0 + + session.delete(deployment) + session.delete(barometer) + session.commit() + + +def test_a_removed_logger_leaves_the_well_uninstrumented( + water_well_thing, sensor, sensor_to_water_well_thing_deployment, yesterday +): + with session_ctx() as session: + deployment = session.get(Deployment, sensor_to_water_well_thing_deployment.id) + deployment.removal_date = yesterday + session.commit() + + row = _row(session, water_well_thing.id, "has_datalogger") + assert row.has_datalogger is False + + deployment.removal_date = None + session.commit() + + +# ------------------------------------------------------------- measurements + + +def test_last_depth_to_water_uses_the_measuring_point_convention( + water_well_thing, groundwater_level_sample, today +): + from db import Observation + from tests import get_parameter_id + + with session_ctx() as session: + readings = [] + for day, value, measuring_point_height in ((1, 6.0, 1.0), (2, 9.0, 2.0)): + observation = Observation( + observation_datetime=datetime(2025, 1, day, 12, 0, 0), + sample_id=groundwater_level_sample.id, + parameter_id=get_parameter_id("groundwater level", "Field Parameter"), + release_status="public", + value=value, + unit="ft", + measuring_point_height=measuring_point_height, + groundwater_level_reason="Water level not affected", + ) + session.add(observation) + readings.append(observation) + session.commit() + _refresh_stats(session) + + row = _row( + session, + water_well_thing.id, + "manual_water_level_count, manual_water_level_first_date, " + "manual_water_level_last_date, last_depth_to_water_ft, " + "days_since_manual_water_level", + ) + + assert row.manual_water_level_count == 2 + assert row.manual_water_level_first_date == date(2025, 1, 1) + assert row.manual_water_level_last_date == date(2025, 1, 2) + # Latest reading is 9 ft from a measuring point 2 ft above ground. + assert abs(float(row.last_depth_to_water_ft) - 7.0) < 1e-9 + # Computed against today, not against the last refresh. + assert row.days_since_manual_water_level == (today - date(2025, 1, 2)).days + + for observation in readings: + session.delete(observation) + session.commit() + _refresh_stats(session) + + +def test_private_readings_are_counted_on_this_internal_layer( + water_well_thing, groundwater_level_sample +): + from db import Observation + from tests import get_parameter_id + + with session_ctx() as session: + observation = Observation( + observation_datetime=datetime(2025, 2, 1, 12, 0, 0), + sample_id=groundwater_level_sample.id, + parameter_id=get_parameter_id("groundwater level", "Field Parameter"), + release_status="private", + value=4.0, + unit="ft", + measuring_point_height=1.0, + groundwater_level_reason="Water level not affected", + ) + session.add(observation) + session.commit() + _refresh_stats(session) + + row = _row(session, water_well_thing.id, "manual_water_level_count") + # The internal mount is unfiltered by design; this layer has no public + # twin to keep in step with. + assert row.manual_water_level_count == 1 + + session.delete(observation) + session.commit() + _refresh_stats(session) + + +# ------------------------------------------------------ multi-valued columns + + +def test_multi_valued_columns_are_comma_joined_text( + water_well_thing, domestic_well_purpose, irrigation_well_purpose +): + with session_ctx() as session: + row = _row(session, water_well_thing.id, "well_purposes") + + # Text, not an array: this layer is exported to File Geodatabase and + # GeoPackage for offline field use, and neither format has a list type. + assert isinstance(row.well_purposes, str) + assert row.well_purposes == "Domestic, Irrigation" + + +def test_contacts_and_access_notes_are_published(water_well_thing, contact, phone): + with session_ctx() as session: + note = Notes( + target_id=water_well_thing.id, + target_table="thing", + note_type="Access", + content="Gate is locked, call ahead", + ) + session.add(note) + session.commit() + + row = _row( + session, + water_well_thing.id, + "contact_count, primary_contact_name, primary_contact_type, " + "primary_contact_role, primary_contact_phone, access_notes", + ) + + assert row.contact_count == 1 + assert row.primary_contact_name == contact.name + assert row.primary_contact_type == "Primary" + assert row.primary_contact_role == "Owner" + assert row.primary_contact_phone == phone.phone_number + assert row.access_notes == "Gate is locked, call ahead" + + session.delete(note) + session.commit() + + +# ------------------------------------------------------------------- mounts + + +def test_layer_is_served_on_the_internal_mount_only(ogc_client): + internal = ogc_client.get( + "/ogcapi-internal/collections/water_well_field_operations/items?limit=1" + ) + assert internal.status_code == 200 + assert internal.json()["type"] == "FeatureCollection" + + public = ogc_client.get( + "/ogcapi/collections/water_well_field_operations/items?limit=1" + ) + # The layer publishes landowner contact details; it must not exist at all + # on the anonymous mount. + assert public.status_code == 404 + + +def test_every_column_is_documented_on_the_internal_mount(ogc_client): + response = ogc_client.get( + "/ogcapi-internal/collections/water_well_field_operations/schema" + ) + assert response.status_code == 200 + + properties = response.json()["properties"] + gaps = [ + name + for name, prop in properties.items() + if name != "geometry" and not prop.get("description") + ] + assert not gaps, f"columns with no YAML entry: {gaps}" + + +def test_permission_columns_explain_their_null_meaning(ogc_client): + response = ogc_client.get( + "/ogcapi-internal/collections/water_well_field_operations/schema" + ) + properties = response.json()["properties"] + + # The three-valued meaning only reaches a consumer through this prose. + for column in ( + "may_measure_water_level", + "may_sample_water_chemistry", + "may_install_datalogger", + ): + assert "null" in properties[column]["description"].lower() diff --git a/tests/test_pygeoapi_mount.py b/tests/test_pygeoapi_mount.py index e267d1803..2572874ed 100644 --- a/tests/test_pygeoapi_mount.py +++ b/tests/test_pygeoapi_mount.py @@ -115,12 +115,15 @@ def test_loading_a_mount_restores_config_env_vars(): # Layers hidden from the public catalog but still served to staff GIS # clients on /ogcapi-internal: locations duplicates the thing-type layers # (BDMS-978), avg_tds_wells and latest_depth_to_water_wells are misleading -# or redundant (BDMS-977), other_things is internal vocabulary (BDMS-979). +# or redundant (BDMS-977), other_things is internal vocabulary (BDMS-979), +# and water_well_field_operations carries landowner contact details and +# staff-written access notes, so it has no public form at all. INTERNAL_ONLY_COLLECTIONS = { "locations", "avg_tds_wells", "latest_depth_to_water_wells", "other_things", + "water_well_field_operations", } From 94a07d20afebf0412227edef5b553f7b68f2a4d3 Mon Sep 17 00:00:00 2001 From: jirhiker <2035568+jirhiker@users.noreply.github.com> Date: Tue, 1 Sep 2026 02:23:37 +0000 Subject: [PATCH 2/5] Formatting changes --- tests/test_cli_commands.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/test_cli_commands.py b/tests/test_cli_commands.py index 257e4c320..9923fb38c 100644 --- a/tests/test_cli_commands.py +++ b/tests/test_cli_commands.py @@ -705,12 +705,10 @@ def _write_csv(path: Path, *, well_name: str, notes: str): "Water level accurate to within two hundreths of a foot," f"{notes}" ) - csv_text = textwrap.dedent( - f"""\ + csv_text = textwrap.dedent(f"""\ {header} {row} - """ - ) + """) path.write_text(csv_text) unique_notes = f"pytest-{uuid.uuid4()}" From 809c899e042e140144596ca54d59983a01e52a78 Mon Sep 17 00:00:00 2001 From: Kelsey Smuczynski Date: Tue, 1 Sep 2026 11:12:06 -0600 Subject: [PATCH 3/5] feat(ogc): consolidate field-operations columns Two independently-drafted internal water-well layers existed after a side-by-side field comparison: feat/amp-field-operations-ogc-layer and a competing draft, kas-water-well-operations-ogc-layer- bdms-1202. Rather than ship both under colliding migration IDs, this branch (kas-water-well-field-operations-ogc-layer-bdms-1202) started from feat/amp-field-operations-ogc-layer. It already had tests, indexes, docs, and correct date-window current-record semantics. The agreed changes from the comparison follow: - Dropped nma_pk_welldata, county, state, quad_name, elevation_method, nma_formation_zone, and measuring_point_start_date. None of them serve this layer's operational purpose. - Dropped every `_since`/`_reason` column (one per status type, plus monitoring_frequency_since). The current value itself (well_status, monitoring_status, monitoring_frequency, etc.) stays; only the history around it was dropped. - Renamed thing_type, well_casing_materials, well_purposes, measuring_point_height/description, and field_event_last_date to match the naming already established on the public thing-type views (station_type, well_casing_material, well_purpose, mp_height/mp_description, date_last_visited). - Added formation_completion_description and aquifer_system_name, ported from the competing draft. - Restored well_screen to full per-interval detail (screen_depth_top, screen_depth_bottom, screen_description), ported from the competing draft, instead of the count-plus-min/max summary this branch shipped with. A driller doing rehab work needs the actual screened intervals, not an overall range. - Broadened "currently installed equipment" from logger-only to any currently installed sensor, so a camera or barometer is no longer invisible. has_datalogger/datalogger_deployment_count stay logger-scoped for whoever needs that narrower signal. - Published a column per remaining notes.note_type value (13 total), extending the existing access_notes/directions_notes pattern. - Dropped access_status, since status_value has no lexicon terms scoped to Access Status. The column could only ever read NULL, and access_notes already covers the same need. - Wrapped every aggregated equipment and well_screen expression except sensor_type in COALESCE(..., '') before string_agg, so a row with a null value in one field (e.g. a camera with no recording_interval, or a screen with no recorded description) leaves an empty segment instead of silently shortening that column's list and breaking alignment with the others. Config, field descriptions, design docs, and tests updated to match. --- ...6_add_water_well_field_operations_layer.py | 287 ++++++++++----- core/ogc-field-descriptions.yml | 287 +++++++++------ core/pygeoapi-config-internal.yml | 19 +- docs/water-well-field-operations-columns.md | 65 ++-- docs/water-well-field-operations-layer.md | 245 +++++++++---- tests/test_ogc_water_well_field_operations.py | 327 +++++++++++++++--- 6 files changed, 881 insertions(+), 349 deletions(-) diff --git a/alembic/versions/e1f2a3b4c5d6_add_water_well_field_operations_layer.py b/alembic/versions/e1f2a3b4c5d6_add_water_well_field_operations_layer.py index 12276cf41..cf4242f51 100644 --- a/alembic/versions/e1f2a3b4c5d6_add_water_well_field_operations_layer.py +++ b/alembic/versions/e1f2a3b4c5d6_add_water_well_field_operations_layer.py @@ -68,6 +68,51 @@ prior state. The supporting indexes are also created here: none of these foreign keys was indexed (Postgres does not index foreign keys on its own). +Consolidated from two independently-drafted layers (this one and +kas-water-well-operations-ogc-layer-bdms-1202) after a side-by-side review. +The deltas from the version that first merged (3499f414): + +- Dropped: nma_pk_welldata, county, state, quad_name, elevation_method (and + its data_provenance LATERAL lookup, now unused), nma_formation_zone, + measuring_point_start_date, and every `*_since`/`*_reason` status and + monitoring-frequency column. access_status itself is also dropped (not just + its `_since` companion): status_value has no terms scoped to Access Status + in the lexicon, so the column could only ever read NULL, and access_notes + already carries staff-written access information for a well. +- Renamed to match the naming already established on the public thing-type + views: thing_type -> station_type, well_casing_materials -> + well_casing_material, well_purposes -> well_purpose, + measuring_point_height/measuring_point_description -> mp_height/ + mp_description, field_event_last_date (from the stats matview) -> + date_last_visited. +- Added formation_completion_description (lexicon_term.definition for the + term named by formation_completion_code) and aquifer_system_name + (thing_aquifer_association -> aquifer_system, comma-joined), both scalar + lookups with no date-window semantics of their own -- neither source table + carries a start_date/end_date. +- Broadened "currently installed equipment" from logger-only to every + currently-installed sensor. The original datalogger_sensor_type/model/ + serial_no/sensor_status/installed_date/recording_interval/ + recording_interval_units/hanging_point_description picked a single row + (DISTINCT ON, most-recently-installed) from deployments filtered to + LOGGER_SENSOR_TYPES, so a currently-installed camera or barometer was + invisible. The unprefixed sensor_type/model/serial_no/sensor_status/ + installed_date/recording_interval/recording_interval_units/ + hanging_point_desc columns now aggregate every currently-installed sensor + regardless of type (semicolon-joined, ordered by sensor_type, same + convention as the comma-joined columns above but semicolon because these + are genuinely positional -- position N in one list is the same deployment as + position N in the others). has_datalogger/datalogger_deployment_count stay + logger-scoped exactly as before, for whoever specifically needs "is this + well instrumented," now computed from the broader deployment set filtered + inline rather than a separate pre-filtered CTE. +- Added one column per notes.note_type value (13 total, including the + pre-existing access_notes/directions_notes), `_notes`-suffixed, same + string_agg(' | ', id DESC) pattern as the original two. `OwnerComment` -> + owner_comment_notes and `Site Notes (legacy)` -> site_notes_legacy are the + two that do not mechanically snake-case; the legacy one is not + double-suffixed since the term already says "notes". + Revision ID: e1f2a3b4c5d6 Revises: c9d0e1f2a3b4 Create Date: 2026-08-31 00:00:00.000000 @@ -93,7 +138,6 @@ "sample", "field_activity", "field_event", - "data_provenance", "status_history", "permission_history", "measuring_point_history", @@ -112,6 +156,9 @@ "well_casing_material", "well_screen", "thing_id_link", + "lexicon_term", + "thing_aquifer_association", + "aquifer_system", } STATS_VIEW = "ogc_internal_water_well_field_operations_stats" @@ -137,6 +184,26 @@ "'Diver Cable'", ) +# notes.note_type -> published column name. Order matches core/lexicon.json's +# note_type category. Two do not mechanically snake-case: OwnerComment has no +# separator to split on, and "Site Notes (legacy)" already says "notes" so it +# is not double-suffixed. +NOTE_TYPES = ( + ("Access", "access_notes"), + ("Directions", "directions_notes"), + ("Communication", "communication_notes"), + ("Construction", "construction_notes"), + ("Maintenance", "maintenance_notes"), + ("Historical", "historical_notes"), + ("General", "general_notes"), + ("Water", "water_notes"), + ("Water Quality", "water_quality_notes"), + ("Sampling Procedure", "sampling_procedure_notes"), + ("Coordinate", "coordinate_notes"), + ("OwnerComment", "owner_comment_notes"), + ("Site Notes (legacy)", "site_notes_legacy"), +) + # Indexes the per-well lookups need. Each is created IF NOT EXISTS: none of # these existed when this migration was written, but the thing_type one in # particular is the sort of thing another migration may add first. @@ -169,6 +236,10 @@ ("ix_well_screen_thing_id", "well_screen (thing_id)"), ("ix_thing_id_link_thing_id", "thing_id_link (thing_id)"), ("ix_thing_thing_type", "thing (thing_type)"), + ( + "ix_thing_aquifer_association_thing_id", + "thing_aquifer_association (thing_id)", + ), # No index on transducer_observation (deployment_id, ...): the existing # uq_transducer_observation_deployment_parameter_datetime leads with # deployment_id and already serves the per-deployment aggregate. @@ -302,6 +373,17 @@ def _create_feature_view() -> str: safe_name = _safe_relation_name(FEATURE_VIEW) safe_stats = _safe_relation_name(STATS_VIEW) logger_types = ", ".join(LOGGER_SENSOR_TYPES) + notes_columns_sql = ",\n ".join( + f"""( + SELECT string_agg(n.content, ' | ' ORDER BY n.id DESC) + FROM notes AS n + WHERE + n.target_table = 'thing' + AND n.target_id = t.id + AND n.note_type = '{note_type}' + ) AS {_safe_relation_name(column_name)}""" + for note_type, column_name in NOTE_TYPES + ) return f""" CREATE VIEW {safe_name} AS WITH latest_location AS ( @@ -358,7 +440,11 @@ def _create_feature_view() -> str: AND (mf.end_date IS NULL OR mf.end_date >= CURRENT_DATE) ORDER BY mf.thing_id, mf.start_date DESC, mf.id DESC ), - logger_deployments AS ( + installed_deployments AS ( + -- Every currently-installed sensor, not just loggers -- a camera + -- or barometer must not be invisible just because it cannot log + -- on its own. See logger_count below for the narrower has_datalogger + -- signal. SELECT d.id AS deployment_id, d.thing_id, @@ -372,22 +458,88 @@ def _create_feature_view() -> str: se.sensor_status FROM deployment AS d JOIN sensor AS se ON se.id = d.sensor_id - WHERE - d.installation_date IS NOT NULL - AND d.removal_date IS NULL - AND se.sensor_type IN ({logger_types}) + WHERE d.installation_date IS NOT NULL AND d.removal_date IS NULL ), - current_logger AS ( - SELECT DISTINCT ON (ld.thing_id) ld.* - FROM logger_deployments AS ld - ORDER BY ld.thing_id, ld.installation_date DESC, ld.deployment_id DESC + installed_equipment AS ( + -- Aggregated, not DISTINCT ON: a well running more than one + -- current sensor lists all of them. All eight columns are ordered + -- by sensor_type, so position N in one list is the same + -- deployment as position N in the others -- which is why every + -- expression but sensor_type itself is wrapped in COALESCE(.., + -- ''): sensor_type is NOT NULL on sensor, but a deployment can + -- easily have a null model, recording_interval, etc. (a camera + -- has no recording interval), and plain string_agg silently + -- drops null inputs, shortening that one column's list and + -- breaking the position-N-means-the-same-deployment guarantee. + -- An empty segment between two ';' means "this sensor has no + -- value for this field," not "this sensor doesn't exist." + SELECT + idpl.thing_id, + string_agg( + idpl.sensor_type, '; ' ORDER BY idpl.sensor_type + ) AS sensor_type, + string_agg( + COALESCE(idpl.model, ''), '; ' ORDER BY idpl.sensor_type + ) AS model, + string_agg( + COALESCE(idpl.serial_no, ''), '; ' ORDER BY idpl.sensor_type + ) AS serial_no, + string_agg( + COALESCE(idpl.sensor_status, ''), '; ' ORDER BY idpl.sensor_type + ) AS sensor_status, + string_agg( + COALESCE(idpl.installation_date::text, ''), + '; ' ORDER BY idpl.sensor_type + ) AS installed_date, + string_agg( + COALESCE(idpl.recording_interval::text, ''), + '; ' ORDER BY idpl.sensor_type + ) AS recording_interval, + string_agg( + COALESCE(idpl.recording_interval_units, ''), + '; ' ORDER BY idpl.sensor_type + ) AS recording_interval_units, + string_agg( + COALESCE(idpl.hanging_point_description, ''), + '; ' ORDER BY idpl.sensor_type + ) AS hanging_point_desc + FROM installed_deployments AS idpl + GROUP BY idpl.thing_id ), logger_count AS ( SELECT - ld.thing_id, + idpl.thing_id, COUNT(*)::integer AS datalogger_deployment_count - FROM logger_deployments AS ld - GROUP BY ld.thing_id + FROM installed_deployments AS idpl + WHERE idpl.sensor_type IN ({logger_types}) + GROUP BY idpl.thing_id + ), + screens AS ( + -- Full per-interval detail, not a min/max summary: a well with + -- more than one screen keeps them all rather than collapsing to + -- an overall depth range. All three ordered by screen_depth_top + -- (nulls last), and each wrapped in COALESCE(..., '') for the + -- same reason as installed_equipment above -- screen_depth_top, + -- screen_depth_bottom, and screen_description are all nullable + -- independently of each other, and plain string_agg would drop + -- a null and misalign the other two columns' positions. + SELECT + ws.thing_id, + COUNT(*)::integer AS screen_count, + string_agg( + COALESCE(ws.screen_depth_top::text, ''), '; ' + ORDER BY ws.screen_depth_top NULLS LAST + ) AS screen_depth_top, + string_agg( + COALESCE(ws.screen_depth_bottom::text, ''), '; ' + ORDER BY ws.screen_depth_top NULLS LAST + ) AS screen_depth_bottom, + string_agg( + COALESCE(ws.screen_description, ''), '; ' + ORDER BY ws.screen_depth_top NULLS LAST + ) AS screen_description + FROM well_screen AS ws + GROUP BY ws.thing_id ), thing_contacts AS ( SELECT @@ -431,9 +583,8 @@ def _create_feature_view() -> str: SELECT t.id AS id, t.name, - 'water well'::text AS thing_type, + 'water well'::text AS station_type, t.release_status, - t.nma_pk_welldata, ( SELECT string_agg( DISTINCT @@ -449,16 +600,12 @@ def _create_feature_view() -> str: FROM thing_id_link AS tl WHERE tl.thing_id = t.id ) AS alternate_ids, - l.county, - l.state, - l.quad_name, -- Decimal degrees alongside the geometry. A crew types these into -- a handheld GPS or reads them over the radio; the geometry column -- is for the map, and a CSV export of this layer drops it. ST_Y(l.point) AS latitude, ST_X(l.point) AS longitude, l.elevation, - dpl.collection_method AS elevation_method, -- Construction, same column names as the thing-type views. t.well_depth, @@ -471,40 +618,43 @@ def _create_feature_view() -> str: t.well_pump_type, t.well_pump_depth, t.formation_completion_code, - t.nma_formation_zone, + ( + SELECT lt.definition + FROM lexicon_term AS lt + WHERE lt.term = t.formation_completion_code + ) AS formation_completion_description, ( SELECT string_agg(DISTINCT wp.purpose, ', ' ORDER BY wp.purpose) FROM well_purpose AS wp WHERE wp.thing_id = t.id - ) AS well_purposes, + ) AS well_purpose, ( SELECT string_agg( DISTINCT wcm.material, ', ' ORDER BY wcm.material ) FROM well_casing_material AS wcm WHERE wcm.thing_id = t.id - ) AS well_casing_materials, + ) AS well_casing_material, + ( + SELECT string_agg(DISTINCT asys.name, ', ' ORDER BY asys.name) + FROM thing_aquifer_association AS taa + JOIN aquifer_system AS asys ON asys.id = taa.aquifer_system_id + WHERE taa.thing_id = t.id + ) AS aquifer_system_name, scr.screen_count, scr.screen_depth_top, scr.screen_depth_bottom, + scr.screen_description, -- Measuring point, current record. - cmp.measuring_point_height, - cmp.measuring_point_description, - cmp.start_date AS measuring_point_start_date, + cmp.measuring_point_height AS mp_height, + cmp.measuring_point_description AS mp_description, -- Status, current record per status type. well_st.status_value AS well_status, - well_st.start_date AS well_status_since, mon_st.status_value AS monitoring_status, - mon_st.start_date AS monitoring_status_since, - mon_st.reason AS monitoring_status_reason, - acc_st.status_value AS access_status, - acc_st.start_date AS access_status_since, open_st.status_value AS open_status, - open_st.start_date AS open_status_since, dl_st.status_value AS datalogger_suitability_status, - dl_st.start_date AS datalogger_suitability_status_since, -- Permission, current grants. Three-valued: NULL means no -- permission is on record, which is not the same as denied. @@ -515,7 +665,6 @@ def _create_feature_view() -> str: -- Monitoring programme. cmf.monitoring_frequency, - cmf.start_date AS monitoring_frequency_since, grp.group_names, grp.group_types, @@ -537,20 +686,22 @@ def _create_feature_view() -> str: -- Field visits (from the stats matview). COALESCE(st.field_event_count, 0) AS field_event_count, - st.field_event_last_date, + st.field_event_last_date AS date_last_visited, - -- Data logger, current deployment. - (cl.deployment_id IS NOT NULL) AS has_datalogger, + -- Currently installed equipment, any sensor type -- see + -- installed_equipment above. has_datalogger/ + -- datalogger_deployment_count stay logger-scoped. + (lc.thing_id IS NOT NULL) AS has_datalogger, COALESCE(lc.datalogger_deployment_count, 0) AS datalogger_deployment_count, - cl.sensor_type AS datalogger_sensor_type, - cl.model AS datalogger_model, - cl.serial_no AS datalogger_serial_no, - cl.sensor_status AS datalogger_sensor_status, - cl.installation_date AS datalogger_installed_date, - cl.recording_interval AS datalogger_recording_interval, - cl.recording_interval_units AS datalogger_recording_interval_units, - cl.hanging_point_description AS datalogger_hanging_point_description, + ie.sensor_type, + ie.model, + ie.serial_no, + ie.sensor_status, + ie.installed_date, + ie.recording_interval, + ie.recording_interval_units, + ie.hanging_point_desc, COALESCE(st.continuous_reading_count, 0) AS continuous_reading_count, st.continuous_first_datetime, st.continuous_last_datetime, @@ -581,24 +732,10 @@ def _create_feature_view() -> str: ) AS primary_contact_email, cag.contact_names, - -- Visit instructions. Separator is ' | ' rather than ', ' because - -- the content is free text and routinely contains commas. - ( - SELECT string_agg(n.content, ' | ' ORDER BY n.id DESC) - FROM notes AS n - WHERE - n.target_table = 'thing' - AND n.target_id = t.id - AND n.note_type = 'Access' - ) AS access_notes, - ( - SELECT string_agg(n.content, ' | ' ORDER BY n.id DESC) - FROM notes AS n - WHERE - n.target_table = 'thing' - AND n.target_id = t.id - AND n.note_type = 'Directions' - ) AS directions_notes, + -- Notes, one column per note_type. Separator is ' | ' rather than + -- ', ' because the content is free text and routinely contains + -- commas. + {notes_columns_sql}, l.point FROM thing AS t @@ -610,8 +747,6 @@ def _create_feature_view() -> str: LEFT JOIN current_status AS mon_st ON mon_st.thing_id = t.id AND mon_st.status_type = 'Monitoring Status' - LEFT JOIN current_status AS acc_st - ON acc_st.thing_id = t.id AND acc_st.status_type = 'Access Status' LEFT JOIN current_status AS open_st ON open_st.thing_id = t.id AND open_st.status_type = 'Open Status' LEFT JOIN current_status AS dl_st @@ -629,31 +764,11 @@ def _create_feature_view() -> str: LEFT JOIN contact AS granter ON granter.id = wl_perm.contact_id LEFT JOIN current_measuring_point AS cmp ON cmp.thing_id = t.id LEFT JOIN current_monitoring_frequency AS cmf ON cmf.thing_id = t.id - LEFT JOIN current_logger AS cl ON cl.thing_id = t.id + LEFT JOIN installed_equipment AS ie ON ie.thing_id = t.id LEFT JOIN logger_count AS lc ON lc.thing_id = t.id LEFT JOIN contact_agg AS cag ON cag.thing_id = t.id LEFT JOIN primary_contact AS pc ON pc.thing_id = t.id - LEFT JOIN LATERAL ( - -- Character-for-character the elevation-provenance lookup in - -- ogc_water_well_summary (2d3c3a268652), so the two layers cannot - -- disagree about where a well's elevation came from. - SELECT dp.collection_method - FROM data_provenance AS dp - WHERE - dp.target_table = 'location' - AND dp.target_id = l.id - AND dp.field_name = 'elevation' - ORDER BY dp.id DESC - LIMIT 1 - ) AS dpl ON true - LEFT JOIN LATERAL ( - SELECT - COUNT(*)::integer AS screen_count, - MIN(ws.screen_depth_top) AS screen_depth_top, - MAX(ws.screen_depth_bottom) AS screen_depth_bottom - FROM well_screen AS ws - WHERE ws.thing_id = t.id - ) AS scr ON TRUE + LEFT JOIN screens AS scr ON scr.thing_id = t.id LEFT JOIN LATERAL ( -- group_thing_association has no unique constraint on -- (group_id, thing_id), so DISTINCT before aggregating. Both diff --git a/core/ogc-field-descriptions.yml b/core/ogc-field-descriptions.yml index 244525f83..110efe945 100644 --- a/core/ogc-field-descriptions.yml +++ b/core/ogc-field-descriptions.yml @@ -2103,17 +2103,14 @@ minor_chemistry_wells: # contact details and staff-written access notes, so it has no public twin. # See docs/water-well-field-operations-layer.md. water_well_field_operations: - quad_name: - title: USGS quadrangle - description: >- - Name of the USGS 7.5-minute topographic quadrangle the well falls in. - elevation_method: - title: Elevation method + station_type: + title: Feature type description: >- - How the ground-surface elevation was determined, such as GPS survey or - read from a digital elevation model. Governs how much precision the - elevation deserves. - enum-lexicon: collection_method + Controlled-vocabulary type of the monitoring point. Named station_type + rather than thing_type in this layer to match the naming already + established on the public thing-type views; every row on this layer is + a water well, so the value is always the same literal. + enum-lexicon: thing_type latitude: title: Latitude description: >- @@ -2137,38 +2134,56 @@ water_well_field_operations: Identifiers other organisations use for this well, each written as organisation followed by the identifier and separated by commas. Empty where no cross-reference has been recorded. - well_purposes: + well_purpose: title: Well purposes description: >- What the well is used for, from the controlled well-purpose vocabulary -- domestic, irrigation, monitoring, and so on. A well may serve more than one purpose, in which case the purposes are separated by commas. - well_casing_materials: + well_casing_material: title: Casing materials description: >- Materials the well casing is made of, from the controlled casing-material vocabulary. Separated by commas where a well has more than one. + formation_completion_description: + title: Completion formation, description + description: >- + Human-readable definition of the geologic formation named by + formation_completion_code, looked up from the lexicon. Null where the + code has no matching lexicon entry. + aquifer_system_name: + title: Aquifer system(s) + description: >- + Name(s) of the aquifer system(s) the well is associated with, separated + by commas where it is associated with more than one; order carries no + meaning beyond de-duplication. screen_count: title: Screened intervals description: >- Number of discrete screened intervals recorded for the well. Zero means no screen record exists, which is not the same as an unscreened well. screen_depth_top: - title: Shallowest screen top - description: >- - Depth to the top of the shallowest screened interval, below ground - surface. - x-ogc-unit: https://qudt.org/vocab/unit/FT - x-ogc-unitLang: QUDT + title: Screen top depth(s) + description: >- + Depth to the top of each screened interval, below ground surface, in + feet. A well with more than one screen lists each interval's top + depth, semicolon-separated, ordered shallowest first. screen_depth_top, + screen_depth_bottom, and screen_description are ordered the same way + and COALESCEd to an empty string where a value is missing, so position + N in one list is always the same interval as position N in the + others -- never a dropped position from a different one. screen_depth_bottom: - title: Deepest screen bottom + title: Screen bottom depth(s) description: >- - Depth to the bottom of the deepest screened interval, below ground - surface. With the shallowest screen top this gives the outer extent of - the screened section, not the length of any one screen. - x-ogc-unit: https://qudt.org/vocab/unit/FT - x-ogc-unitLang: QUDT - measuring_point_height: + Depth to the bottom of each screened interval, below ground surface, + in feet. See screen_depth_top for how multiple intervals are listed + and ordered. + screen_description: + title: Screen description(s) + description: >- + Free-text description of each screened interval. See screen_depth_top + for how multiple intervals are listed and ordered. + mp_height: title: Measuring point height description: >- Height of the current measuring point above the ground surface. Manual @@ -2176,19 +2191,13 @@ water_well_field_operations: a raw reading to a depth below ground surface. x-ogc-unit: https://qudt.org/vocab/unit/FT x-ogc-unitLang: QUDT - measuring_point_description: + mp_description: title: Measuring point description description: >- Where on the wellhead the measuring point is, as written for the person standing at the well -- for example the north side of the casing at the top of the PVC. The single most useful string on this layer for a crew about to take a reading. - measuring_point_start_date: - title: Measuring point in use since - description: >- - Date the current measuring point configuration took effect. A recent date - means the wellhead was altered and readings before it are referenced to a - different point. well_status: title: Well status description: >- @@ -2198,9 +2207,6 @@ water_well_field_operations: - Inactive, exists but not used - Destroyed, exists but not usable - Abandoned - well_status_since: - title: Well status since - description: Date the current well status took effect. monitoring_status: title: Monitoring status description: >- @@ -2209,25 +2215,6 @@ water_well_field_operations: enum: - Currently monitored - Not currently monitored - monitoring_status_since: - title: Monitoring status since - description: Date the current monitoring status took effect. - monitoring_status_reason: - title: Monitoring status reason - description: >- - Why the well carries its current monitoring status, in the words of the - staff member who recorded it. This is where a landowner's request to stop - visiting is written down. - access_status: - title: Access status - description: >- - Current recorded state of physical access to the well. No values are - scoped to this status type in the controlled vocabulary yet, so the - column is expected to be sparse; treat a null as "not recorded" and - confirm access from the access notes and the landowner contact. - access_status_since: - title: Access status since - description: Date the current access status took effect. open_status: title: Open status description: >- @@ -2238,9 +2225,6 @@ water_well_field_operations: - Open - Open (unequipped) - Closed - open_status_since: - title: Open status since - description: Date the current open status took effect. datalogger_suitability_status: title: Datalogger suitability description: >- @@ -2250,9 +2234,6 @@ water_well_field_operations: enum: - Datalogger can be installed - Datalogger cannot be installed - datalogger_suitability_status_since: - title: Datalogger suitability since - description: Date the current datalogger suitability status took effect. may_measure_water_level: title: May measure water level description: >- @@ -2286,9 +2267,6 @@ water_well_field_operations: How often the well is scheduled to be visited under its current monitoring plan. Null where no frequency is on record. enum-lexicon: monitoring_frequency - monitoring_frequency_since: - title: Monitoring frequency since - description: Date the current monitoring frequency took effect. group_names: title: Groups description: >- @@ -2352,7 +2330,7 @@ water_well_field_operations: Number of recorded visits to the well. Broader than either measurement count: a visit that produced no reading, because the gate was locked or the well was dry, is still a visit. - field_event_last_date: + date_last_visited: title: Last field visit description: >- Date of the most recent recorded visit, as a UTC calendar date. A gap @@ -2361,55 +2339,73 @@ water_well_field_operations: has_datalogger: title: Has a datalogger description: >- - Whether a logger is currently installed in the well -- a deployment with - an installation date and no removal date, of a data logger, pressure - transducer, DiverLink or diver cable. Other equipment deployed at the - well does not count. + Whether a logger -- a data logger, pressure transducer, DiverLink, or + diver cable -- is currently installed in the well, regardless of what + other equipment (a camera, a weather station) may also be deployed + there. See sensor_type for every currently-installed sensor, logger or + not. datalogger_deployment_count: title: Current logger deployments description: >- - Number of logger deployments currently open at the well. Normally zero or - one; a higher number means the single logger columns describe only the - most recently installed of several. - datalogger_sensor_type: - title: Logger type - description: Kind of logger currently installed. - enum: - - Data Logger - - Pressure Transducer - - DiverLink - - Diver Cable - datalogger_model: - title: Logger model - description: Model of the logger currently installed. - datalogger_serial_no: - title: Logger serial number - description: >- - Serial number of the logger currently installed, for matching the - instrument in hand against the record. - datalogger_sensor_status: - title: Logger status - description: >- - Service state of the installed logger. A logger still recorded as - installed but marked retired or lost is a record worth correcting in the - field. - enum-lexicon: sensor_status - datalogger_installed_date: - title: Logger installed - description: Date the current logger deployment began. - datalogger_recording_interval: - title: Logger recording interval - description: >- - How often the installed logger records a reading, in the units given by - the recording interval units column. - datalogger_recording_interval_units: - title: Logger recording interval units - description: Time units the logger recording interval is expressed in. - datalogger_hanging_point_description: - title: Logger hanging point - description: >- - Where the logger cable is secured at the wellhead, as written for the - person about to pull it. Null where the deployment did not record one. + Number of logger deployments currently open at the well -- data + loggers, pressure transducers, DiverLinks, and diver cables only. + Normally zero or one. Other currently-installed equipment is not + counted here; see sensor_type for the full list. + sensor_type: + title: Currently installed sensor type(s) + description: >- + Type(s) of sensor currently deployed at the well (a deployment with an + installation date and no removal date), from a controlled vocabulary, + not limited to data loggers -- a camera, weather station, or other + equipment counts too. A well running more than one sensor at once + lists all of them, semicolon-separated, ordered the same way as model, + serial_no, sensor_status, installed_date, recording_interval, + recording_interval_units, and hanging_point_desc, so position N in one + list is always the same deployment as position N in the others -- an + empty segment in one of those other fields (e.g. "; 15" for a well + with two sensors) means that particular sensor has no value recorded + for that field, not that it doesn't exist. See has_datalogger for the + narrower "is a logger installed" question. + model: + title: Currently installed sensor model(s) + description: >- + Model of each currently deployed sensor. See sensor_type for how + multiple concurrent deployments are listed and ordered. + serial_no: + title: Currently installed sensor serial number(s) + description: >- + Serial number of each currently deployed sensor, for matching the + instrument in hand against the record. See sensor_type for how + multiple concurrent deployments are listed and ordered. + sensor_status: + title: Currently installed sensor status(es) + description: >- + Equipment status of each currently deployed sensor, such as In Service + or In Repair, from a controlled vocabulary. See sensor_type for how + multiple concurrent deployments are listed and ordered. + installed_date: + title: Currently installed sensor installation date(s) + description: >- + Date each currently deployed sensor was installed. See sensor_type for + how multiple concurrent deployments are listed and ordered. + recording_interval: + title: Currently installed sensor recording interval(s) + description: >- + Recording interval of each currently deployed sensor. See sensor_type + for how multiple concurrent deployments are listed and ordered. + recording_interval_units: + title: Currently installed sensor recording interval unit(s) + description: >- + Unit of recording_interval for each currently deployed sensor. See + sensor_type for how multiple concurrent deployments are listed and + ordered. + hanging_point_desc: + title: Currently installed sensor hanging point description(s) + description: >- + Where each currently deployed sensor's cable is secured at the + wellhead, as written for the person about to pull it. See sensor_type + for how multiple concurrent deployments are listed and ordered. Null + where a deployment did not record one. continuous_reading_count: title: Continuous readings description: >- @@ -2489,6 +2485,77 @@ water_well_field_operations: Staff-written directions for finding the well on the ground, which coordinates alone do not give. Where several notes exist they are separated by vertical bars, most recently recorded first. + communication_notes: + title: Communication notes + description: >- + Staff-written record of communications about the well -- calls, + emails, or conversations worth keeping with the record. Where several + notes exist they are separated by vertical bars, most recently + recorded first. + construction_notes: + title: Construction notes + description: >- + Staff-written notes about how the well was built, beyond what the + construction columns capture. Where several notes exist they are + separated by vertical bars, most recently recorded first. + maintenance_notes: + title: Maintenance notes + description: >- + Staff-written notes about repairs or upkeep performed at the well. + Where several notes exist they are separated by vertical bars, most + recently recorded first. + historical_notes: + title: Historical notes + description: >- + Staff-written notes about the well's history that do not fit elsewhere + on the record. Where several notes exist they are separated by + vertical bars, most recently recorded first. + general_notes: + title: General notes + description: >- + Staff-written notes that do not fall into any of this layer's other + note categories. Where several notes exist they are separated by + vertical bars, most recently recorded first. + water_notes: + title: Water notes + description: >- + Staff-written notes about the well's water, distinct from the + structured water-chemistry and water-level records elsewhere in the + catalogue. Where several notes exist they are separated by vertical + bars, most recently recorded first. + water_quality_notes: + title: Water quality notes + description: >- + Staff-written notes about water quality observed at the well, distinct + from the structured chemistry analyte results elsewhere in the + catalogue. Where several notes exist they are separated by vertical + bars, most recently recorded first. + sampling_procedure_notes: + title: Sampling procedure notes + description: >- + Staff-written notes about how to sample this particular well -- + anything a standard procedure does not already cover. Where several + notes exist they are separated by vertical bars, most recently + recorded first. + coordinate_notes: + title: Coordinate notes + description: >- + Staff-written notes about how the well's coordinates were obtained, or + any doubt about their accuracy. Where several notes exist they are + separated by vertical bars, most recently recorded first. + owner_comment_notes: + title: Owner comment notes + description: >- + Comments attributed to the well's owner, as recorded by staff. Where + several notes exist they are separated by vertical bars, most recently + recorded first. + site_notes_legacy: + title: Site notes (legacy) + description: >- + Free-text site notes carried over from the legacy NM_Aquifer record, + kept for continuity rather than actively maintained. Where several + exist they are separated by vertical bars, most recently recorded + first. # EDR collections. Keys here are parameter names read out of the data, not # column names -- ogc_waterlevels stamps a single literal, while diff --git a/core/pygeoapi-config-internal.yml b/core/pygeoapi-config-internal.yml index 428fb81f5..4b2e8fdff 100644 --- a/core/pygeoapi-config-internal.yml +++ b/core/pygeoapi-config-internal.yml @@ -85,14 +85,16 @@ resources: every other well layer answers a scientific question, this one answers an operational one: whether a crew may go to a well, what they are permitted to do when they get there, and whether it is overdue. It pairs the well's - construction and measuring-point detail with its current well, - monitoring, access, open and datalogger-suitability status, the standing - landowner permissions for water-level measurement, chemistry sampling and - logger installation, the logger installed in it if any, how long it has + construction, completion formation and aquifer system with its + measuring-point detail, current well, monitoring, open and + datalogger-suitability status, the standing landowner permissions for + water-level measurement, chemistry sampling and logger installation, every + sensor currently deployed at it (not just data loggers), how long it has been since it was last measured and last sampled, and the landowner - contact and access notes needed to reach it. Published on the internal - mount only, because it carries personal contact details for landowners - and operators alongside staff-written access instructions. + contact and staff-written notes -- access, directions, maintenance and + the rest -- needed to reach it. Published on the internal mount only, + because it carries personal contact details for landowners and operators + alongside staff-written access instructions. keywords: [ water-wells, @@ -100,7 +102,8 @@ resources: landowner-contacts, access-permissions, monitoring-status, - data-logger, + currently-installed-equipment, + aquifer-system, internal, ] extents: diff --git a/docs/water-well-field-operations-columns.md b/docs/water-well-field-operations-columns.md index 423ad6eb7..d7f4ee03f 100644 --- a/docs/water-well-field-operations-columns.md +++ b/docs/water-well-field-operations-columns.md @@ -20,17 +20,12 @@ prose lives in `core/ogc-field-descriptions.yml`; the design rationale lives in | --- | --- | | `id` | `thing.id` | | `name` | `thing.name` | -| `thing_type` | Literal `'water well'` — the view's row filter | +| `station_type` | Literal `'water well'` — the view's row filter. Named `station_type` rather than `thing_type` to match the naming already established on the public thing-type views | | `release_status` | `thing.release_status` | -| `nma_pk_welldata` | `thing.nma_pk_welldata` | | `alternate_ids` | `thing_id_link.alternate_organization` + `.alternate_id`, comma-joined | -| `county` | `location.county`, most recent association | -| `state` | `location.state`, most recent association | -| `quad_name` | `location.quad_name`, most recent association | | `latitude` | `ST_Y(location.point)` — decimal degrees, WGS 84 | | `longitude` | `ST_X(location.point)` — decimal degrees, WGS 84 | | `elevation` | `location.elevation`, most recent association | -| `elevation_method` | `data_provenance.collection_method` where `target_table = 'location'` and `field_name = 'elevation'`, latest `id` | | `well_depth` | `thing.well_depth` | | `hole_depth` | `thing.hole_depth` | | `well_casing_diameter` | `thing.well_casing_diameter` | @@ -41,32 +36,25 @@ prose lives in `core/ogc-field-descriptions.yml`; the design rationale lives in | `well_pump_type` | `thing.well_pump_type` | | `well_pump_depth` | `thing.well_pump_depth` | | `formation_completion_code` | `thing.formation_completion_code` | -| `nma_formation_zone` | `thing.nma_formation_zone` | -| `well_purposes` | `well_purpose.purpose`, comma-joined | -| `well_casing_materials` | `well_casing_material.material`, comma-joined | +| `formation_completion_description` | `lexicon_term.definition` where `term = thing.formation_completion_code`. Neither this nor `aquifer_system_name` below is date-windowed — neither source table carries a `start_date`/`end_date` | +| `well_purpose` | `well_purpose.purpose`, comma-joined | +| `well_casing_material` | `well_casing_material.material`, comma-joined | +| `aquifer_system_name` | `aquifer_system.name` via `thing_aquifer_association`, comma-joined | | `screen_count` | `count(well_screen)` | -| `screen_depth_top` | `min(well_screen.screen_depth_top)` | -| `screen_depth_bottom` | `max(well_screen.screen_depth_bottom)` | -| `measuring_point_height` | `measuring_point_history.measuring_point_height`, current record | -| `measuring_point_description` | `measuring_point_history.measuring_point_description`, current record | -| `measuring_point_start_date` | `measuring_point_history.start_date`, current record | +| `screen_depth_top` | `well_screen.screen_depth_top`, every interval, semicolon-joined, ordered shallowest first, `COALESCE(..., '')` for the same reason as the equipment columns | +| `screen_depth_bottom` | `well_screen.screen_depth_bottom`, same intervals, same order, same `COALESCE(..., '')` treatment | +| `screen_description` | `well_screen.screen_description`, same intervals, same order, same `COALESCE(..., '')` treatment | +| `mp_height` | `measuring_point_history.measuring_point_height`, current record | +| `mp_description` | `measuring_point_history.measuring_point_description`, current record | | `well_status` | `status_history.status_value` where `status_type = 'Well Status'`, current record | -| `well_status_since` | `status_history.start_date`, same record | | `monitoring_status` | `status_history.status_value` where `status_type = 'Monitoring Status'`, current record | -| `monitoring_status_since` | `status_history.start_date`, same record | -| `monitoring_status_reason` | `status_history.reason`, same record | -| `access_status` | `status_history.status_value` where `status_type = 'Access Status'`, current record | -| `access_status_since` | `status_history.start_date`, same record | | `open_status` | `status_history.status_value` where `status_type = 'Open Status'`, current record | -| `open_status_since` | `status_history.start_date`, same record | | `datalogger_suitability_status` | `status_history.status_value` where `status_type = 'Datalogger Suitability Status'`, current record | -| `datalogger_suitability_status_since` | `status_history.start_date`, same record | | `may_measure_water_level` | `permission_history.permission_allowed` where `permission_type = 'Water Level Sample'`, current record | | `may_sample_water_chemistry` | `permission_history.permission_allowed` where `permission_type = 'Water Chemistry Sample'`, current record | | `may_install_datalogger` | `permission_history.permission_allowed` where `permission_type = 'Datalogger Installation'`, current record | | `permission_granted_by` | `contact.name` via `permission_history.contact_id` on the current water-level grant | | `monitoring_frequency` | `monitoring_frequency_history.monitoring_frequency`, current record | -| `monitoring_frequency_since` | `monitoring_frequency_history.start_date`, current record | | `group_names` | `group.name` via `group_thing_association`, comma-joined | | `group_types` | `group.group_type`, same order as `group_names` | | `manual_water_level_count` | `count(observation)` via `sample` → `field_activity` → `field_event`, `activity_type = 'groundwater level'` *(stats)* | @@ -78,17 +66,17 @@ prose lives in `core/ogc-field-descriptions.yml`; the design rationale lives in | `chemistry_sample_last_date` | `max(sample.sample_date)`, UTC date, same filter *(stats)* | | `days_since_chemistry_sample` | `CURRENT_DATE - chemistry_sample_last_date` | | `field_event_count` | `count(field_event)` for the well *(stats)* | -| `field_event_last_date` | `max(field_event.event_date)`, UTC date *(stats)* | -| `has_datalogger` | `true` when an open logger deployment exists — see `datalogger_sensor_type` | -| `datalogger_deployment_count` | `count(deployment)` open logger deployments | -| `datalogger_sensor_type` | `sensor.sensor_type` on the current deployment, restricted to Data Logger / Pressure Transducer / DiverLink / Diver Cable | -| `datalogger_model` | `sensor.model`, same deployment | -| `datalogger_serial_no` | `sensor.serial_no`, same deployment | -| `datalogger_sensor_status` | `sensor.sensor_status`, same deployment | -| `datalogger_installed_date` | `deployment.installation_date`, same deployment | -| `datalogger_recording_interval` | `deployment.recording_interval`, same deployment | -| `datalogger_recording_interval_units` | `deployment.recording_interval_units`, same deployment | -| `datalogger_hanging_point_description` | `deployment.hanging_point_description`, same deployment | +| `date_last_visited` | `max(field_event.event_date)`, UTC date *(stats, column named `field_event_last_date` there)* | +| `has_datalogger` | `true` when a currently-installed deployment exists whose `sensor.sensor_type` is Data Logger / Pressure Transducer / DiverLink / Diver Cable. Stays logger-scoped even though the columns below do not | +| `datalogger_deployment_count` | `count(deployment)`, same logger-only filter as `has_datalogger` | +| `sensor_type` | `sensor.sensor_type` for every currently-installed deployment (`installation_date IS NOT NULL AND removal_date IS NULL`), **any sensor type, not just loggers** — semicolon-joined, ordered by `sensor_type` | +| `model` | `sensor.model`, same deployments, same order as `sensor_type`. `COALESCE(..., '')` before aggregating, so a null value is an empty segment, not a dropped position | +| `serial_no` | `sensor.serial_no`, same deployments, same order, same `COALESCE(..., '')` treatment | +| `sensor_status` | `sensor.sensor_status`, same deployments, same order, same `COALESCE(..., '')` treatment | +| `installed_date` | `deployment.installation_date`, same deployments, same order. In practice never null -- `installed_deployments` filters on `installation_date IS NOT NULL` -- but `COALESCE`d anyway for consistency with its siblings | +| `recording_interval` | `deployment.recording_interval`, same deployments, same order, same `COALESCE(..., '')` treatment — `text`, not `integer`, because `string_agg` produces `text` regardless of how many deployments a given well has | +| `recording_interval_units` | `deployment.recording_interval_units`, same deployments, same order, same `COALESCE(..., '')` treatment | +| `hanging_point_desc` | `deployment.hanging_point_description`, same deployments, same order, same `COALESCE(..., '')` treatment | | `continuous_reading_count` | `count(transducer_observation)` via `deployment` *(stats)* | | `continuous_first_datetime` | `min(transducer_observation.observation_datetime)` *(stats)* | | `continuous_last_datetime` | `max(transducer_observation.observation_datetime)` *(stats)* | @@ -103,4 +91,15 @@ prose lives in `core/ogc-field-descriptions.yml`; the design rationale lives in | `contact_names` | `contact.name` for every associated contact, comma-joined | | `access_notes` | `notes.content` where `note_type = 'Access'`, newest first, joined with ` | ` | | `directions_notes` | `notes.content` where `note_type = 'Directions'`, newest first, joined with ` | ` | +| `communication_notes` | `notes.content` where `note_type = 'Communication'`, newest first, joined with ` | ` | +| `construction_notes` | `notes.content` where `note_type = 'Construction'`, newest first, joined with ` | ` | +| `maintenance_notes` | `notes.content` where `note_type = 'Maintenance'`, newest first, joined with ` | ` | +| `historical_notes` | `notes.content` where `note_type = 'Historical'`, newest first, joined with ` | ` | +| `general_notes` | `notes.content` where `note_type = 'General'`, newest first, joined with ` | ` | +| `water_notes` | `notes.content` where `note_type = 'Water'`, newest first, joined with ` | ` | +| `water_quality_notes` | `notes.content` where `note_type = 'Water Quality'`, newest first, joined with ` | ` | +| `sampling_procedure_notes` | `notes.content` where `note_type = 'Sampling Procedure'`, newest first, joined with ` | ` | +| `coordinate_notes` | `notes.content` where `note_type = 'Coordinate'`, newest first, joined with ` | ` | +| `owner_comment_notes` | `notes.content` where `note_type = 'OwnerComment'`, newest first, joined with ` | ` | +| `site_notes_legacy` | `notes.content` where `note_type = 'Site Notes (legacy)'`, newest first, joined with ` | `. Not `site_notes_legacy_notes` — the lexicon term already says "notes" | | `point` | `location.point`, most recent association (PostGIS Point, EPSG:4326) | diff --git a/docs/water-well-field-operations-layer.md b/docs/water-well-field-operations-layer.md index 9e2c61308..4a24334bf 100644 --- a/docs/water-well-field-operations-layer.md +++ b/docs/water-well-field-operations-layer.md @@ -1,7 +1,10 @@ # Water well field operations layer — design -Status: **implemented** on `feat/amp-field-operations-ogc-layer`. Migration -`e1f2a3b4c5d6`; column-by-column source table in +Status: **implemented** on `feat/amp-field-operations-ogc-layer`, then +consolidated with an independently-drafted layer +(`kas-water-well-operations-ogc-layer-bdms-1202`) on +`kas-water-well-field-operations-ogc-layer-bdms-1202` after a side-by-side +column comparison. Migration `e1f2a3b4c5d6`; column-by-column source table in `docs/water-well-field-operations-columns.md`. Two questions this document opened were settled before implementation: "AMP" is @@ -9,6 +12,20 @@ a label for the crews' view of the register, not a subset of it, so the layer covers every water well; and the contact detail in §4 is published in full (option A). +The consolidation dropped several columns this document originally specified +(`nma_pk_welldata`, `county`, `state`, `quad_name`, `elevation_method`, +`nma_formation_zone`, `measuring_point_start_date`, every `*_since`/`*_reason` +status/frequency column, and `access_status` itself -- open question 2 below, +resolved), renamed +several to match the naming already established on the public thing-type views +(`station_type`, `well_purpose`, `well_casing_material`, `mp_height`/ +`mp_description`, `date_last_visited`), added `formation_completion_description` +and `aquifer_system_name`, broadened the equipment columns from logger-only to +every currently-installed sensor, and added a column per remaining +`notes.note_type` value. The rest of this document describes the layer as +originally designed; §6 and §11 call out where the shipped column set now +differs. + ## 0. Why the name does not say AMP The layer was requested as `amp_field_well_operations` and renamed before it @@ -188,59 +205,86 @@ trip; an array does not. | --- | --- | | `id` | `thing.id` | | `name` | `thing.name` | -| `thing_type` | `'water well'::text` | +| `station_type` | `'water well'::text`. Named `station_type` rather than `thing_type` to match the naming already established on the public thing-type views (consolidation, see the note at the top of this document) | | `release_status` | `thing.release_status` | -| `nma_pk_welldata` | `thing.nma_pk_welldata` — legacy NM_Aquifer key, still what crews say out loud | | `alternate_ids` | `thing_id_link`, joined as `organization:alternate_id` pairs | -| `county`, `quad_name`, `state` | `location` | | `latitude`, `longitude` | `ST_Y`/`ST_X` of the same point, decimal degrees on WGS 84 -- a plain number survives a CSV export, which drops the geometry, and a crew can read it into a handheld GPS | | `elevation` | `location.elevation` | -| `elevation_method` | `data_provenance.collection_method` for that location's elevation -- the same lookup `water_well_summary` uses, so the two layers cannot disagree about how an elevation was obtained | | `point` | `location.point`, most recent association (the shared `LATEST_LOCATION_CTE`) | +`nma_pk_welldata`, `county`, `state`, `quad_name`, `nma_formation_zone`, and +`elevation_method` (and its `data_provenance` lookup) were dropped in the +consolidation. None of the four history-table current-record joins below +needed them, and none of the columns this layer is actually for -- status, +permission, equipment, contacts -- depend on them either. + ### Well construction `well_depth`, `hole_depth`, `well_casing_diameter`, `well_casing_depth`, `well_completion_date`, `well_driller_name`, `well_construction_method`, -`well_pump_type`, `well_pump_depth`, `formation_completion_code`, -`nma_formation_zone` — all straight from `thing`, same columns and same names -as the thing-view template, so a crew reading both layers sees one vocabulary. +`well_pump_type`, `well_pump_depth`, `formation_completion_code` — all straight +from `thing`, same columns and same names as the thing-view template, so a +crew reading both layers sees one vocabulary. Plus, aggregated: | Column | Source | | --- | --- | -| `well_purposes` | `well_purpose.purpose`, joined | -| `well_casing_materials` | `well_casing_material.material`, joined | +| `formation_completion_description` | `lexicon_term.definition` where `term = thing.formation_completion_code` -- the code alone is not self-explanatory | +| `well_purpose` | `well_purpose.purpose`, joined | +| `well_casing_material` | `well_casing_material.material`, joined | +| `aquifer_system_name` | `aquifer_system.name` via `thing_aquifer_association`, joined | | `screen_count` | `count(well_screen)` | -| `screen_depth_top`, `screen_depth_bottom` | `min`/`max` across that well's screens, ft below ground surface | +| `screen_depth_top`, `screen_depth_bottom`, `screen_description` | every screened interval, ft below ground surface, semicolon-joined, ordered shallowest first | + +The original design collapsed `well_screen` to `screen_count` plus an overall +`min`/`max` depth range, avoiding the per-interval alignment problem below at +the cost of the actual intervals: for a well with two screens, a range says +there's screened section somewhere between the shallowest top and the +deepest bottom, not where the gap between them is, or what each one is made +of. The consolidation restored full per-interval detail instead, on the +premise that a driller doing rehab work needs the real intervals, not a +range. All three columns are ordered by `screen_depth_top` (nulls last) and +each is wrapped in `COALESCE(..., '')` before `string_agg`, for the same +reason as the equipment columns below: `screen_depth_top`, +`screen_depth_bottom`, and `screen_description` are all nullable +independently of each other, and plain `string_agg` would drop a null and +misalign the other two columns' positions. + +`formation_completion_description` and `aquifer_system_name` are both from the +consolidation. Neither is date-windowed like the four history tables below -- +`thing.formation_completion_code` is a plain column and +`thing_aquifer_association` carries no `start_date`/`end_date` at all. ### Measuring point (current record only) | Column | Source | | --- | --- | -| `measuring_point_height` | `measuring_point_history.measuring_point_height`, ft above ground surface | -| `measuring_point_description` | e.g. "North side of casing, top of PVC" | -| `measuring_point_start_date` | when the current configuration took effect | +| `mp_height` | `measuring_point_history.measuring_point_height`, ft above ground surface | +| `mp_description` | e.g. "North side of casing, top of PVC" | The description is the single most useful string on the layer for a crew -standing at a wellhead. Current record per §7. +standing at a wellhead. Current record per §7. `measuring_point_start_date` +(when the current configuration took effect) was dropped in the consolidation, +along with the equivalent `_since` column for every status type below. ### Status (current record per status type) -`status_history.status_type` has five values in the lexicon, and all five are -published, each as `` plus `_since`: +`status_history.status_type` has five values in the lexicon. `Access Status` +is not published -- see open question 2, resolved in the consolidation -- so +four of the five are: | Column | `status_type` | Values | | --- | --- | --- | | `well_status` | Well Status | Abandoned; Active, pumping well; Destroyed, exists but not usable; Inactive, exists but not used | | `monitoring_status` | Monitoring Status | Currently monitored; Not currently monitored | -| `access_status` | Access Status | (lexicon does not yet scope values to type — open question 3) | | `open_status` | Open Status | Open; Open (unequipped); Closed | | `datalogger_suitability_status` | Datalogger Suitability Status | Datalogger can be installed; Datalogger cannot be installed | -`monitoring_status_reason` is also published — `status_history.reason`, which -is where "landowner asked us to stop" is written down. +The original design also published a `_since` column per status type +plus `monitoring_status_reason` (`status_history.reason`, where "landowner +asked us to stop" is written down). All six were dropped in the +consolidation; the current value is kept, the history around it is not. ### Permission (current grants) @@ -269,10 +313,12 @@ water-level grant — is published alongside, so a crew can name who said yes. | Column | Source | | --- | --- | | `monitoring_frequency` | current `monitoring_frequency_history` record | -| `monitoring_frequency_since` | its `start_date` | | `group_names` | `group` via `group_thing_association`, joined, de-duplicated | | `group_types` | Monitoring Plan / Geographic Area / Historical, joined, index-aligned with `group_names` | +`monitoring_frequency_since` (its `start_date`) was dropped in the +consolidation, same as the status `_since` columns above. + ### Manual measurement statistics Manual groundwater levels, reached the way every other water-level view in this @@ -304,34 +350,68 @@ Same chain, `field_activity.activity_type = 'water chemistry'`: ### Field visit statistics -`field_event_count`, `field_event_last_date` from `field_event`. Broader than -either measurement chain: a visit that produced no reading is still a visit, -and the gap between `field_event_last_date` and +`field_event_count`, `date_last_visited` from `field_event` +(`field_event_last_date` on the `_stats` matview itself; renamed on the way +out of the feature view to match `thing.first_visit_date`'s counterpart +naming). Broader than either measurement chain: a visit that produced no +reading is still a visit, and the gap between `date_last_visited` and `manual_water_level_last_date` is itself a signal. ### Data logger / continuous record -"Has a logger" is a deployment question, not a sensor-inventory question. A -deployment is current when `installation_date IS NOT NULL` and `removal_date IS -NULL`, and the sensor is one of `Data Logger`, `Pressure Transducer`, -`DiverLink`, `Diver Cable` (`sensor.sensor_type`). +**`has_datalogger`/`datalogger_deployment_count` stayed logger-scoped in the +consolidation; every other column in this section was broadened to any +currently-installed sensor.** The original design published only a single +current logger deployment's detail (`datalogger_sensor_type`, +`datalogger_model`, etc., picking the most recently installed one via +`DISTINCT ON` if more than one logger was deployed) and had no way to show a +currently-installed camera, barometer, or weather station at all -- it was +simply invisible. That was a real gap against this layer's own purpose: a +crew visiting a well to swap an SD card or check a barometer's battery needs +to know it is there. + +"Has a logger" is still a deployment question, not a sensor-inventory +question: a deployment is current when `installation_date IS NOT NULL` and +`removal_date IS NULL`, and counts towards `has_datalogger` only when the +sensor is one of `Data Logger`, `Pressure Transducer`, `DiverLink`, `Diver +Cable` (`sensor.sensor_type`). | Column | Definition | | --- | --- | -| `has_datalogger` | boolean, true when a current logger deployment exists | -| `datalogger_sensor_type`, `datalogger_model`, `datalogger_serial_no` | from `sensor` on the current deployment | -| `datalogger_sensor_status` | In Service / In Repair / Retired / Lost | -| `datalogger_installed_date` | `deployment.installation_date` | -| `datalogger_recording_interval`, `datalogger_recording_interval_units` | how often it logs | -| `datalogger_hanging_point_description` | where the cable hangs from — a field instruction | +| `has_datalogger` | boolean, true when a current *logger* deployment exists | +| `datalogger_deployment_count` | count of current logger deployments only | +| `sensor_type`, `model`, `serial_no`, `sensor_status`, `installed_date`, `recording_interval`, `recording_interval_units`, `hanging_point_desc` | from `sensor`/`deployment` for **every** currently-installed deployment, any sensor type | | `continuous_reading_count` | rows in `transducer_observation` for this well's deployments | | `continuous_first_datetime`, `continuous_last_datetime` | series extent | | `days_since_continuous_reading` | live, in the plain view | -Where a well has more than one current logger deployment, the singular -`datalogger_*` columns take the most recently installed one, and -`datalogger_deployment_count` says how many there are so nothing is silently -hidden. +The eight equipment columns are no longer a single row's worth of scalar +values -- a well running more than one current sensor lists all of them, +semicolon-joined, all eight ordered the same way by `sensor_type` so position +N in one list is the same deployment as position N in the others. This +follows the same "comma-joined text, not an array" convention as +`well_purpose`/`well_casing_material` above, with a semicolon instead of a +comma because these columns are genuinely positional across each other, not +independently de-duplicated lists. `recording_interval` is `text`, not +`integer`, as a consequence: `string_agg` always returns `text`, even for a +well with exactly one current sensor. + +Every expression but `sensor_type` itself is wrapped in `COALESCE(..., '')` +before aggregating. `sensor_type` is `NOT NULL` on `sensor`, but the other +seven columns are all nullable -- a camera, for instance, has no +`recording_interval` -- and plain `string_agg` silently **drops** a `NULL` +input rather than preserving its position. Caught empirically: a well with a +transducer (`recording_interval = 15`) and a camera (`recording_interval = +NULL`) produced `sensor_type = "Camera; Pressure Transducer"` but +`recording_interval = "15"` -- one segment short, so position 0 read as the +camera's interval when it was actually the transducer's. With the `COALESCE`, +the same well now reads `recording_interval = "; 15"`: an empty segment for +the camera, keeping every column's segment count equal to +`installed_deployments`' row count for that well. + +`sensor_status` values are In Service / In Repair / Retired / Lost. +`datalogger_deployment_count` still says how many *logger* deployments are +open, independent of how many total sensors `sensor_type` lists. `continuous_reading_count` aggregates the largest table in the schema. It is the reason `_stats` is materialized. If the refresh proves too slow, the @@ -364,18 +444,36 @@ reaches a landowner, and it is the most sensitive field on the record. ### Notes `notes` is polymorphic on `(target_id, target_table = 'thing')` with a -`note_type` lexicon. Two types are operational and are published as their own -columns, most recent first, joined: +`note_type` lexicon. The original design published only two operational +types (`access_notes`, `directions_notes`) and kept the rest out, on the +theory that a well's non-visit history is where PII leaks. The consolidation +published a column per remaining `note_type` value instead, same pattern, +most recent first, joined with ` | `: | Column | `note_type` | | --- | --- | | `access_notes` | Access | | `directions_notes` | Directions | - -The rest (Construction, Maintenance, Historical, General, Water, Water Quality, -Sampling Procedure, Coordinate, OwnerComment, Site Notes (legacy)) stay out. -They are a well's history, not its visit instructions, and free text is where -PII leaks — see open question 4. +| `communication_notes` | Communication | +| `construction_notes` | Construction | +| `maintenance_notes` | Maintenance | +| `historical_notes` | Historical | +| `general_notes` | General | +| `water_notes` | Water | +| `water_quality_notes` | Water Quality | +| `sampling_procedure_notes` | Sampling Procedure | +| `coordinate_notes` | Coordinate | +| `owner_comment_notes` | OwnerComment | +| `site_notes_legacy` | Site Notes (legacy) | + +Two do not mechanically snake-case: `OwnerComment` has no separator to split +on, so `owner_comment_notes` is a judgment call, not a derivation. `Site Notes +(legacy)` becomes `site_notes_legacy`, not `site_notes_legacy_notes` -- the +term already says "notes". + +This does not revisit the PII conclusion in §4 -- if anything it strengthens +it, since these note types are exactly where a landowner's name, a gate code, +or a phone number ends up in free text. See open question 4. ## 7. The "current record" rule @@ -414,13 +512,20 @@ walks both mounts and fails on any published column without a `description`. Columns already in `_defaults` (`id`, `name`, `thing_type`, `release_status`, `elevation`, `well_depth`, `hole_depth`, `well_casing_diameter`, …) are -inherited and must not be restated. Everything new to this layer needs an -entry, and the entry says what the value *means*, never how the view is built. - -Controlled vocabularies use `enum-lexicon` rather than a duplicated `enum` -list: `status_value` for the five status columns, `monitoring_frequency`, -`sensor_type`, `sensor_status`, `well_purpose`, `casing_material`, `role`, -`group_type`. Depth and height columns carry +inherited and must not be restated. `station_type` needs its own entry despite +being lexicon-governed the same way `thing_type` is, precisely because it is +not named `thing_type` here -- a per-table entry only inherits from +`_defaults` by column name. Everything new to this layer needs an entry, and +the entry says what the value *means*, never how the view is built. + +Not every lexicon-governed column carries `enum-lexicon`: the five status +columns use a literal `enum` list instead (pre-dating the consolidation, and +left as-is rather than relitigated here), and `monitoring_frequency`, `role`, +and `contact_type` use `enum-lexicon`. `sensor_type`, `sensor_status`, +`well_purpose`, and `well_casing_material` carry **neither** -- each is an +aggregated, multi-valued column (`string_agg` over more than one row), and an +`enum` on a column whose value can be `"Barometer; Pressure Transducer"` would +be actively wrong, not just incomplete. Depth and height columns carry `x-ogc-unit: https://qudt.org/vocab/unit/FT` with `x-ogc-unitLang: QUDT`, matching the rest of the file. @@ -472,6 +577,8 @@ already added four on the observation chain. - `notes (target_id, target_table)` — already exists as `ix_notes_polymorphic_link` - `deployment (thing_id)` - `thing_contact_association (thing_id)`, `(contact_id)` +- `thing_aquifer_association (thing_id)` — for `aquifer_system_name`, added in + the consolidation - `transducer_observation (deployment_id, observation_datetime)` — for the continuous aggregates; the largest table in the schema and the one that will decide whether the nightly refresh is acceptable @@ -484,6 +591,19 @@ Settled before implementation: is no group predicate. - **Contact detail is published in full** (§4, option A). +Settled in the consolidation: + +- **Access Status has no values -- column removed, not left publishing NULL.** + `status_type` carried `Access Status`, but the `status_value` lexicon has no + access-related terms; the eleven values map only to Well, Monitoring, Open, + and Datalogger Suitability status. Rather than ship a column that could only + ever read NULL, `access_status` and its join were dropped. `access_notes` + (§6, Notes) already carries staff-written access information for a well -- + gate codes, locks, who to call -- so nothing a crew needs is lost. If the + lexicon later gains terms scoped to Access Status, this is worth + reconsidering, but as a fresh addition rather than un-deleting a column that + published nothing. + Still open: 1. **Which `thing_type` values count as a well?** Shipped as `'water well'` @@ -491,20 +611,15 @@ Still open: visit piezometers and monitoring wells, the predicate widens to a list — and then it should widen in the other four layers too, as its own ticket, rather than this layer quietly disagreeing with them. -2. **Access Status has no values.** `status_type` carries `Access Status` but - the `status_value` lexicon has no access-related terms — the eleven values - map to Well, Monitoring, Open, and Datalogger Suitability status. The column - is published and will read null until the vocabulary gains terms scoped to - it; the field description says so. Either access statuses are recorded with - values from another category, or the column is empty in practice, and the - answer needs a look at production data. -3. **Free-text PII.** `access_notes` and `directions_notes` are staff-written - and will contain phone numbers, gate codes, and names. A further reason the - layer is internal-only, and the §4 decision covers them too. -4. **Refresh cadence.** Nightly, via the existing pg_cron job. If crews plan +2. **Free-text PII.** All 13 note columns are staff-written and will contain + phone numbers, gate codes, and names -- more of them since the + consolidation, which published the eleven note types the original design + deliberately kept out (§6, Notes). A further reason the layer is + internal-only, and the §4 decision covers them too. +3. **Refresh cadence.** Nightly, via the existing pg_cron job. If crews plan the next day's route the evening before, nightly is fine; if they re-plan in the field, the stats columns are the ones that go stale. -5. **`continuous_reading_count` cost at production scale.** It aggregates the +4. **`continuous_reading_count` cost at production scale.** It aggregates the largest table in the schema. If the nightly refresh proves too slow, the fallback is to source the three continuous columns from `transducer_daily_data` instead. diff --git a/tests/test_ogc_water_well_field_operations.py b/tests/test_ogc_water_well_field_operations.py index a2b65541b..eb5c43a6c 100644 --- a/tests/test_ogc_water_well_field_operations.py +++ b/tests/test_ogc_water_well_field_operations.py @@ -16,9 +16,11 @@ """The internal-only water well field operations layer (e1f2a3b4c5d6). What is worth testing here is not that the columns exist but that the layer's -three load-bearing rules hold: a well with no measurements still appears, a -permission with no record reads NULL rather than false, and a history record -whose window has closed is not treated as current. +load-bearing rules hold: a well with no measurements still appears, a +permission with no record reads NULL rather than false, a history record +whose window has closed is not treated as current, and a currently-installed +sensor that is not a logger is still visible in the equipment columns even +though it does not count towards has_datalogger. See docs/water-well-field-operations-layer.md. """ @@ -40,12 +42,16 @@ ) from core.factory import create_api_app from db import ( - DataProvenance, + AquiferSystem, Deployment, + LexiconTerm, Notes, PermissionHistory, Sensor, StatusHistory, + Thing, + ThingAquiferAssociation, + WellScreen, ) from db.engine import session_ctx from tests import override_authentication @@ -156,46 +162,134 @@ def test_latitude_and_longitude_match_the_geometry(water_well_thing): assert -90 <= row.latitude <= 90 -def test_elevation_method_reads_the_location_provenance(water_well_thing, location): +# ------------------------------------------------------------- construction + + +def test_formation_completion_description_reads_the_lexicon_definition( + water_well_thing, +): + with session_ctx() as session: + term = LexiconTerm( + term="Test Formation XYZ", + definition="A test formation used only by this test", + ) + session.add(term) + session.commit() + + thing = session.get(Thing, water_well_thing.id) + thing.formation_completion_code = term.term + session.commit() + + row = _row( + session, + water_well_thing.id, + "formation_completion_code, formation_completion_description", + ) + assert row.formation_completion_code == "Test Formation XYZ" + assert ( + row.formation_completion_description + == "A test formation used only by this test" + ) + + thing.formation_completion_code = None + session.commit() + session.delete(term) + session.commit() + + +def test_formation_completion_description_is_null_without_a_code(water_well_thing): with session_ctx() as session: - row = _row(session, water_well_thing.id, "elevation_method") - # No provenance record is "not recorded", not a method of "unknown". - assert row.elevation_method is None + row = _row(session, water_well_thing.id, "formation_completion_description") + + assert row.formation_completion_description is None + - provenance = DataProvenance( - target_id=location.id, - target_table="location", - field_name="elevation", - collection_method="Survey-grade GPS", +def test_aquifer_system_name_is_comma_joined(water_well_thing): + with session_ctx() as session: + system_a = AquiferSystem( + name="Test Aquifer A", primary_aquifer_type="Unconfined multiple aquifers" + ) + system_b = AquiferSystem( + name="Test Aquifer B", primary_aquifer_type="Confined multiple aquifers" ) - session.add(provenance) + session.add_all([system_a, system_b]) session.commit() - row = _row(session, water_well_thing.id, "elevation_method") - assert row.elevation_method == "Survey-grade GPS" + associations = [ + ThingAquiferAssociation( + thing_id=water_well_thing.id, aquifer_system_id=system_a.id + ), + ThingAquiferAssociation( + thing_id=water_well_thing.id, aquifer_system_id=system_b.id + ), + ] + session.add_all(associations) + session.commit() + + row = _row(session, water_well_thing.id, "aquifer_system_name") + assert row.aquifer_system_name == "Test Aquifer A, Test Aquifer B" - session.delete(provenance) + for association in associations: + session.delete(association) + session.delete(system_a) + session.delete(system_b) session.commit() -def test_elevation_method_ignores_provenance_for_other_fields( - water_well_thing, location +def test_screens_list_every_interval_position_aligned( + water_well_thing, well_screen, second_well_screen +): + with session_ctx() as session: + row = _row( + session, + water_well_thing.id, + "screen_count, screen_depth_top, screen_depth_bottom, " + "screen_description", + ) + + # well_screen is 10-20ft, second_well_screen is 30-40ft, so + # screen_depth_top's ascending order puts well_screen first throughout. + assert row.screen_count == 2 + assert row.screen_depth_top == "10; 30" + assert row.screen_depth_bottom == "20; 40" + assert row.screen_description == ( + "Test well screen description; Test well screen description" + ) + + +def test_a_null_screen_field_leaves_an_empty_slot_not_a_dropped_position( + water_well_thing, well_screen ): + # A screen with no recorded bottom depth or description. Plain + # string_agg would drop those NULLs, shortening screen_depth_bottom and + # screen_description to one entry each while screen_depth_top still had + # two -- position 0 would then read as well_screen's bottom depth when + # it is actually the incomplete screen's. with session_ctx() as session: - provenance = DataProvenance( - target_id=location.id, - target_table="location", - field_name="point", - collection_method="Survey-grade GPS", + incomplete_screen = WellScreen( + thing_id=water_well_thing.id, + screen_depth_top=5.0, + screen_depth_bottom=None, + screen_description=None, + release_status="draft", ) - session.add(provenance) + session.add(incomplete_screen) session.commit() - row = _row(session, water_well_thing.id, "elevation_method") - # How the coordinates were obtained says nothing about the elevation. - assert row.elevation_method is None + row = _row( + session, + water_well_thing.id, + "screen_depth_top, screen_depth_bottom, screen_description", + ) + + # 5.0 sorts before well_screen's 10.0, so position 0 is the + # incomplete screen throughout -- an empty segment where its values + # are null, not a shorter list. + assert row.screen_depth_top == "5; 10" + assert row.screen_depth_bottom == "; 20" + assert row.screen_description == "; Test well screen description" - session.delete(provenance) + session.delete(incomplete_screen) session.commit() @@ -314,14 +408,11 @@ def test_status_reads_the_current_record_not_the_latest( row = _row( session, water_well_thing.id, - "monitoring_status, monitoring_status_since, well_status, " - "well_status_since", + "monitoring_status, well_status", ) assert row.monitoring_status is None - assert row.monitoring_status_since is None assert row.well_status == "Active, pumping well" - assert row.well_status_since == last_year session.delete(closed) session.delete(open_status) @@ -344,15 +435,13 @@ def test_status_types_do_not_bleed_into_each_other(water_well_thing, last_year): row = _row( session, water_well_thing.id, - "monitoring_status, monitoring_status_reason, well_status, " - "open_status, access_status, datalogger_suitability_status", + "monitoring_status, well_status, " + "open_status, datalogger_suitability_status", ) assert row.monitoring_status == "Not currently monitored" - assert row.monitoring_status_reason == "landowner asked us to stop" assert row.well_status is None assert row.open_status is None - assert row.access_status is None assert row.datalogger_suitability_status is None session.delete(monitoring) @@ -370,15 +459,16 @@ def test_has_datalogger_counts_only_logger_equipment( session, water_well_thing.id, "has_datalogger, datalogger_deployment_count, " - "datalogger_sensor_type, datalogger_serial_no, " - "datalogger_recording_interval", + "sensor_type, serial_no, recording_interval", ) assert row.has_datalogger is True assert row.datalogger_deployment_count == 1 - assert row.datalogger_sensor_type == "Pressure Transducer" - assert row.datalogger_serial_no == "123456" - assert row.datalogger_recording_interval == 24 + # Single current sensor: aggregation degenerates to a bare value, no + # delimiter. + assert row.sensor_type == "Pressure Transducer" + assert row.serial_no == "123456" + assert row.recording_interval == "24" def test_non_logger_equipment_does_not_make_a_well_instrumented( @@ -405,17 +495,126 @@ def test_non_logger_equipment_does_not_make_a_well_instrumented( row = _row( session, water_well_thing.id, - "has_datalogger, datalogger_deployment_count", + "has_datalogger, datalogger_deployment_count, sensor_type", ) - # A barometer at the well is equipment, not a logger in the well. + # A barometer at the well is equipment, not a logger in the well -- + # but it must still be visible as installed equipment, unlike the + # old logger-only columns this replaces. assert row.has_datalogger is False assert row.datalogger_deployment_count == 0 + assert row.sensor_type == "Barometer" session.delete(deployment) session.delete(barometer) session.commit() +def test_installed_equipment_lists_every_sensor_position_aligned( + water_well_thing, sensor, sensor_to_water_well_thing_deployment, last_year +): + with session_ctx() as session: + barometer = Sensor( + name="Test Barometer", + sensor_type="Barometer", + model="BaroTroll", + serial_no="BT-002", + sensor_status="In Service", + release_status="draft", + ) + session.add(barometer) + session.commit() + barometer_deployment = Deployment( + sensor_id=barometer.id, + thing_id=water_well_thing.id, + installation_date=last_year, + removal_date=None, + recording_interval=60, + recording_interval_units="minute", + hanging_point_description="Strapped to fence post", + ) + session.add(barometer_deployment) + session.commit() + + row = _row( + session, + water_well_thing.id, + "sensor_type, model, serial_no, recording_interval, " + "recording_interval_units, hanging_point_desc, has_datalogger, " + "datalogger_deployment_count", + ) + + # Ordered alphabetically by sensor_type ("Barometer" < + # "Pressure Transducer"), and every column ordered the same way, so + # position 0 in each list describes the barometer and position 1 the + # transducer. + assert row.sensor_type == "Barometer; Pressure Transducer" + assert row.model == "BaroTroll; Model X" + assert row.serial_no == "BT-002; 123456" + assert row.recording_interval == "60; 24" + assert row.recording_interval_units == "minute; hour" + assert row.hanging_point_desc == "Strapped to fence post; hang 10" + # The non-logger sensor does not count towards the logger-only signal. + assert row.has_datalogger is True + assert row.datalogger_deployment_count == 1 + + session.delete(barometer_deployment) + session.delete(barometer) + session.commit() + + +def test_a_null_field_on_one_sensor_leaves_an_empty_slot_not_a_dropped_position( + water_well_thing, sensor, sensor_to_water_well_thing_deployment, last_year +): + # A camera has no recording interval. Plain string_agg would silently + # drop that NULL, shortening recording_interval's list to one entry while + # sensor_type still has two -- position 0 would then read as the + # camera's interval when it is actually the transducer's. The view + # COALESCEs to '' specifically so this doesn't happen. + with session_ctx() as session: + camera = Sensor( + name="Test Camera", + sensor_type="Camera", + model="Reconyx HC600", + serial_no=None, + sensor_status="In Service", + release_status="draft", + ) + session.add(camera) + session.commit() + camera_deployment = Deployment( + sensor_id=camera.id, + thing_id=water_well_thing.id, + installation_date=last_year, + removal_date=None, + recording_interval=None, + recording_interval_units=None, + hanging_point_description=None, + ) + session.add(camera_deployment) + session.commit() + + row = _row( + session, + water_well_thing.id, + "sensor_type, model, serial_no, recording_interval, " + "recording_interval_units, hanging_point_desc", + ) + + # "Camera" sorts before "Pressure Transducer", so position 0 is the + # camera throughout -- an empty segment where its value is null, not + # a shorter list. + assert row.sensor_type == "Camera; Pressure Transducer" + assert row.model == "Reconyx HC600; Model X" + assert row.serial_no == "; 123456" + assert row.recording_interval == "; 24" + assert row.recording_interval_units == "; hour" + assert row.hanging_point_desc == "; hang 10" + + session.delete(camera_deployment) + session.delete(camera) + session.commit() + + def test_a_removed_logger_leaves_the_well_uninstrumented( water_well_thing, sensor, sensor_to_water_well_thing_deployment, yesterday ): @@ -518,12 +717,46 @@ def test_multi_valued_columns_are_comma_joined_text( water_well_thing, domestic_well_purpose, irrigation_well_purpose ): with session_ctx() as session: - row = _row(session, water_well_thing.id, "well_purposes") + row = _row(session, water_well_thing.id, "well_purpose") # Text, not an array: this layer is exported to File Geodatabase and # GeoPackage for offline field use, and neither format has a list type. - assert isinstance(row.well_purposes, str) - assert row.well_purposes == "Domestic, Irrigation" + assert isinstance(row.well_purpose, str) + assert row.well_purpose == "Domestic, Irrigation" + + +def test_note_types_do_not_bleed_into_each_other(water_well_thing): + with session_ctx() as session: + water_note = Notes( + target_id=water_well_thing.id, + target_table="thing", + note_type="Water", + content="Well produces slightly sulfurous water", + ) + maintenance_note = Notes( + target_id=water_well_thing.id, + target_table="thing", + note_type="Maintenance", + content="Pump replaced 2024", + ) + session.add_all([water_note, maintenance_note]) + session.commit() + + row = _row( + session, + water_well_thing.id, + "water_notes, maintenance_notes, coordinate_notes, " "owner_comment_notes", + ) + + assert row.water_notes == "Well produces slightly sulfurous water" + assert row.maintenance_notes == "Pump replaced 2024" + # Untouched note types stay null rather than inheriting either note. + assert row.coordinate_notes is None + assert row.owner_comment_notes is None + + session.delete(water_note) + session.delete(maintenance_note) + session.commit() def test_contacts_and_access_notes_are_published(water_well_thing, contact, phone): From 19cd51e87cfffdc1dcfc108ecd618e3c7718d2b7 Mon Sep 17 00:00:00 2001 From: Kelsey Smuczynski Date: Tue, 1 Sep 2026 12:34:33 -0600 Subject: [PATCH 4/5] fix(ogc): allow internal relations with no public twin tests/features/ogc-cleanup-sprint1.feature's "Internal database relations are separate from public relations" scenario asserts that every ogc_internal_ relation in the schema has a distinct ogc_ public counterpart. That held for every internal-only collection so far (locations, avg_tds_wells, latest_depth_to_water_wells): each is hidden from the public collections catalog, but both relations still exist in the database. water_well_field_operations breaks that pattern on purpose. ogc_water_well_field_operations does not exist at all, by design (see docs/water-well-field-operations-layer.md section 3), since the layer publishes landowner contact details and staff-written notes. The check had no way to know that was intentional, so it failed CI on this PR. Added an explicit exception set for the two relations this layer introduces, rather than loosening the assertion generally. The check should still fail if some future ogc_internal_ relation is missing its public twin by accident. --- tests/features/steps/ogc-cleanup-sprint1.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/features/steps/ogc-cleanup-sprint1.py b/tests/features/steps/ogc-cleanup-sprint1.py index 5ccf0b229..53e91367a 100644 --- a/tests/features/steps/ogc-cleanup-sprint1.py +++ b/tests/features/steps/ogc-cleanup-sprint1.py @@ -835,11 +835,24 @@ def step_then_schema_contains_relations_prefixed(context, prefix): assert matching, f"expected at least one relation prefixed {prefix!r}, found none" +# Relations that are internal-only by design, with no public counterpart at +# all -- not an accidental gap this check should catch. This layer publishes +# landowner contact details and staff-written notes; see +# docs/water-well-field-operations-layer.md section 3 for why it deliberately +# has no ogc_water_well_field_operations public twin. +INTERNAL_ONLY_NO_PUBLIC_COUNTERPART = { + "ogc_internal_water_well_field_operations", + "ogc_internal_water_well_field_operations_stats", +} + + @then("no ogc_internal_ relation is shared with the public /ogcapi endpoint") def step_then_no_internal_relation_shared_with_public(context): internal = {r for r in context.schema_relations if r.startswith("ogc_internal_")} assert internal, "no ogc_internal_ relations found in the schema" for relation in internal: + if relation in INTERNAL_ONLY_NO_PUBLIC_COUNTERPART: + continue public_equivalent = relation.replace("ogc_internal_", "ogc_", 1) assert public_equivalent in context.schema_relations, ( f"{relation} has no distinct public counterpart ({public_equivalent}) in " From 7f7ae11b862236173f64fdef92797de16dd0f392 Mon Sep 17 00:00:00 2001 From: Jake Ross Date: Tue, 1 Sep 2026 12:57:16 -0700 Subject: [PATCH 5/5] Potential fix for pull request finding Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../e1f2a3b4c5d6_add_water_well_field_operations_layer.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/alembic/versions/e1f2a3b4c5d6_add_water_well_field_operations_layer.py b/alembic/versions/e1f2a3b4c5d6_add_water_well_field_operations_layer.py index cf4242f51..845cfc01c 100644 --- a/alembic/versions/e1f2a3b4c5d6_add_water_well_field_operations_layer.py +++ b/alembic/versions/e1f2a3b4c5d6_add_water_well_field_operations_layer.py @@ -656,12 +656,10 @@ def _create_feature_view() -> str: open_st.status_value AS open_status, dl_st.status_value AS datalogger_suitability_status, - -- Permission, current grants. Three-valued: NULL means no - -- permission is on record, which is not the same as denied. wl_perm.permission_allowed AS may_measure_water_level, chem_perm.permission_allowed AS may_sample_water_chemistry, dl_perm.permission_allowed AS may_install_datalogger, - granter.name AS permission_granted_by, + CASE WHEN wl_perm.permission_allowed IS TRUE THEN granter.name END AS permission_granted_by, -- Monitoring programme. cmf.monitoring_frequency,