diff --git a/CLAUDE.md b/CLAUDE.md index a5f9358c..bcc472bb 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -204,6 +204,18 @@ ArcGIS Pro cannot send a bearer token at all and neither desktop client can refresh an Authentik token. Read **`docs/internal-ogc-desktop-gis.md`** before changing the credential paths. +**`/ogcapi-internal` carries landowner PII.** The `water_well_field_operations` +collection publishes contact name, organization, role, phone and email for well +owners and operators, plus staff-written access notes. Every credential the +mount accepts reaches it, the static keys included. It is internal-only with +**no public twin** — `ogc_water_well_field_operations` does not exist and must +never be created. The layer also honours `end_date` when reading history +tables, unlike `ogc_actively_monitored_wells`, so "may we sample here" cannot +outlive the permission that granted it. Read +**`docs/water-well-field-operations-layer.md`** before changing it, and +**`docs/water-well-field-operations-columns.md`** for where each column comes +from. + ### OGC field descriptions Per-column `title`/`description`/unit for every collection lives in diff --git a/alembic/versions/e1f2a3b4c5d6_add_water_well_field_operations_layer.py b/alembic/versions/e1f2a3b4c5d6_add_water_well_field_operations_layer.py new file mode 100644 index 00000000..845cfc01 --- /dev/null +++ b/alembic/versions/e1f2a3b4c5d6_add_water_well_field_operations_layer.py @@ -0,0 +1,831 @@ +"""add the water well field operations layer + +An internal-only OGC layer for field crews: one row per water well, carrying +what a crew needs to plan and execute a visit -- where the well is, who owns it +and how to reach them, what the crew is permitted to do there, what is +installed in it, and when it was last measured. Every other well layer answers +a scientific question; this one answers an operational one. + +Two relations, not one: + + ogc_internal_water_well_field_operations_stats MATERIALIZED VIEW + Counts, first/last dates and aggregates over `observation` and + `transducer_observation`. Expensive, refreshed nightly by the pg_cron + job (b6c7d8e9f0a1), which discovers matviews from the catalog and so + needs no schedule change. + + ogc_internal_water_well_field_operations VIEW <- pygeoapi reads this + Live join of thing, location, status_history, permission_history, + measuring_point_history, monitoring_frequency_history, deployment, + sensor, contact and notes, LEFT JOINed to the stats matview. + +The split is not tidiness. Staleness is dangerous on exactly the columns that +are cheap to read: a revoked sampling permission that still reads `true` until +the next nightly refresh sends a crew onto land they are no longer welcome on. +And the current-record rule below is written against CURRENT_DATE, which in a +materialized view would freeze at refresh time -- a permission that expired +this morning would still read current tonight. Those columns have to be in the +plain view. + +Internal only, and deliberately without a public twin: the layer publishes +landowner contact details and staff-written access notes. +`ogc_water_well_field_operations` does not exist and must never be created. The +parity guard in tests/test_migration_view_parity.py reads two named migration +files (f4a5b6c7d8e9 and 2d3c3a268652) and is unaffected by a relation created +here. + +Current-record rule, applied identically to all four history tables +(status_history, permission_history, measuring_point_history, +monitoring_frequency_history): + + start_date <= CURRENT_DATE + AND (end_date IS NULL OR end_date >= CURRENT_DATE) + ORDER BY start_date DESC, id DESC -- via DISTINCT ON + +This diverges from ogc_actively_monitored_wells, which takes the greatest +start_date and ignores end_date entirely, so a status closed in 2019 still +reads as current there. Tolerable on a summary layer; not on one whose job is +to say what is true today. See docs/ogc_conventions.md. + +The three permission columns are three-valued and must stay that way: true = +a current grant says allowed, false = a current grant says not allowed, NULL = +no permission on record. NULL is not "denied" -- it means nobody has asked the +landowner yet. + +Depth to water uses (value - COALESCE(measuring_point_height, 0)) -- the +reading minus the height of the measuring point above ground, a missing height +treated as ground level. Character-for-character the convention in +ogc_water_well_summary, ogc_latest_depth_to_water_wells and +ogc_well_water_column, so the four layers cannot disagree about what a depth to +water is. + +Multi-valued columns are emitted as comma-joined text rather than as arrays +(which ogc_actively_monitored_wells uses). This layer exists to be pulled into +ArcGIS Pro and QGIS and exported to a File Geodatabase or GeoPackage for +offline field use, and neither format has a list type. + +Both relations are new here, so downgrade() drops them rather than restoring a +prior state. The supporting indexes are also created here: none of these +foreign keys was indexed (Postgres does not index foreign keys on its own). + +Consolidated from two independently-drafted layers (this one and +kas-water-well-operations-ogc-layer-bdms-1202) after a side-by-side review. +The deltas from the version that first merged (3499f414): + +- Dropped: nma_pk_welldata, county, state, quad_name, elevation_method (and + its data_provenance LATERAL lookup, now unused), nma_formation_zone, + measuring_point_start_date, and every `*_since`/`*_reason` status and + monitoring-frequency column. access_status itself is also dropped (not just + its `_since` companion): status_value has no terms scoped to Access Status + in the lexicon, so the column could only ever read NULL, and access_notes + already carries staff-written access information for a well. +- Renamed to match the naming already established on the public thing-type + views: thing_type -> station_type, well_casing_materials -> + well_casing_material, well_purposes -> well_purpose, + measuring_point_height/measuring_point_description -> mp_height/ + mp_description, field_event_last_date (from the stats matview) -> + date_last_visited. +- Added formation_completion_description (lexicon_term.definition for the + term named by formation_completion_code) and aquifer_system_name + (thing_aquifer_association -> aquifer_system, comma-joined), both scalar + lookups with no date-window semantics of their own -- neither source table + carries a start_date/end_date. +- Broadened "currently installed equipment" from logger-only to every + currently-installed sensor. The original datalogger_sensor_type/model/ + serial_no/sensor_status/installed_date/recording_interval/ + recording_interval_units/hanging_point_description picked a single row + (DISTINCT ON, most-recently-installed) from deployments filtered to + LOGGER_SENSOR_TYPES, so a currently-installed camera or barometer was + invisible. The unprefixed sensor_type/model/serial_no/sensor_status/ + installed_date/recording_interval/recording_interval_units/ + hanging_point_desc columns now aggregate every currently-installed sensor + regardless of type (semicolon-joined, ordered by sensor_type, same + convention as the comma-joined columns above but semicolon because these + are genuinely positional -- position N in one list is the same deployment as + position N in the others). has_datalogger/datalogger_deployment_count stay + logger-scoped exactly as before, for whoever specifically needs "is this + well instrumented," now computed from the broader deployment set filtered + inline rather than a separate pre-filtered CTE. +- Added one column per notes.note_type value (13 total, including the + pre-existing access_notes/directions_notes), `_notes`-suffixed, same + string_agg(' | ', id DESC) pattern as the original two. `OwnerComment` -> + owner_comment_notes and `Site Notes (legacy)` -> site_notes_legacy are the + two that do not mechanically snake-case; the legacy one is not + double-suffixed since the term already says "notes". + +Revision ID: e1f2a3b4c5d6 +Revises: c9d0e1f2a3b4 +Create Date: 2026-08-31 00:00:00.000000 +""" + +import re +from typing import Sequence, Union + +from alembic import op +from sqlalchemy import inspect, text + +# revision identifiers, used by Alembic. +revision: str = "e1f2a3b4c5d6" +down_revision: Union[str, Sequence[str], None] = "c9d0e1f2a3b4" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +REQUIRED_TABLES = { + "thing", + "location", + "location_thing_association", + "observation", + "sample", + "field_activity", + "field_event", + "status_history", + "permission_history", + "measuring_point_history", + "monitoring_frequency_history", + "deployment", + "sensor", + "transducer_observation", + "contact", + "thing_contact_association", + "phone", + "email", + "notes", + "group", + "group_thing_association", + "well_purpose", + "well_casing_material", + "well_screen", + "thing_id_link", + "lexicon_term", + "thing_aquifer_association", + "aquifer_system", +} + +STATS_VIEW = "ogc_internal_water_well_field_operations_stats" +FEATURE_VIEW = "ogc_internal_water_well_field_operations" + +LATEST_LOCATION_CTE = """ +SELECT DISTINCT ON (lta.thing_id) + lta.thing_id, + lta.location_id, + lta.effective_start +FROM location_thing_association AS lta +WHERE lta.effective_end IS NULL +ORDER BY lta.thing_id, lta.effective_start DESC +""".strip() + +# sensor_type values that mean "this well is logging by itself". Anything else +# deployed at a well (a barometer, a camera) is equipment, not a logger, and +# must not make has_datalogger true. +LOGGER_SENSOR_TYPES = ( + "'Data Logger'", + "'Pressure Transducer'", + "'DiverLink'", + "'Diver Cable'", +) + +# notes.note_type -> published column name. Order matches core/lexicon.json's +# note_type category. Two do not mechanically snake-case: OwnerComment has no +# separator to split on, and "Site Notes (legacy)" already says "notes" so it +# is not double-suffixed. +NOTE_TYPES = ( + ("Access", "access_notes"), + ("Directions", "directions_notes"), + ("Communication", "communication_notes"), + ("Construction", "construction_notes"), + ("Maintenance", "maintenance_notes"), + ("Historical", "historical_notes"), + ("General", "general_notes"), + ("Water", "water_notes"), + ("Water Quality", "water_quality_notes"), + ("Sampling Procedure", "sampling_procedure_notes"), + ("Coordinate", "coordinate_notes"), + ("OwnerComment", "owner_comment_notes"), + ("Site Notes (legacy)", "site_notes_legacy"), +) + +# Indexes the per-well lookups need. Each is created IF NOT EXISTS: none of +# these existed when this migration was written, but the thing_type one in +# particular is the sort of thing another migration may add first. +SUPPORTING_INDEXES = [ + ( + "ix_status_history_target_type_start", + "status_history (target_table, target_id, status_type, start_date DESC)", + ), + ( + "ix_permission_history_target_type_start", + "permission_history " + "(target_table, target_id, permission_type, start_date DESC)", + ), + ( + "ix_measuring_point_history_thing_start", + "measuring_point_history (thing_id, start_date DESC)", + ), + ( + "ix_monitoring_frequency_history_thing_start", + "monitoring_frequency_history (thing_id, start_date DESC)", + ), + ("ix_deployment_thing_id", "deployment (thing_id)"), + ("ix_thing_contact_association_thing_id", "thing_contact_association (thing_id)"), + ( + "ix_thing_contact_association_contact_id", + "thing_contact_association (contact_id)", + ), + ("ix_well_purpose_thing_id", "well_purpose (thing_id)"), + ("ix_well_casing_material_thing_id", "well_casing_material (thing_id)"), + ("ix_well_screen_thing_id", "well_screen (thing_id)"), + ("ix_thing_id_link_thing_id", "thing_id_link (thing_id)"), + ("ix_thing_thing_type", "thing (thing_type)"), + ( + "ix_thing_aquifer_association_thing_id", + "thing_aquifer_association (thing_id)", + ), + # No index on transducer_observation (deployment_id, ...): the existing + # uq_transducer_observation_deployment_parameter_datetime leads with + # deployment_id and already serves the per-deployment aggregate. +] + + +def _safe_relation_name(name: str) -> str: + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", name): + raise ValueError(f"Unsafe relation name: {name!r}") + return name + + +def _check_required_tables() -> None: + bind = op.get_bind() + inspector = inspect(bind) + existing_tables = set(inspector.get_table_names(schema="public")) + missing = REQUIRED_TABLES - existing_tables + if missing: + raise RuntimeError( + "Cannot create the water well field operations layer. " + f"Missing required tables: {', '.join(sorted(missing))}" + ) + + +def _create_stats_view() -> str: + """The expensive half: aggregates over the observation-scale tables.""" + safe_name = _safe_relation_name(STATS_VIEW) + return f""" + CREATE MATERIALIZED VIEW {safe_name} AS + WITH wells AS ( + SELECT t.id AS thing_id + FROM thing AS t + WHERE t.thing_type = 'water well' + ), + manual_obs AS ( + SELECT + fe.thing_id, + o.id AS observation_id, + o.observation_datetime, + (o.value - COALESCE(o.measuring_point_height, 0)) AS depth_to_water + FROM observation AS o + JOIN sample AS s ON s.id = o.sample_id + JOIN field_activity AS fa ON fa.id = s.field_activity_id + JOIN field_event AS fe ON fe.id = fa.field_event_id + JOIN wells AS w ON w.thing_id = fe.thing_id + WHERE + fa.activity_type = 'groundwater level' + AND o.value IS NOT NULL + AND o.observation_datetime IS NOT NULL + ), + manual_agg AS ( + SELECT + m.thing_id, + COUNT(*)::integer AS manual_water_level_count, + ( + MIN(m.observation_datetime) AT TIME ZONE 'UTC' + )::date AS manual_water_level_first_date, + ( + MAX(m.observation_datetime) AT TIME ZONE 'UTC' + )::date AS manual_water_level_last_date + FROM manual_obs AS m + GROUP BY m.thing_id + ), + manual_last AS ( + SELECT DISTINCT ON (m.thing_id) + m.thing_id, + m.depth_to_water AS last_depth_to_water_ft + FROM manual_obs AS m + ORDER BY m.thing_id, m.observation_datetime DESC, m.observation_id DESC + ), + chemistry_agg AS ( + SELECT + fe.thing_id, + COUNT(DISTINCT s.id)::integer AS chemistry_sample_count, + ( + MAX(s.sample_date) AT TIME ZONE 'UTC' + )::date AS chemistry_sample_last_date + FROM sample AS s + JOIN field_activity AS fa ON fa.id = s.field_activity_id + JOIN field_event AS fe ON fe.id = fa.field_event_id + JOIN wells AS w ON w.thing_id = fe.thing_id + WHERE fa.activity_type = 'water chemistry' + GROUP BY fe.thing_id + ), + field_event_agg AS ( + SELECT + fe.thing_id, + COUNT(*)::integer AS field_event_count, + ( + MAX(fe.event_date) AT TIME ZONE 'UTC' + )::date AS field_event_last_date + FROM field_event AS fe + JOIN wells AS w ON w.thing_id = fe.thing_id + GROUP BY fe.thing_id + ), + continuous_agg AS ( + SELECT + d.thing_id, + COUNT(*)::bigint AS continuous_reading_count, + MIN(tobs.observation_datetime) AS continuous_first_datetime, + MAX(tobs.observation_datetime) AS continuous_last_datetime + FROM transducer_observation AS tobs + JOIN deployment AS d ON d.id = tobs.deployment_id + JOIN wells AS w ON w.thing_id = d.thing_id + GROUP BY d.thing_id + ) + SELECT + w.thing_id, + COALESCE(ma.manual_water_level_count, 0) AS manual_water_level_count, + ma.manual_water_level_first_date, + ma.manual_water_level_last_date, + ml.last_depth_to_water_ft, + COALESCE(ca.chemistry_sample_count, 0) AS chemistry_sample_count, + ca.chemistry_sample_last_date, + COALESCE(fea.field_event_count, 0) AS field_event_count, + fea.field_event_last_date, + COALESCE(co.continuous_reading_count, 0) AS continuous_reading_count, + co.continuous_first_datetime, + co.continuous_last_datetime + FROM wells AS w + LEFT JOIN manual_agg AS ma ON ma.thing_id = w.thing_id + LEFT JOIN manual_last AS ml ON ml.thing_id = w.thing_id + LEFT JOIN chemistry_agg AS ca ON ca.thing_id = w.thing_id + LEFT JOIN field_event_agg AS fea ON fea.thing_id = w.thing_id + LEFT JOIN continuous_agg AS co ON co.thing_id = w.thing_id + """ + + +def _create_feature_view() -> str: + """The live half: everything a stale answer would misreport.""" + safe_name = _safe_relation_name(FEATURE_VIEW) + safe_stats = _safe_relation_name(STATS_VIEW) + logger_types = ", ".join(LOGGER_SENSOR_TYPES) + notes_columns_sql = ",\n ".join( + f"""( + SELECT string_agg(n.content, ' | ' ORDER BY n.id DESC) + FROM notes AS n + WHERE + n.target_table = 'thing' + AND n.target_id = t.id + AND n.note_type = '{note_type}' + ) AS {_safe_relation_name(column_name)}""" + for note_type, column_name in NOTE_TYPES + ) + return f""" + CREATE VIEW {safe_name} AS + WITH latest_location AS ( +{LATEST_LOCATION_CTE} + ), + current_status AS ( + SELECT DISTINCT ON (sh.target_id, sh.status_type) + sh.target_id AS thing_id, + sh.status_type, + sh.status_value, + sh.start_date, + sh.reason + FROM status_history AS sh + WHERE + sh.target_table = 'thing' + AND sh.start_date <= CURRENT_DATE + AND (sh.end_date IS NULL OR sh.end_date >= CURRENT_DATE) + ORDER BY sh.target_id, sh.status_type, sh.start_date DESC, sh.id DESC + ), + current_permission AS ( + SELECT DISTINCT ON (ph.target_id, ph.permission_type) + ph.target_id AS thing_id, + ph.permission_type, + ph.permission_allowed, + ph.contact_id + FROM permission_history AS ph + WHERE + ph.target_table = 'thing' + AND ph.start_date <= CURRENT_DATE + AND (ph.end_date IS NULL OR ph.end_date >= CURRENT_DATE) + ORDER BY + ph.target_id, ph.permission_type, ph.start_date DESC, ph.id DESC + ), + current_measuring_point AS ( + SELECT DISTINCT ON (mp.thing_id) + mp.thing_id, + mp.measuring_point_height, + mp.measuring_point_description, + mp.start_date + FROM measuring_point_history AS mp + WHERE + mp.start_date <= CURRENT_DATE + AND (mp.end_date IS NULL OR mp.end_date >= CURRENT_DATE) + ORDER BY mp.thing_id, mp.start_date DESC, mp.id DESC + ), + current_monitoring_frequency AS ( + SELECT DISTINCT ON (mf.thing_id) + mf.thing_id, + mf.monitoring_frequency, + mf.start_date + FROM monitoring_frequency_history AS mf + WHERE + mf.start_date <= CURRENT_DATE + AND (mf.end_date IS NULL OR mf.end_date >= CURRENT_DATE) + ORDER BY mf.thing_id, mf.start_date DESC, mf.id DESC + ), + installed_deployments AS ( + -- Every currently-installed sensor, not just loggers -- a camera + -- or barometer must not be invisible just because it cannot log + -- on its own. See logger_count below for the narrower has_datalogger + -- signal. + SELECT + d.id AS deployment_id, + d.thing_id, + d.installation_date, + d.recording_interval, + d.recording_interval_units, + d.hanging_point_description, + se.sensor_type, + se.model, + se.serial_no, + se.sensor_status + FROM deployment AS d + JOIN sensor AS se ON se.id = d.sensor_id + WHERE d.installation_date IS NOT NULL AND d.removal_date IS NULL + ), + installed_equipment AS ( + -- Aggregated, not DISTINCT ON: a well running more than one + -- current sensor lists all of them. All eight columns are ordered + -- by sensor_type, so position N in one list is the same + -- deployment as position N in the others -- which is why every + -- expression but sensor_type itself is wrapped in COALESCE(.., + -- ''): sensor_type is NOT NULL on sensor, but a deployment can + -- easily have a null model, recording_interval, etc. (a camera + -- has no recording interval), and plain string_agg silently + -- drops null inputs, shortening that one column's list and + -- breaking the position-N-means-the-same-deployment guarantee. + -- An empty segment between two ';' means "this sensor has no + -- value for this field," not "this sensor doesn't exist." + SELECT + idpl.thing_id, + string_agg( + idpl.sensor_type, '; ' ORDER BY idpl.sensor_type + ) AS sensor_type, + string_agg( + COALESCE(idpl.model, ''), '; ' ORDER BY idpl.sensor_type + ) AS model, + string_agg( + COALESCE(idpl.serial_no, ''), '; ' ORDER BY idpl.sensor_type + ) AS serial_no, + string_agg( + COALESCE(idpl.sensor_status, ''), '; ' ORDER BY idpl.sensor_type + ) AS sensor_status, + string_agg( + COALESCE(idpl.installation_date::text, ''), + '; ' ORDER BY idpl.sensor_type + ) AS installed_date, + string_agg( + COALESCE(idpl.recording_interval::text, ''), + '; ' ORDER BY idpl.sensor_type + ) AS recording_interval, + string_agg( + COALESCE(idpl.recording_interval_units, ''), + '; ' ORDER BY idpl.sensor_type + ) AS recording_interval_units, + string_agg( + COALESCE(idpl.hanging_point_description, ''), + '; ' ORDER BY idpl.sensor_type + ) AS hanging_point_desc + FROM installed_deployments AS idpl + GROUP BY idpl.thing_id + ), + logger_count AS ( + SELECT + idpl.thing_id, + COUNT(*)::integer AS datalogger_deployment_count + FROM installed_deployments AS idpl + WHERE idpl.sensor_type IN ({logger_types}) + GROUP BY idpl.thing_id + ), + screens AS ( + -- Full per-interval detail, not a min/max summary: a well with + -- more than one screen keeps them all rather than collapsing to + -- an overall depth range. All three ordered by screen_depth_top + -- (nulls last), and each wrapped in COALESCE(..., '') for the + -- same reason as installed_equipment above -- screen_depth_top, + -- screen_depth_bottom, and screen_description are all nullable + -- independently of each other, and plain string_agg would drop + -- a null and misalign the other two columns' positions. + SELECT + ws.thing_id, + COUNT(*)::integer AS screen_count, + string_agg( + COALESCE(ws.screen_depth_top::text, ''), '; ' + ORDER BY ws.screen_depth_top NULLS LAST + ) AS screen_depth_top, + string_agg( + COALESCE(ws.screen_depth_bottom::text, ''), '; ' + ORDER BY ws.screen_depth_top NULLS LAST + ) AS screen_depth_bottom, + string_agg( + COALESCE(ws.screen_description, ''), '; ' + ORDER BY ws.screen_depth_top NULLS LAST + ) AS screen_description + FROM well_screen AS ws + GROUP BY ws.thing_id + ), + thing_contacts AS ( + SELECT + tca.thing_id, + c.id AS contact_id, + c.name, + c.organization, + c.role, + c.contact_type + FROM thing_contact_association AS tca + JOIN contact AS c ON c.id = tca.contact_id + ), + contact_agg AS ( + SELECT + tc.thing_id, + COUNT(DISTINCT tc.contact_id)::integer AS contact_count, + string_agg( + DISTINCT tc.name, ', ' ORDER BY tc.name + ) AS contact_names + FROM thing_contacts AS tc + GROUP BY tc.thing_id + ), + primary_contact AS ( + -- Prefer a contact recorded as Primary, but fall back to any + -- contact on the well rather than publishing a blank name next to + -- a non-zero contact_count. primary_contact_type says which case + -- this is, so the fallback is visible rather than implied. + SELECT DISTINCT ON (tc.thing_id) + tc.thing_id, + tc.contact_id, + tc.name, + tc.organization, + tc.role, + tc.contact_type + FROM thing_contacts AS tc + ORDER BY + tc.thing_id, + (tc.contact_type = 'Primary') DESC NULLS LAST, + tc.contact_id + ) + SELECT + t.id AS id, + t.name, + 'water well'::text AS station_type, + t.release_status, + ( + SELECT string_agg( + DISTINCT + COALESCE(tl.alternate_organization, 'unknown') + || ': ' + || tl.alternate_id, + ', ' + ORDER BY + COALESCE(tl.alternate_organization, 'unknown') + || ': ' + || tl.alternate_id + ) + FROM thing_id_link AS tl + WHERE tl.thing_id = t.id + ) AS alternate_ids, + -- Decimal degrees alongside the geometry. A crew types these into + -- a handheld GPS or reads them over the radio; the geometry column + -- is for the map, and a CSV export of this layer drops it. + ST_Y(l.point) AS latitude, + ST_X(l.point) AS longitude, + l.elevation, + + -- Construction, same column names as the thing-type views. + t.well_depth, + t.hole_depth, + t.well_casing_diameter, + t.well_casing_depth, + t.well_completion_date, + t.well_driller_name, + t.well_construction_method, + t.well_pump_type, + t.well_pump_depth, + t.formation_completion_code, + ( + SELECT lt.definition + FROM lexicon_term AS lt + WHERE lt.term = t.formation_completion_code + ) AS formation_completion_description, + ( + SELECT string_agg(DISTINCT wp.purpose, ', ' ORDER BY wp.purpose) + FROM well_purpose AS wp + WHERE wp.thing_id = t.id + ) AS well_purpose, + ( + SELECT string_agg( + DISTINCT wcm.material, ', ' ORDER BY wcm.material + ) + FROM well_casing_material AS wcm + WHERE wcm.thing_id = t.id + ) AS well_casing_material, + ( + SELECT string_agg(DISTINCT asys.name, ', ' ORDER BY asys.name) + FROM thing_aquifer_association AS taa + JOIN aquifer_system AS asys ON asys.id = taa.aquifer_system_id + WHERE taa.thing_id = t.id + ) AS aquifer_system_name, + scr.screen_count, + scr.screen_depth_top, + scr.screen_depth_bottom, + scr.screen_description, + + -- Measuring point, current record. + cmp.measuring_point_height AS mp_height, + cmp.measuring_point_description AS mp_description, + + -- Status, current record per status type. + well_st.status_value AS well_status, + mon_st.status_value AS monitoring_status, + open_st.status_value AS open_status, + dl_st.status_value AS datalogger_suitability_status, + + wl_perm.permission_allowed AS may_measure_water_level, + chem_perm.permission_allowed AS may_sample_water_chemistry, + dl_perm.permission_allowed AS may_install_datalogger, + CASE WHEN wl_perm.permission_allowed IS TRUE THEN granter.name END AS permission_granted_by, + + -- Monitoring programme. + cmf.monitoring_frequency, + grp.group_names, + grp.group_types, + + -- Manual water levels (from the stats matview). + COALESCE(st.manual_water_level_count, 0) AS manual_water_level_count, + st.manual_water_level_first_date, + st.manual_water_level_last_date, + ( + CURRENT_DATE - st.manual_water_level_last_date + ) AS days_since_manual_water_level, + st.last_depth_to_water_ft, + + -- Chemistry sampling (from the stats matview). + COALESCE(st.chemistry_sample_count, 0) AS chemistry_sample_count, + st.chemistry_sample_last_date, + ( + CURRENT_DATE - st.chemistry_sample_last_date + ) AS days_since_chemistry_sample, + + -- Field visits (from the stats matview). + COALESCE(st.field_event_count, 0) AS field_event_count, + st.field_event_last_date AS date_last_visited, + + -- Currently installed equipment, any sensor type -- see + -- installed_equipment above. has_datalogger/ + -- datalogger_deployment_count stay logger-scoped. + (lc.thing_id IS NOT NULL) AS has_datalogger, + COALESCE(lc.datalogger_deployment_count, 0) + AS datalogger_deployment_count, + ie.sensor_type, + ie.model, + ie.serial_no, + ie.sensor_status, + ie.installed_date, + ie.recording_interval, + ie.recording_interval_units, + ie.hanging_point_desc, + COALESCE(st.continuous_reading_count, 0) AS continuous_reading_count, + st.continuous_first_datetime, + st.continuous_last_datetime, + ( + CURRENT_DATE + - (st.continuous_last_datetime AT TIME ZONE 'UTC')::date + ) AS days_since_continuous_reading, + + -- Contacts. + COALESCE(cag.contact_count, 0) AS contact_count, + pc.name AS primary_contact_name, + pc.organization AS primary_contact_organization, + pc.role AS primary_contact_role, + pc.contact_type AS primary_contact_type, + ( + SELECT p.phone_number + FROM phone AS p + WHERE p.contact_id = pc.contact_id + ORDER BY p.id + LIMIT 1 + ) AS primary_contact_phone, + ( + SELECT e.email + FROM email AS e + WHERE e.contact_id = pc.contact_id + ORDER BY e.id + LIMIT 1 + ) AS primary_contact_email, + cag.contact_names, + + -- Notes, one column per note_type. Separator is ' | ' rather than + -- ', ' because the content is free text and routinely contains + -- commas. + {notes_columns_sql}, + + l.point + FROM thing AS t + JOIN latest_location AS ll ON ll.thing_id = t.id + JOIN location AS l ON l.id = ll.location_id + LEFT JOIN {safe_stats} AS st ON st.thing_id = t.id + LEFT JOIN current_status AS well_st + ON well_st.thing_id = t.id AND well_st.status_type = 'Well Status' + LEFT JOIN current_status AS mon_st + ON mon_st.thing_id = t.id + AND mon_st.status_type = 'Monitoring Status' + LEFT JOIN current_status AS open_st + ON open_st.thing_id = t.id AND open_st.status_type = 'Open Status' + LEFT JOIN current_status AS dl_st + ON dl_st.thing_id = t.id + AND dl_st.status_type = 'Datalogger Suitability Status' + LEFT JOIN current_permission AS wl_perm + ON wl_perm.thing_id = t.id + AND wl_perm.permission_type = 'Water Level Sample' + LEFT JOIN current_permission AS chem_perm + ON chem_perm.thing_id = t.id + AND chem_perm.permission_type = 'Water Chemistry Sample' + LEFT JOIN current_permission AS dl_perm + ON dl_perm.thing_id = t.id + AND dl_perm.permission_type = 'Datalogger Installation' + LEFT JOIN contact AS granter ON granter.id = wl_perm.contact_id + LEFT JOIN current_measuring_point AS cmp ON cmp.thing_id = t.id + LEFT JOIN current_monitoring_frequency AS cmf ON cmf.thing_id = t.id + LEFT JOIN installed_equipment AS ie ON ie.thing_id = t.id + LEFT JOIN logger_count AS lc ON lc.thing_id = t.id + LEFT JOIN contact_agg AS cag ON cag.thing_id = t.id + LEFT JOIN primary_contact AS pc ON pc.thing_id = t.id + LEFT JOIN screens AS scr ON scr.thing_id = t.id + LEFT JOIN LATERAL ( + -- group_thing_association has no unique constraint on + -- (group_id, thing_id), so DISTINCT before aggregating. Both + -- strings are ordered by the same group name so they stay + -- index-aligned with each other. + SELECT + string_agg(dm.group_name, ', ' ORDER BY dm.group_name) + AS group_names, + string_agg(dm.group_type, ', ' ORDER BY dm.group_name) + AS group_types + FROM ( + SELECT DISTINCT + g.id AS group_id, + g.name AS group_name, + g.group_type + FROM group_thing_association AS gta + JOIN "group" AS g ON g.id = gta.group_id + WHERE gta.thing_id = t.id + ) AS dm + ) AS grp ON TRUE + WHERE t.thing_type = 'water well' + """ + + +def upgrade() -> None: + _check_required_tables() + + safe_stats = _safe_relation_name(STATS_VIEW) + safe_feature = _safe_relation_name(FEATURE_VIEW) + + # The feature view depends on the stats matview, so it is dropped first and + # created last. + op.execute(text(f"DROP VIEW IF EXISTS {safe_feature}")) + op.execute(text(f"DROP MATERIALIZED VIEW IF EXISTS {safe_stats}")) + + op.execute(text(_create_stats_view())) + # Unique index required for REFRESH MATERIALIZED VIEW CONCURRENTLY. + op.execute( + text( + f"CREATE UNIQUE INDEX ix_{safe_stats}_thing_id " + f"ON {safe_stats} (thing_id)" + ) + ) + op.execute(text(_create_feature_view())) + + for index_name, definition in SUPPORTING_INDEXES: + op.execute( + text( + f"CREATE INDEX IF NOT EXISTS {_safe_relation_name(index_name)} ON {definition}" + ) + ) + + +def downgrade() -> None: + safe_stats = _safe_relation_name(STATS_VIEW) + safe_feature = _safe_relation_name(FEATURE_VIEW) + + op.execute(text(f"DROP VIEW IF EXISTS {safe_feature}")) + op.execute(text(f"DROP MATERIALIZED VIEW IF EXISTS {safe_stats}")) + + for index_name, _definition in SUPPORTING_INDEXES: + op.execute(text(f"DROP INDEX IF EXISTS {_safe_relation_name(index_name)}")) diff --git a/core/ogc-field-descriptions.yml b/core/ogc-field-descriptions.yml index c462b6dd..110efe94 100644 --- a/core/ogc-field-descriptions.yml +++ b/core/ogc-field-descriptions.yml @@ -2099,6 +2099,464 @@ minor_chemistry_wells: recorded them. +# water well field operations. Internal mount only -- the layer carries landowner +# contact details and staff-written access notes, so it has no public twin. +# See docs/water-well-field-operations-layer.md. +water_well_field_operations: + station_type: + title: Feature type + description: >- + Controlled-vocabulary type of the monitoring point. Named station_type + rather than thing_type in this layer to match the naming already + established on the public thing-type views; every row on this layer is + a water well, so the value is always the same literal. + enum-lexicon: thing_type + latitude: + title: Latitude + description: >- + Latitude of the well in decimal degrees on the WGS 84 datum, the same + coordinate the feature geometry carries. Published as a plain number so + it survives a spreadsheet or CSV export of this layer, which drops the + geometry, and so a crew can read it into a handheld GPS. + x-ogc-unit: https://qudt.org/vocab/unit/DEG + x-ogc-unitLang: QUDT + longitude: + title: Longitude + description: >- + Longitude of the well in decimal degrees on the WGS 84 datum, negative + west of Greenwich, the same coordinate the feature geometry carries. + Published as a plain number for the same reason as the latitude. + x-ogc-unit: https://qudt.org/vocab/unit/DEG + x-ogc-unitLang: QUDT + alternate_ids: + title: Alternate identifiers + description: >- + Identifiers other organisations use for this well, each written as + organisation followed by the identifier and separated by commas. Empty + where no cross-reference has been recorded. + well_purpose: + title: Well purposes + description: >- + What the well is used for, from the controlled well-purpose vocabulary -- + domestic, irrigation, monitoring, and so on. A well may serve more than + one purpose, in which case the purposes are separated by commas. + well_casing_material: + title: Casing materials + description: >- + Materials the well casing is made of, from the controlled casing-material + vocabulary. Separated by commas where a well has more than one. + formation_completion_description: + title: Completion formation, description + description: >- + Human-readable definition of the geologic formation named by + formation_completion_code, looked up from the lexicon. Null where the + code has no matching lexicon entry. + aquifer_system_name: + title: Aquifer system(s) + description: >- + Name(s) of the aquifer system(s) the well is associated with, separated + by commas where it is associated with more than one; order carries no + meaning beyond de-duplication. + screen_count: + title: Screened intervals + description: >- + Number of discrete screened intervals recorded for the well. Zero means + no screen record exists, which is not the same as an unscreened well. + screen_depth_top: + title: Screen top depth(s) + description: >- + Depth to the top of each screened interval, below ground surface, in + feet. A well with more than one screen lists each interval's top + depth, semicolon-separated, ordered shallowest first. screen_depth_top, + screen_depth_bottom, and screen_description are ordered the same way + and COALESCEd to an empty string where a value is missing, so position + N in one list is always the same interval as position N in the + others -- never a dropped position from a different one. + screen_depth_bottom: + title: Screen bottom depth(s) + description: >- + Depth to the bottom of each screened interval, below ground surface, + in feet. See screen_depth_top for how multiple intervals are listed + and ordered. + screen_description: + title: Screen description(s) + description: >- + Free-text description of each screened interval. See screen_depth_top + for how multiple intervals are listed and ordered. + mp_height: + title: Measuring point height + description: >- + Height of the current measuring point above the ground surface. Manual + depth-to-water readings are taken from this point, so it is what converts + a raw reading to a depth below ground surface. + x-ogc-unit: https://qudt.org/vocab/unit/FT + x-ogc-unitLang: QUDT + mp_description: + title: Measuring point description + description: >- + Where on the wellhead the measuring point is, as written for the person + standing at the well -- for example the north side of the casing at the + top of the PVC. The single most useful string on this layer for a crew + about to take a reading. + well_status: + title: Well status + description: >- + Current condition of the well itself. Null where no status is on record. + enum: + - Active, pumping well + - Inactive, exists but not used + - Destroyed, exists but not usable + - Abandoned + monitoring_status: + title: Monitoring status + description: >- + Whether the well is currently on a monitoring schedule. Null where no + status is on record. + enum: + - Currently monitored + - Not currently monitored + open_status: + title: Open status + description: >- + Whether the well is physically open to a sounder or tape. A closed well + cannot be measured by hand however good the access. Null where no status + is on record. + enum: + - Open + - Open (unequipped) + - Closed + datalogger_suitability_status: + title: Datalogger suitability + description: >- + Whether the well is physically suitable for a logger installation. This + is a judgement about the well, separate from whether the landowner has + given permission to install one. Null where no status is on record. + enum: + - Datalogger can be installed + - Datalogger cannot be installed + may_measure_water_level: + title: May measure water level + description: >- + Whether a current permission allows a manual water-level measurement at + this well. True means a permission on record allows it; false means a + permission on record refuses it; null means no permission has been + recorded either way, which is not the same as a refusal -- it means + nobody has asked the landowner yet. + may_sample_water_chemistry: + title: May sample water chemistry + description: >- + Whether a current permission allows a water-chemistry sample to be + collected at this well. True allows, false refuses, null means no + permission is on record -- not a refusal. + may_install_datalogger: + title: May install datalogger + description: >- + Whether a current permission allows a logger to be installed in this + well. True allows, false refuses, null means no permission is on record + -- not a refusal. Separate from datalogger suitability, which says + whether the well could take one. + permission_granted_by: + title: Permission granted by + description: >- + Name of the contact who granted the current water-level permission, so a + crew can say who agreed to the visit. Null where no water-level + permission is on record. + monitoring_frequency: + title: Monitoring frequency + description: >- + How often the well is scheduled to be visited under its current + monitoring plan. Null where no frequency is on record. + enum-lexicon: monitoring_frequency + group_names: + title: Groups + description: >- + Names of the monitoring plans, geographic areas and historical groupings + the well belongs to, separated by commas. A well often sits in several. + group_types: + title: Group types + description: >- + Type of each group named in the groups column, in the same order and + separated by commas -- a monitoring plan, a geographic area, or a + historical grouping. + manual_water_level_count: + title: Manual water level readings + description: >- + Number of hand-measured groundwater level readings on record for the + well. Continuous readings from an installed logger are counted separately. + manual_water_level_first_date: + title: First manual reading + description: >- + Date of the earliest hand-measured groundwater level on record, as a UTC + calendar date. + manual_water_level_last_date: + title: Last manual reading + description: >- + Date of the most recent hand-measured groundwater level on record, as a + UTC calendar date. + days_since_manual_water_level: + title: Days since last manual reading + description: >- + Days elapsed since the most recent hand-measured groundwater level. + Computed against today, not against the last refresh, so it is current on + every request. Null where the well has never been measured by hand. + last_depth_to_water_ft: + title: Latest depth to water + description: >- + Most recent hand-measured depth to water, below ground surface -- the + measured value minus the height of the measuring point above ground, with + a reading that has no recorded measuring-point height treated as taken at + ground level. The same convention as the water-level summary and water + column layers. + x-ogc-unit: https://qudt.org/vocab/unit/FT + x-ogc-unitLang: QUDT + chemistry_sample_count: + title: Chemistry samples + description: >- + Number of water-chemistry samples collected at the well. Counts samples, + not the individual analyte results measured from them. + chemistry_sample_last_date: + title: Last chemistry sample + description: >- + Date the most recent water-chemistry sample was collected, as a UTC + calendar date. + days_since_chemistry_sample: + title: Days since last chemistry sample + description: >- + Days elapsed since the most recent water-chemistry sample. Computed + against today. Null where the well has never been sampled. + field_event_count: + title: Field visits + description: >- + Number of recorded visits to the well. Broader than either measurement + count: a visit that produced no reading, because the gate was locked or + the well was dry, is still a visit. + date_last_visited: + title: Last field visit + description: >- + Date of the most recent recorded visit, as a UTC calendar date. A gap + between this and the last reading dates means someone went and came back + with nothing. + has_datalogger: + title: Has a datalogger + description: >- + Whether a logger -- a data logger, pressure transducer, DiverLink, or + diver cable -- is currently installed in the well, regardless of what + other equipment (a camera, a weather station) may also be deployed + there. See sensor_type for every currently-installed sensor, logger or + not. + datalogger_deployment_count: + title: Current logger deployments + description: >- + Number of logger deployments currently open at the well -- data + loggers, pressure transducers, DiverLinks, and diver cables only. + Normally zero or one. Other currently-installed equipment is not + counted here; see sensor_type for the full list. + sensor_type: + title: Currently installed sensor type(s) + description: >- + Type(s) of sensor currently deployed at the well (a deployment with an + installation date and no removal date), from a controlled vocabulary, + not limited to data loggers -- a camera, weather station, or other + equipment counts too. A well running more than one sensor at once + lists all of them, semicolon-separated, ordered the same way as model, + serial_no, sensor_status, installed_date, recording_interval, + recording_interval_units, and hanging_point_desc, so position N in one + list is always the same deployment as position N in the others -- an + empty segment in one of those other fields (e.g. "; 15" for a well + with two sensors) means that particular sensor has no value recorded + for that field, not that it doesn't exist. See has_datalogger for the + narrower "is a logger installed" question. + model: + title: Currently installed sensor model(s) + description: >- + Model of each currently deployed sensor. See sensor_type for how + multiple concurrent deployments are listed and ordered. + serial_no: + title: Currently installed sensor serial number(s) + description: >- + Serial number of each currently deployed sensor, for matching the + instrument in hand against the record. See sensor_type for how + multiple concurrent deployments are listed and ordered. + sensor_status: + title: Currently installed sensor status(es) + description: >- + Equipment status of each currently deployed sensor, such as In Service + or In Repair, from a controlled vocabulary. See sensor_type for how + multiple concurrent deployments are listed and ordered. + installed_date: + title: Currently installed sensor installation date(s) + description: >- + Date each currently deployed sensor was installed. See sensor_type for + how multiple concurrent deployments are listed and ordered. + recording_interval: + title: Currently installed sensor recording interval(s) + description: >- + Recording interval of each currently deployed sensor. See sensor_type + for how multiple concurrent deployments are listed and ordered. + recording_interval_units: + title: Currently installed sensor recording interval unit(s) + description: >- + Unit of recording_interval for each currently deployed sensor. See + sensor_type for how multiple concurrent deployments are listed and + ordered. + hanging_point_desc: + title: Currently installed sensor hanging point description(s) + description: >- + Where each currently deployed sensor's cable is secured at the + wellhead, as written for the person about to pull it. See sensor_type + for how multiple concurrent deployments are listed and ordered. Null + where a deployment did not record one. + continuous_reading_count: + title: Continuous readings + description: >- + Number of readings logged automatically by instruments deployed in this + well, across all of its deployments. Separate from the hand-measured + count. + continuous_first_datetime: + title: First continuous reading + description: >- + Timestamp of the earliest automatically logged reading held for the well. + continuous_last_datetime: + title: Last continuous reading + description: >- + Timestamp of the most recent automatically logged reading held for the + well. A well whose logger has stopped reporting shows an old value here + while still reading as having a logger installed. + days_since_continuous_reading: + title: Days since last continuous reading + description: >- + Days elapsed since the most recent automatically logged reading, computed + against today. Null where the well has no logged readings at all. Rising + values on an instrumented well mean the logger needs attention. + contact_count: + title: Contacts + description: >- + Number of contacts associated with the well -- landowners, operators, and + anyone else recorded as a point of contact for it. + primary_contact_name: + title: Primary contact + description: >- + Name of the contact to approach about this well. This is the contact + recorded as primary where one exists; where none does, it is another + contact on the well, and the primary contact type column says which case + applies. + primary_contact_organization: + title: Primary contact organisation + description: Organisation the primary contact belongs to, where recorded. + primary_contact_role: + title: Primary contact role + description: >- + Role the primary contact holds in relation to the well, such as owner, + manager, or operator. + enum-lexicon: role + primary_contact_type: + title: Primary contact type + description: >- + Whether the contact shown is the one recorded as primary for the well or + a fallback used because no primary contact is on record. Read it before + relying on the contact details beside it. + enum-lexicon: contact_type + primary_contact_phone: + title: Primary contact phone + description: >- + Phone number for the primary contact. Where several are on record this is + the first one recorded, so a crew that cannot reach it should check the + full contact record. + primary_contact_email: + title: Primary contact email + description: >- + Email address for the primary contact. Where several are on record this + is the first one recorded. + contact_names: + title: All contacts + description: >- + Names of every contact associated with the well, separated by commas, so + a crew can see who else can be approached if the primary contact cannot + be reached. + access_notes: + title: Access notes + description: >- + Staff-written instructions for getting to the well and getting into it -- + gates, keys, locks, livestock, who to call first. Where several notes + exist they are separated by vertical bars, most recently recorded first. + directions_notes: + title: Directions + description: >- + Staff-written directions for finding the well on the ground, which + coordinates alone do not give. Where several notes exist they are + separated by vertical bars, most recently recorded first. + communication_notes: + title: Communication notes + description: >- + Staff-written record of communications about the well -- calls, + emails, or conversations worth keeping with the record. Where several + notes exist they are separated by vertical bars, most recently + recorded first. + construction_notes: + title: Construction notes + description: >- + Staff-written notes about how the well was built, beyond what the + construction columns capture. Where several notes exist they are + separated by vertical bars, most recently recorded first. + maintenance_notes: + title: Maintenance notes + description: >- + Staff-written notes about repairs or upkeep performed at the well. + Where several notes exist they are separated by vertical bars, most + recently recorded first. + historical_notes: + title: Historical notes + description: >- + Staff-written notes about the well's history that do not fit elsewhere + on the record. Where several notes exist they are separated by + vertical bars, most recently recorded first. + general_notes: + title: General notes + description: >- + Staff-written notes that do not fall into any of this layer's other + note categories. Where several notes exist they are separated by + vertical bars, most recently recorded first. + water_notes: + title: Water notes + description: >- + Staff-written notes about the well's water, distinct from the + structured water-chemistry and water-level records elsewhere in the + catalogue. Where several notes exist they are separated by vertical + bars, most recently recorded first. + water_quality_notes: + title: Water quality notes + description: >- + Staff-written notes about water quality observed at the well, distinct + from the structured chemistry analyte results elsewhere in the + catalogue. Where several notes exist they are separated by vertical + bars, most recently recorded first. + sampling_procedure_notes: + title: Sampling procedure notes + description: >- + Staff-written notes about how to sample this particular well -- + anything a standard procedure does not already cover. Where several + notes exist they are separated by vertical bars, most recently + recorded first. + coordinate_notes: + title: Coordinate notes + description: >- + Staff-written notes about how the well's coordinates were obtained, or + any doubt about their accuracy. Where several notes exist they are + separated by vertical bars, most recently recorded first. + owner_comment_notes: + title: Owner comment notes + description: >- + Comments attributed to the well's owner, as recorded by staff. Where + several notes exist they are separated by vertical bars, most recently + recorded first. + site_notes_legacy: + title: Site notes (legacy) + description: >- + Free-text site notes carried over from the legacy NM_Aquifer record, + kept for continuity rather than actively maintained. Where several + exist they are separated by vertical bars, most recently recorded + first. + # EDR collections. Keys here are parameter names read out of the data, not # column names -- ogc_waterlevels stamps a single literal, while # ogc_water_chemistry carries the analyte text exactly as the laboratory diff --git a/core/pygeoapi-config-internal.yml b/core/pygeoapi-config-internal.yml index ca318979..4b2e8fdf 100644 --- a/core/pygeoapi-config-internal.yml +++ b/core/pygeoapi-config-internal.yml @@ -77,6 +77,53 @@ resources: table: ogc_internal_locations geom_field: point + water_well_field_operations: + type: collection + title: Water Well Field Operations + description: >- + One record per water well, assembled for the crews who visit them. Where + every other well layer answers a scientific question, this one answers an + operational one: whether a crew may go to a well, what they are permitted + to do when they get there, and whether it is overdue. It pairs the well's + construction, completion formation and aquifer system with its + measuring-point detail, current well, monitoring, open and + datalogger-suitability status, the standing landowner permissions for + water-level measurement, chemistry sampling and logger installation, every + sensor currently deployed at it (not just data loggers), how long it has + been since it was last measured and last sampled, and the landowner + contact and staff-written notes -- access, directions, maintenance and + the rest -- needed to reach it. Published on the internal mount only, + because it carries personal contact details for landowners and operators + alongside staff-written access instructions. + keywords: + [ + water-wells, + field-operations, + landowner-contacts, + access-permissions, + monitoring-status, + currently-installed-equipment, + aquifer-system, + internal, + ] + extents: + spatial: + bbox: [-109.05, 31.33, -103.00, 37.00] + crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 + providers: + - type: feature + name: core.feature_provider.DescribedPostgreSQLProvider + data: + host: {postgres_host} + port: {postgres_port} + dbname: {postgres_db} + user: {postgres_user} + password: {postgres_password_env} + search_path: [public] + id_field: id + table: ogc_internal_water_well_field_operations + geom_field: point + latest_depth_to_water_wells: type: collection title: Latest Depth to Water (Water Wells) diff --git a/docs/internal-ogc-desktop-gis.md b/docs/internal-ogc-desktop-gis.md index 7efa6640..045bebd3 100644 --- a/docs/internal-ogc-desktop-gis.md +++ b/docs/internal-ogc-desktop-gis.md @@ -5,6 +5,16 @@ collections. It is gated by `core/internal_ogc_auth.py`, an ASGI middleware that runs in front of the raw Starlette Mount — FastAPI's `Depends()` machinery never sees these requests, so none of the `*_dependency` role parameters apply here. +> **An internal credential dispenses personal information.** The +> `water_well_field_operations` collection publishes landowner and operator +> contact details — name, organisation, role, phone number, email address — +> alongside staff-written access notes that routinely contain gate codes and +> names. Every credential accepted by this mount reaches it: an Authentik JWT +> carrying `OGCInternal`, and the static keys in `INTERNAL_OGC_API_KEYS`, which +> are shared secrets revocable only by redeploy. Any credential path added +> later inherits the same reach. Issue keys with that in mind, and read +> `docs/water-well-field-operations-layer.md` before widening who holds one. + ## Why there are static API keys at all The mount originally accepted only `Authorization: Bearer `. diff --git a/docs/ogc_conventions.md b/docs/ogc_conventions.md index 7a0d4569..d929279b 100644 --- a/docs/ogc_conventions.md +++ b/docs/ogc_conventions.md @@ -156,6 +156,33 @@ human-readable display name). | `locations` | N/A | Locations | N/A | N/A | Conforms. Hidden from public catalog — scope decision, not a naming fix | | `avg_tds_wells` | `water_well_average_total_dissolved_solids` | Average Total Dissolved Solids (Water Wells) | N/A | 2 | Title implemented. `avg` abbreviation remains in the ID; hidden from public catalog but the rule still applies for internal consumers | | `latest_depth_to_water_wells` | N/A | Latest Depth to Water (Water Wells) | N/A | N/A | Conforms. Hidden from public catalog — audit flagged it as redundant with `water_well_summary`, not a naming defect | +| `water_well_field_operations` | N/A | Water Well Field Operations | N/A | N/A | Conforms. Internal-only with no public form at all — it publishes landowner contact details and staff-written access notes | + +## Current-record semantics + +Layers that read a history table (`status_history`, `permission_history`, +`measuring_point_history`, `monitoring_frequency_history`) disagree about what +"current" means, and the disagreement is deliberate. + +`ogc_actively_monitored_wells` takes the row with the greatest `start_date` and +**ignores `end_date`**, so a monitoring status closed in 2019 still reads as +current there. That is tolerable on a summary layer, where the question is +roughly "is this well in the programme". + +`ogc_internal_water_well_field_operations` honours the window: + +```sql +WHERE h.start_date <= CURRENT_DATE + AND (h.end_date IS NULL OR h.end_date >= CURRENT_DATE) +ORDER BY h.start_date DESC, h.id DESC +``` + +Its columns answer "may a crew do this today", and a permission that ran out +last month must not read as a permission. A row whose window has not opened yet +is not current either, and reads null. + +New layers should follow the second form. The first is kept only because +changing it would move rows in a published layer. ## References diff --git a/docs/water-well-field-operations-columns.md b/docs/water-well-field-operations-columns.md new file mode 100644 index 00000000..d7f4ee03 --- /dev/null +++ b/docs/water-well-field-operations-columns.md @@ -0,0 +1,105 @@ +# water well field operations — column sources + +Every column published by `ogc_internal_water_well_field_operations`, in view +order, and where its value comes from. Columns marked *(stats)* are read from +the `ogc_internal_water_well_field_operations_stats` materialized view and are as +fresh as its last refresh; everything else is joined live on each request. + +"Current record" means the history row satisfying +`start_date <= CURRENT_DATE AND (end_date IS NULL OR end_date >= CURRENT_DATE)`, +latest `start_date` first. "Most recent association" means the +`location_thing_association` row with no `effective_end`, latest +`effective_start` first. + +Generated from the view definition in +`alembic/versions/e1f2a3b4c5d6_add_water_well_field_operations_layer.py`. Field +prose lives in `core/ogc-field-descriptions.yml`; the design rationale lives in +`docs/water-well-field-operations-layer.md`. + +| Column | Source | +| --- | --- | +| `id` | `thing.id` | +| `name` | `thing.name` | +| `station_type` | Literal `'water well'` — the view's row filter. Named `station_type` rather than `thing_type` to match the naming already established on the public thing-type views | +| `release_status` | `thing.release_status` | +| `alternate_ids` | `thing_id_link.alternate_organization` + `.alternate_id`, comma-joined | +| `latitude` | `ST_Y(location.point)` — decimal degrees, WGS 84 | +| `longitude` | `ST_X(location.point)` — decimal degrees, WGS 84 | +| `elevation` | `location.elevation`, most recent association | +| `well_depth` | `thing.well_depth` | +| `hole_depth` | `thing.hole_depth` | +| `well_casing_diameter` | `thing.well_casing_diameter` | +| `well_casing_depth` | `thing.well_casing_depth` | +| `well_completion_date` | `thing.well_completion_date` | +| `well_driller_name` | `thing.well_driller_name` | +| `well_construction_method` | `thing.well_construction_method` | +| `well_pump_type` | `thing.well_pump_type` | +| `well_pump_depth` | `thing.well_pump_depth` | +| `formation_completion_code` | `thing.formation_completion_code` | +| `formation_completion_description` | `lexicon_term.definition` where `term = thing.formation_completion_code`. Neither this nor `aquifer_system_name` below is date-windowed — neither source table carries a `start_date`/`end_date` | +| `well_purpose` | `well_purpose.purpose`, comma-joined | +| `well_casing_material` | `well_casing_material.material`, comma-joined | +| `aquifer_system_name` | `aquifer_system.name` via `thing_aquifer_association`, comma-joined | +| `screen_count` | `count(well_screen)` | +| `screen_depth_top` | `well_screen.screen_depth_top`, every interval, semicolon-joined, ordered shallowest first, `COALESCE(..., '')` for the same reason as the equipment columns | +| `screen_depth_bottom` | `well_screen.screen_depth_bottom`, same intervals, same order, same `COALESCE(..., '')` treatment | +| `screen_description` | `well_screen.screen_description`, same intervals, same order, same `COALESCE(..., '')` treatment | +| `mp_height` | `measuring_point_history.measuring_point_height`, current record | +| `mp_description` | `measuring_point_history.measuring_point_description`, current record | +| `well_status` | `status_history.status_value` where `status_type = 'Well Status'`, current record | +| `monitoring_status` | `status_history.status_value` where `status_type = 'Monitoring Status'`, current record | +| `open_status` | `status_history.status_value` where `status_type = 'Open Status'`, current record | +| `datalogger_suitability_status` | `status_history.status_value` where `status_type = 'Datalogger Suitability Status'`, current record | +| `may_measure_water_level` | `permission_history.permission_allowed` where `permission_type = 'Water Level Sample'`, current record | +| `may_sample_water_chemistry` | `permission_history.permission_allowed` where `permission_type = 'Water Chemistry Sample'`, current record | +| `may_install_datalogger` | `permission_history.permission_allowed` where `permission_type = 'Datalogger Installation'`, current record | +| `permission_granted_by` | `contact.name` via `permission_history.contact_id` on the current water-level grant | +| `monitoring_frequency` | `monitoring_frequency_history.monitoring_frequency`, current record | +| `group_names` | `group.name` via `group_thing_association`, comma-joined | +| `group_types` | `group.group_type`, same order as `group_names` | +| `manual_water_level_count` | `count(observation)` via `sample` → `field_activity` → `field_event`, `activity_type = 'groundwater level'` *(stats)* | +| `manual_water_level_first_date` | `min(observation.observation_datetime)`, UTC date, same chain *(stats)* | +| `manual_water_level_last_date` | `max(observation.observation_datetime)`, UTC date, same chain *(stats)* | +| `days_since_manual_water_level` | `CURRENT_DATE - manual_water_level_last_date` | +| `last_depth_to_water_ft` | `observation.value - COALESCE(observation.measuring_point_height, 0)` on the latest reading *(stats)* | +| `chemistry_sample_count` | `count(DISTINCT sample.id)`, `activity_type = 'water chemistry'` *(stats)* | +| `chemistry_sample_last_date` | `max(sample.sample_date)`, UTC date, same filter *(stats)* | +| `days_since_chemistry_sample` | `CURRENT_DATE - chemistry_sample_last_date` | +| `field_event_count` | `count(field_event)` for the well *(stats)* | +| `date_last_visited` | `max(field_event.event_date)`, UTC date *(stats, column named `field_event_last_date` there)* | +| `has_datalogger` | `true` when a currently-installed deployment exists whose `sensor.sensor_type` is Data Logger / Pressure Transducer / DiverLink / Diver Cable. Stays logger-scoped even though the columns below do not | +| `datalogger_deployment_count` | `count(deployment)`, same logger-only filter as `has_datalogger` | +| `sensor_type` | `sensor.sensor_type` for every currently-installed deployment (`installation_date IS NOT NULL AND removal_date IS NULL`), **any sensor type, not just loggers** — semicolon-joined, ordered by `sensor_type` | +| `model` | `sensor.model`, same deployments, same order as `sensor_type`. `COALESCE(..., '')` before aggregating, so a null value is an empty segment, not a dropped position | +| `serial_no` | `sensor.serial_no`, same deployments, same order, same `COALESCE(..., '')` treatment | +| `sensor_status` | `sensor.sensor_status`, same deployments, same order, same `COALESCE(..., '')` treatment | +| `installed_date` | `deployment.installation_date`, same deployments, same order. In practice never null -- `installed_deployments` filters on `installation_date IS NOT NULL` -- but `COALESCE`d anyway for consistency with its siblings | +| `recording_interval` | `deployment.recording_interval`, same deployments, same order, same `COALESCE(..., '')` treatment — `text`, not `integer`, because `string_agg` produces `text` regardless of how many deployments a given well has | +| `recording_interval_units` | `deployment.recording_interval_units`, same deployments, same order, same `COALESCE(..., '')` treatment | +| `hanging_point_desc` | `deployment.hanging_point_description`, same deployments, same order, same `COALESCE(..., '')` treatment | +| `continuous_reading_count` | `count(transducer_observation)` via `deployment` *(stats)* | +| `continuous_first_datetime` | `min(transducer_observation.observation_datetime)` *(stats)* | +| `continuous_last_datetime` | `max(transducer_observation.observation_datetime)` *(stats)* | +| `days_since_continuous_reading` | `CURRENT_DATE - continuous_last_datetime::date` | +| `contact_count` | `count(DISTINCT contact)` via `thing_contact_association` | +| `primary_contact_name` | `contact.name`, `contact_type = 'Primary'` preferred, else lowest `contact.id` | +| `primary_contact_organization` | `contact.organization`, same contact | +| `primary_contact_role` | `contact.role`, same contact | +| `primary_contact_type` | `contact.contact_type`, same contact — says whether the row above is a real primary or a fallback | +| `primary_contact_phone` | `phone.phone_number`, lowest `phone.id` for that contact | +| `primary_contact_email` | `email.email`, lowest `email.id` for that contact | +| `contact_names` | `contact.name` for every associated contact, comma-joined | +| `access_notes` | `notes.content` where `note_type = 'Access'`, newest first, joined with ` | ` | +| `directions_notes` | `notes.content` where `note_type = 'Directions'`, newest first, joined with ` | ` | +| `communication_notes` | `notes.content` where `note_type = 'Communication'`, newest first, joined with ` | ` | +| `construction_notes` | `notes.content` where `note_type = 'Construction'`, newest first, joined with ` | ` | +| `maintenance_notes` | `notes.content` where `note_type = 'Maintenance'`, newest first, joined with ` | ` | +| `historical_notes` | `notes.content` where `note_type = 'Historical'`, newest first, joined with ` | ` | +| `general_notes` | `notes.content` where `note_type = 'General'`, newest first, joined with ` | ` | +| `water_notes` | `notes.content` where `note_type = 'Water'`, newest first, joined with ` | ` | +| `water_quality_notes` | `notes.content` where `note_type = 'Water Quality'`, newest first, joined with ` | ` | +| `sampling_procedure_notes` | `notes.content` where `note_type = 'Sampling Procedure'`, newest first, joined with ` | ` | +| `coordinate_notes` | `notes.content` where `note_type = 'Coordinate'`, newest first, joined with ` | ` | +| `owner_comment_notes` | `notes.content` where `note_type = 'OwnerComment'`, newest first, joined with ` | ` | +| `site_notes_legacy` | `notes.content` where `note_type = 'Site Notes (legacy)'`, newest first, joined with ` | `. Not `site_notes_legacy_notes` — the lexicon term already says "notes" | +| `point` | `location.point`, most recent association (PostGIS Point, EPSG:4326) | diff --git a/docs/water-well-field-operations-layer.md b/docs/water-well-field-operations-layer.md new file mode 100644 index 00000000..4a24334b --- /dev/null +++ b/docs/water-well-field-operations-layer.md @@ -0,0 +1,634 @@ +# Water well field operations layer — design + +Status: **implemented** on `feat/amp-field-operations-ogc-layer`, then +consolidated with an independently-drafted layer +(`kas-water-well-operations-ogc-layer-bdms-1202`) on +`kas-water-well-field-operations-ogc-layer-bdms-1202` after a side-by-side +column comparison. Migration `e1f2a3b4c5d6`; column-by-column source table in +`docs/water-well-field-operations-columns.md`. + +Two questions this document opened were settled before implementation: "AMP" is +a label for the crews' view of the register, not a subset of it, so the layer +covers every water well; and the contact detail in §4 is published in full +(option A). + +The consolidation dropped several columns this document originally specified +(`nma_pk_welldata`, `county`, `state`, `quad_name`, `elevation_method`, +`nma_formation_zone`, `measuring_point_start_date`, every `*_since`/`*_reason` +status/frequency column, and `access_status` itself -- open question 2 below, +resolved), renamed +several to match the naming already established on the public thing-type views +(`station_type`, `well_purpose`, `well_casing_material`, `mp_height`/ +`mp_description`, `date_last_visited`), added `formation_completion_description` +and `aquifer_system_name`, broadened the equipment columns from logger-only to +every currently-installed sensor, and added a column per remaining +`notes.note_type` value. The rest of this document describes the layer as +originally designed; §6 and §11 call out where the shipped column set now +differs. + +## 0. Why the name does not say AMP + +The layer was requested as `amp_field_well_operations` and renamed before it +merged, per the pre-merge check in `docs/ogc_conventions.md`. Three rules +pushed the same way: + +- `amp` is an unexpanded abbreviation, which the do/don't rules forbid + outright ("Spell abbreviations out in full ... If the accurate name is long, + that's fine"). Its expansion is recorded nowhere in this codebase, so a + consumer could not resolve it even in principle. +- AMP is a label for who uses the layer, not a filter on what is in it — the + row set is every water well, with no group predicate. A name carrying `amp` + would claim a scope the data does not have, which is its own Don't. +- Group B analytic layers prefer a `water_well_` prefix over a `_wells` + suffix, and this is one. + +The AMP crews remain the layer's audience; that belongs in the description, +which says so, rather than in an identifier that is a long-term commitment. + +A new OGC API - Features collection, `water_well_field_operations`, published on +`/ogcapi-internal` only. One feature per water well, carrying everything a +field crew needs to plan and execute a visit: where the well is, who owns it +and how to reach them, what the crew is allowed to do there, what is installed +in it, and when it was last measured. + +Every other well layer in the catalogue answers a scientific question — what is +the water level, what is the chemistry, what is the trend. This one answers an +operational one: **can we go to this well, what are we allowed to do when we +get there, and is it overdue?** + +## 1. Why a new layer rather than a column pass on an existing one + +`water_wells` is the construction register and is public. `water_well_summary` +and `actively_monitored_wells` summarise the measurement record and are public. +The operational fields — landowner phone numbers, access notes, sampling +permissions — cannot go on any of those, because those layers are published +anonymously. The split is a publication boundary, not a modelling preference. + +## 2. Scope + +**In:** water wells. `thing_type = 'water well'`, the same predicate every +other well layer in this catalogue uses. Springs, streams, and met stations are +out even though crews visit them — this is the wells layer, per the request. + +`thing_type` in the lexicon also carries `observation well`, `piezometer`, +`monitoring well`, `production well`, `injection well`, `exploration well`, +`test well`, `abandoned well`, `dry hole`, `artesian well`, and `dug well`. +None of them are included. That matches `ogc_water_wells`, +`ogc_water_well_summary`, and `ogc_well_water_column`, so the layers cannot +disagree about which rows are "wells" — see §11, open question 1. + +**Row set:** every water well with a current location, including wells with no +measurement record at all. A well nobody has ever measured is exactly the well +a field crew needs to find, so this layer must not inherit +`water_well_summary`'s `total_water_levels > 0` restriction. + +**Release status:** unfiltered. The internal mount is unfiltered by design and +this layer has no public twin, so there is no `release_status = 'public'` +predicate anywhere in it. `release_status` is published as a column so a +consumer can see what they are looking at. + +## 3. Placement: internal-only, no public counterpart + +Every other `ogc_internal_*` relation is the unfiltered twin of a public +`ogc_*` one. This layer deliberately breaks that: `ogc_internal_water_well_field_operations` +exists and `ogc_water_well_field_operations` does not, and must never be created. + +This does not disturb the existing parity guard. +`tests/test_migration_view_parity.py::test_internal_migration_mirrors_every_public_relation` +reads two named migration files (`f4a5b6c7d8e9` and `2d3c3a268652`) and asserts +that internal file mirrors that public file's 24 relations. A relation created +by a *new* migration is outside what that test reads, so the count assertion +stays at 24 and stays true. + +Precedent for an internal-only collection: `locations`, `avg_tds_wells`, +`latest_depth_to_water_wells` (static entries in the internal config template +only) and `other_things` (`internal_only: True` in `THING_COLLECTIONS`). + +Because this layer is not a thing-type register, it is wired the same way +`locations` is: a static block in `core/pygeoapi-config-internal.yml`, not an +entry in `THING_COLLECTIONS`. + +## 4. The PII decision — read this before implementing + +This layer publishes landowner and operator contact details: name, +organization, role, phone number, email address. That is what "contact info" +means for a field crew, and it is why the layer is internal-only. + +The consequence is not confined to this layer. `/ogcapi-internal` is reachable +two ways on this branch (`core/internal_ogc_auth.py`): an Authentik JWT +carrying `OGCInternal`, or a static API key from `INTERNAL_OGC_API_KEYS`. +**Shipping this layer means every one of those credentials becomes a credential +that dispenses landowner personal information**, and any credential path added +later -- the user-issued keys on `feat/api-key-management` among them -- +inherits the same reach. The static desktop-GIS keys in particular are shared +secrets that can only be revoked by redeploy. + +pygeoapi has no per-field authorization — a column is either in the view or it +is not — so this is all-or-nothing per layer. Three options: + +| Option | What crews get | Cost | +| --- | --- | --- | +| **A. Full contact detail** (recommended) | Name, organization, role, phone, email | Every internal credential now dispenses PII | +| B. Presence only | `contact_count`, `primary_contact_organization`, `primary_contact_role` — no name, phone, or email | Crews must look the owner up in the UI before every visit; layer half-fails its purpose | +| C. Second layer | `water_well_field_operations` without PII, `water_well_field_contacts` with it | Two layers, two configs, two sets of field docs, same credentials still reach both — the split buys nothing without per-layer auth | + +**Decision: A.** The request was explicit, the mount is authenticated, and B +makes the layer not worth building. Shipped with: + +- A notice at the top of `docs/internal-ogc-desktop-gis.md` stating that an + internal credential now conveys landowner PII, so key issuance is a decision + made with that in mind. **`docs/api-key-management.md` still needs the same + notice** — that file lives on `feat/api-key-management` and does not exist on + this branch, so it could not be edited here. +- The collection `description` says so too, so it is visible in the catalogue + itself. +- The layer does not filter contacts on `release_status`. `Contact` carries one, + but the internal mount is unfiltered by design and this layer has no public + twin to keep in step with. + +## 5. Relations: one matview, one view + +Two relations, not one, because the columns divide cleanly by cost *and* by how +badly staleness hurts: + +``` +ogc_internal_water_well_field_operations_stats MATERIALIZED VIEW, keyed thing_id + counts, first/last dates, aggregates over observation and + transducer_observation. Expensive. Refreshed nightly. + +ogc_internal_water_well_field_operations VIEW <- pygeoapi points here + live join of thing, location, status_history, permission_history, + deployment, sensor, contact, notes + LEFT JOIN the stats matview +``` + +Reasons for the split: + +1. **Staleness is dangerous on exactly the cheap columns.** "May we sample this + well" and "is access currently granted" must not be up to 24 hours old. A + revoked permission that still reads `true` until the next nightly refresh + sends a crew onto land they are no longer welcome on. `permission_history` + and `status_history` are small and indexed by `(target_id, target_table)`; + reading them live costs nothing. +2. **`CURRENT_DATE` freezes in a matview.** The current-record rule in §7 is + date-relative. Evaluated inside a materialized view it would be pinned to + the refresh time, and a permission that expired this morning would still + read as current tonight. It has to be in the plain view. +3. **The expensive columns don't need to be fresh.** A reading count and a + last-measured date are fine at 24 hours old. + +Plain views already back the eleven thing-type collections, so pygeoapi serving +a non-materialized view is established. Filtering and bbox pushdown resolve +against the base tables' indexes. + +`_stats` carries a unique index on `thing_id` so it can be refreshed +`CONCURRENTLY` (`oco refresh-matview --concurrently`). The nightly pg_cron job +discovers every matview in the `public` schema by name, so it is picked up with +no schedule change. Add it to `MATERIALIZED_VIEWS` in +`services/materialized_views.py` for the CLI path. + +## 6. Columns + +`id` is `thing.id` and is unique — pygeoapi's `id_field: id` requires exactly +one row per id for `/items/{id}`. Every one-to-many is therefore aggregated. + +Multi-valued columns are emitted as **`text`, comma-space joined**, not as +Postgres arrays. `ogc_actively_monitored_wells` uses `text[]` and that is fine +for a layer read in a browser, but this one exists to be pulled into ArcGIS Pro +and QGIS and exported to a File Geodatabase or GeoPackage for offline field +use, and neither format has a list type. A joined string survives the round +trip; an array does not. + +### Identity and location + +| Column | Source | +| --- | --- | +| `id` | `thing.id` | +| `name` | `thing.name` | +| `station_type` | `'water well'::text`. Named `station_type` rather than `thing_type` to match the naming already established on the public thing-type views (consolidation, see the note at the top of this document) | +| `release_status` | `thing.release_status` | +| `alternate_ids` | `thing_id_link`, joined as `organization:alternate_id` pairs | +| `latitude`, `longitude` | `ST_Y`/`ST_X` of the same point, decimal degrees on WGS 84 -- a plain number survives a CSV export, which drops the geometry, and a crew can read it into a handheld GPS | +| `elevation` | `location.elevation` | +| `point` | `location.point`, most recent association (the shared `LATEST_LOCATION_CTE`) | + +`nma_pk_welldata`, `county`, `state`, `quad_name`, `nma_formation_zone`, and +`elevation_method` (and its `data_provenance` lookup) were dropped in the +consolidation. None of the four history-table current-record joins below +needed them, and none of the columns this layer is actually for -- status, +permission, equipment, contacts -- depend on them either. + +### Well construction + +`well_depth`, `hole_depth`, `well_casing_diameter`, `well_casing_depth`, +`well_completion_date`, `well_driller_name`, `well_construction_method`, +`well_pump_type`, `well_pump_depth`, `formation_completion_code` — all straight +from `thing`, same columns and same names as the thing-view template, so a +crew reading both layers sees one vocabulary. + +Plus, aggregated: + +| Column | Source | +| --- | --- | +| `formation_completion_description` | `lexicon_term.definition` where `term = thing.formation_completion_code` -- the code alone is not self-explanatory | +| `well_purpose` | `well_purpose.purpose`, joined | +| `well_casing_material` | `well_casing_material.material`, joined | +| `aquifer_system_name` | `aquifer_system.name` via `thing_aquifer_association`, joined | +| `screen_count` | `count(well_screen)` | +| `screen_depth_top`, `screen_depth_bottom`, `screen_description` | every screened interval, ft below ground surface, semicolon-joined, ordered shallowest first | + +The original design collapsed `well_screen` to `screen_count` plus an overall +`min`/`max` depth range, avoiding the per-interval alignment problem below at +the cost of the actual intervals: for a well with two screens, a range says +there's screened section somewhere between the shallowest top and the +deepest bottom, not where the gap between them is, or what each one is made +of. The consolidation restored full per-interval detail instead, on the +premise that a driller doing rehab work needs the real intervals, not a +range. All three columns are ordered by `screen_depth_top` (nulls last) and +each is wrapped in `COALESCE(..., '')` before `string_agg`, for the same +reason as the equipment columns below: `screen_depth_top`, +`screen_depth_bottom`, and `screen_description` are all nullable +independently of each other, and plain `string_agg` would drop a null and +misalign the other two columns' positions. + +`formation_completion_description` and `aquifer_system_name` are both from the +consolidation. Neither is date-windowed like the four history tables below -- +`thing.formation_completion_code` is a plain column and +`thing_aquifer_association` carries no `start_date`/`end_date` at all. + +### Measuring point (current record only) + +| Column | Source | +| --- | --- | +| `mp_height` | `measuring_point_history.measuring_point_height`, ft above ground surface | +| `mp_description` | e.g. "North side of casing, top of PVC" | + +The description is the single most useful string on the layer for a crew +standing at a wellhead. Current record per §7. `measuring_point_start_date` +(when the current configuration took effect) was dropped in the consolidation, +along with the equivalent `_since` column for every status type below. + +### Status (current record per status type) + +`status_history.status_type` has five values in the lexicon. `Access Status` +is not published -- see open question 2, resolved in the consolidation -- so +four of the five are: + +| Column | `status_type` | Values | +| --- | --- | --- | +| `well_status` | Well Status | Abandoned; Active, pumping well; Destroyed, exists but not usable; Inactive, exists but not used | +| `monitoring_status` | Monitoring Status | Currently monitored; Not currently monitored | +| `open_status` | Open Status | Open; Open (unequipped); Closed | +| `datalogger_suitability_status` | Datalogger Suitability Status | Datalogger can be installed; Datalogger cannot be installed | + +The original design also published a `_since` column per status type +plus `monitoring_status_reason` (`status_history.reason`, where "landowner +asked us to stop" is written down). All six were dropped in the +consolidation; the current value is kept, the history around it is not. + +### Permission (current grants) + +From `permission_history`, whose `permission_type` lexicon is exactly the three +things a crew does: + +| Column | `permission_type` | +| --- | --- | +| `may_measure_water_level` | Water Level Sample | +| `may_sample_water_chemistry` | Water Chemistry Sample | +| `may_install_datalogger` | Datalogger Installation | + +**These are three-valued and must stay that way.** `true` = a current grant +says allowed. `false` = a current grant says *not* allowed. `NULL` = **no +permission on record**, which is not the same as denied and must not be +rendered as "no". Collapsing NULL to false would tell a crew a well is off +limits when the truth is that nobody has asked yet; collapsing it to true is +worse. Every consumer-facing rendering of these columns has to carry the third +state. + +`permission_granted_by` — the granting contact's name from the current +water-level grant — is published alongside, so a crew can name who said yes. + +### Monitoring programme + +| Column | Source | +| --- | --- | +| `monitoring_frequency` | current `monitoring_frequency_history` record | +| `group_names` | `group` via `group_thing_association`, joined, de-duplicated | +| `group_types` | Monitoring Plan / Geographic Area / Historical, joined, index-aligned with `group_names` | + +`monitoring_frequency_since` (its `start_date`) was dropped in the +consolidation, same as the status `_since` columns above. + +### Manual measurement statistics + +Manual groundwater levels, reached the way every other water-level view in this +schema reaches them: `observation -> sample -> field_activity -> field_event`, +`field_activity.activity_type = 'groundwater level'`. + +| Column | Definition | +| --- | --- | +| `manual_water_level_count` | count of readings with a value and a timestamp | +| `manual_water_level_first_date` | earliest `observation_datetime`, UTC calendar date | +| `manual_water_level_last_date` | latest, UTC calendar date | +| `days_since_manual_water_level` | `CURRENT_DATE - manual_water_level_last_date`, computed in the plain view so it does not freeze at refresh | +| `last_depth_to_water_ft` | the latest reading, below ground surface | + +`last_depth_to_water_ft` uses `(o.value - COALESCE(o.measuring_point_height, 0))` +— the reading minus the height of the measuring point above ground, with a +missing height treated as taken at ground level. This is the exact convention +in `ogc_water_well_summary`, `ogc_latest_depth_to_water_wells`, and +`ogc_well_water_column`, and it must stay identical or the four layers will +disagree about what a depth to water is. See +`docs/measuring-point-height-null-handling.md`. + +### Chemistry sampling statistics + +Same chain, `field_activity.activity_type = 'water chemistry'`: +`chemistry_sample_count`, `chemistry_sample_last_date`, +`days_since_chemistry_sample`. Sample-level, not analyte-level — a crew wants +"when was this well last sampled", not "what was the sulfate". + +### Field visit statistics + +`field_event_count`, `date_last_visited` from `field_event` +(`field_event_last_date` on the `_stats` matview itself; renamed on the way +out of the feature view to match `thing.first_visit_date`'s counterpart +naming). Broader than either measurement chain: a visit that produced no +reading is still a visit, and the gap between `date_last_visited` and +`manual_water_level_last_date` is itself a signal. + +### Data logger / continuous record + +**`has_datalogger`/`datalogger_deployment_count` stayed logger-scoped in the +consolidation; every other column in this section was broadened to any +currently-installed sensor.** The original design published only a single +current logger deployment's detail (`datalogger_sensor_type`, +`datalogger_model`, etc., picking the most recently installed one via +`DISTINCT ON` if more than one logger was deployed) and had no way to show a +currently-installed camera, barometer, or weather station at all -- it was +simply invisible. That was a real gap against this layer's own purpose: a +crew visiting a well to swap an SD card or check a barometer's battery needs +to know it is there. + +"Has a logger" is still a deployment question, not a sensor-inventory +question: a deployment is current when `installation_date IS NOT NULL` and +`removal_date IS NULL`, and counts towards `has_datalogger` only when the +sensor is one of `Data Logger`, `Pressure Transducer`, `DiverLink`, `Diver +Cable` (`sensor.sensor_type`). + +| Column | Definition | +| --- | --- | +| `has_datalogger` | boolean, true when a current *logger* deployment exists | +| `datalogger_deployment_count` | count of current logger deployments only | +| `sensor_type`, `model`, `serial_no`, `sensor_status`, `installed_date`, `recording_interval`, `recording_interval_units`, `hanging_point_desc` | from `sensor`/`deployment` for **every** currently-installed deployment, any sensor type | +| `continuous_reading_count` | rows in `transducer_observation` for this well's deployments | +| `continuous_first_datetime`, `continuous_last_datetime` | series extent | +| `days_since_continuous_reading` | live, in the plain view | + +The eight equipment columns are no longer a single row's worth of scalar +values -- a well running more than one current sensor lists all of them, +semicolon-joined, all eight ordered the same way by `sensor_type` so position +N in one list is the same deployment as position N in the others. This +follows the same "comma-joined text, not an array" convention as +`well_purpose`/`well_casing_material` above, with a semicolon instead of a +comma because these columns are genuinely positional across each other, not +independently de-duplicated lists. `recording_interval` is `text`, not +`integer`, as a consequence: `string_agg` always returns `text`, even for a +well with exactly one current sensor. + +Every expression but `sensor_type` itself is wrapped in `COALESCE(..., '')` +before aggregating. `sensor_type` is `NOT NULL` on `sensor`, but the other +seven columns are all nullable -- a camera, for instance, has no +`recording_interval` -- and plain `string_agg` silently **drops** a `NULL` +input rather than preserving its position. Caught empirically: a well with a +transducer (`recording_interval = 15`) and a camera (`recording_interval = +NULL`) produced `sensor_type = "Camera; Pressure Transducer"` but +`recording_interval = "15"` -- one segment short, so position 0 read as the +camera's interval when it was actually the transducer's. With the `COALESCE`, +the same well now reads `recording_interval = "; 15"`: an empty segment for +the camera, keeping every column's segment count equal to +`installed_deployments`' row count for that well. + +`sensor_status` values are In Service / In Repair / Retired / Lost. +`datalogger_deployment_count` still says how many *logger* deployments are +open, independent of how many total sensors `sensor_type` lists. + +`continuous_reading_count` aggregates the largest table in the schema. It is +the reason `_stats` is materialized. If the refresh proves too slow, the +fallback is to source the three continuous columns from `transducer_daily_data` +instead — cheaper, one row per day, and adequate for "is the logger still +reporting". + +### Contacts + +Subject to §4. + +| Column | Definition | +| --- | --- | +| `contact_count` | contacts associated with the well | +| `primary_contact_name`, `_organization`, `_role` | prefer the `contact_type = 'Primary'` contact, else any contact on the well, lowest `contact.id` as tie-break | +| `primary_contact_type` | which of those two cases produced the row above | +| `primary_contact_phone` | lowest-id `phone.phone_number` for that contact | +| `primary_contact_email` | lowest-id `email.email` for that contact | +| `contact_names` | every associated contact, joined | + +The fallback is a change from this document's first draft, which took the +primary contact strictly. A well whose only contact is recorded as Secondary +would then show `contact_count = 1` beside a blank name and phone — the exact +failure mode that ruled out option B in §4. `primary_contact_type` keeps the +fallback visible rather than implied. + +Deliberately **not** published: `address`. A mailing address is not how a crew +reaches a landowner, and it is the most sensitive field on the record. + +### Notes + +`notes` is polymorphic on `(target_id, target_table = 'thing')` with a +`note_type` lexicon. The original design published only two operational +types (`access_notes`, `directions_notes`) and kept the rest out, on the +theory that a well's non-visit history is where PII leaks. The consolidation +published a column per remaining `note_type` value instead, same pattern, +most recent first, joined with ` | `: + +| Column | `note_type` | +| --- | --- | +| `access_notes` | Access | +| `directions_notes` | Directions | +| `communication_notes` | Communication | +| `construction_notes` | Construction | +| `maintenance_notes` | Maintenance | +| `historical_notes` | Historical | +| `general_notes` | General | +| `water_notes` | Water | +| `water_quality_notes` | Water Quality | +| `sampling_procedure_notes` | Sampling Procedure | +| `coordinate_notes` | Coordinate | +| `owner_comment_notes` | OwnerComment | +| `site_notes_legacy` | Site Notes (legacy) | + +Two do not mechanically snake-case: `OwnerComment` has no separator to split +on, so `owner_comment_notes` is a judgment call, not a derivation. `Site Notes +(legacy)` becomes `site_notes_legacy`, not `site_notes_legacy_notes` -- the +term already says "notes". + +This does not revisit the PII conclusion in §4 -- if anything it strengthens +it, since these note types are exactly where a landowner's name, a gate code, +or a phone number ends up in free text. See open question 4. + +## 7. The "current record" rule + +Four histories feed this layer — `status_history`, `permission_history`, +`measuring_point_history`, `monitoring_frequency_history` — and all four have +the same `(start_date, end_date)` shape. One rule for all of them: + +```sql +WHERE h.start_date <= CURRENT_DATE + AND (h.end_date IS NULL OR h.end_date >= CURRENT_DATE) +ORDER BY h.start_date DESC, h.id DESC +LIMIT 1 -- via DISTINCT ON per thing +``` + +**This diverges from the existing catalogue on purpose.** +`ogc_actively_monitored_wells` takes the row with the greatest `start_date` and +ignores `end_date` entirely, so a monitoring status that was closed in 2019 +still reads as current there. That is tolerable on a summary layer. It is not +tolerable on a layer whose whole job is to tell a crew what is true today — +"you may install a logger here" must not survive the expiry of the permission +that said so. + +The divergence has to be written into `docs/ogc_conventions.md`, or the next +person will read the two views side by side and assume one is a mistake. + +Rows whose window has not opened yet (`start_date > CURRENT_DATE`) are not +current and read NULL. + +## 8. Field-level documentation + +`core/ogc_field_metadata.py` keys `core/ogc-field-descriptions.yml` by backing +relation with the `ogc_internal_` prefix stripped, so the entry is +`water_well_field_operations`. This is not optional: +`tests/test_ogc_field_descriptions.py::test_every_published_column_has_an_entry` +walks both mounts and fails on any published column without a `description`. + +Columns already in `_defaults` (`id`, `name`, `thing_type`, `release_status`, +`elevation`, `well_depth`, `hole_depth`, `well_casing_diameter`, …) are +inherited and must not be restated. `station_type` needs its own entry despite +being lexicon-governed the same way `thing_type` is, precisely because it is +not named `thing_type` here -- a per-table entry only inherits from +`_defaults` by column name. Everything new to this layer needs an entry, and +the entry says what the value *means*, never how the view is built. + +Not every lexicon-governed column carries `enum-lexicon`: the five status +columns use a literal `enum` list instead (pre-dating the consolidation, and +left as-is rather than relitigated here), and `monitoring_frequency`, `role`, +and `contact_type` use `enum-lexicon`. `sensor_type`, `sensor_status`, +`well_purpose`, and `well_casing_material` carry **neither** -- each is an +aggregated, multi-valued column (`string_agg` over more than one row), and an +`enum` on a column whose value can be `"Barometer; Pressure Transducer"` would +be actively wrong, not just incomplete. Depth and height columns carry +`x-ogc-unit: https://qudt.org/vocab/unit/FT` with +`x-ogc-unitLang: QUDT`, matching the rest of the file. + +The three permission booleans need their NULL meaning spelled out in the +`description` — that is the only place a consumer will ever read it. + +## 9. Wiring + +1. **Migration** — new Alembic revision on the current head (`c9d0e1f2a3b4`), + creating the matview, its unique index, and the view, plus the supporting + indexes in §10. `downgrade()` drops all three; nothing pre-existing is + modified, so there is no prior state to restore. Follow the house style in + `2d3c3a268652`: self-contained, no cross-migration imports, `REQUIRED_TABLES` + guard up front. +2. **`core/pygeoapi-config-internal.yml`** — one static resource block, next to + `locations`. `name: core.feature_provider.DescribedPostgreSQLProvider`, + `id_field: id`, `table: ogc_internal_water_well_field_operations`, + `geom_field: point`. No `time_field`: no single column is the feature's + time, and picking one would make `datetime=` filtering quietly mean + something arbitrary. +3. **`core/pygeoapi-config.yml`** — no change. Ever. +4. **`core/ogc-field-descriptions.yml`** — the `water_well_field_operations` block. +5. **`services/materialized_views.py`** — append + `ogc_internal_water_well_field_operations_stats` to `MATERIALIZED_VIEWS`. +6. **`core/gis-curated-layers.yml`** — *no* entry. Every layer in that file + ships as a `.qlr` and a `.lyrx` artifact, and this one is PII-bearing and + internal; it should be reached by browsing the authenticated connection, not + handed out as a downloadable file. Revisit only if crews ask. +7. **Docs** — `docs/api-key-management.md` and `docs/internal-ogc-desktop-gis.md` + gain the PII notice from §4; `docs/ogc_conventions.md` gains the §7 + divergence. + +Description text has to clear +`tests/test_pygeoapi_mount.py::test_every_collection_description_explains_the_layer`: +at least 200 characters, ends in a full stop, no placeholder words, no +hyphenated word split across a YAML line break, and at least four unique +lowercase-hyphenated keywords. + +## 10. Indexes + +The per-thing lookups this layer adds are foreign-key joins, and Postgres does +not index foreign keys on its own. Check each before creating it — `b8c9d0e1f2a3` +already added four on the observation chain. + +- `status_history (target_table, target_id, status_type, start_date DESC)` +- `permission_history (target_table, target_id, permission_type, start_date DESC)` +- `measuring_point_history (thing_id, start_date DESC)` +- `monitoring_frequency_history (thing_id, start_date DESC)` +- `notes (target_id, target_table)` — already exists as `ix_notes_polymorphic_link` +- `deployment (thing_id)` +- `thing_contact_association (thing_id)`, `(contact_id)` +- `thing_aquifer_association (thing_id)` — for `aquifer_system_name`, added in + the consolidation +- `transducer_observation (deployment_id, observation_datetime)` — for the + continuous aggregates; the largest table in the schema and the one that will + decide whether the nightly refresh is acceptable + +## 11. Settled, and still open + +Settled before implementation: + +- **"AMP" is a label, not a subset.** The layer covers every water well; there + is no group predicate. +- **Contact detail is published in full** (§4, option A). + +Settled in the consolidation: + +- **Access Status has no values -- column removed, not left publishing NULL.** + `status_type` carried `Access Status`, but the `status_value` lexicon has no + access-related terms; the eleven values map only to Well, Monitoring, Open, + and Datalogger Suitability status. Rather than ship a column that could only + ever read NULL, `access_status` and its join were dropped. `access_notes` + (§6, Notes) already carries staff-written access information for a well -- + gate codes, locks, who to call -- so nothing a crew needs is lost. If the + lexicon later gains terms scoped to Access Status, this is worth + reconsidering, but as a fresh addition rather than un-deleting a column that + published nothing. + +Still open: + +1. **Which `thing_type` values count as a well?** Shipped as `'water well'` + only, for consistency with every existing well layer. If AMP crews also + visit piezometers and monitoring wells, the predicate widens to a list — and + then it should widen in the other four layers too, as its own ticket, rather + than this layer quietly disagreeing with them. +2. **Free-text PII.** All 13 note columns are staff-written and will contain + phone numbers, gate codes, and names -- more of them since the + consolidation, which published the eleven note types the original design + deliberately kept out (§6, Notes). A further reason the layer is + internal-only, and the §4 decision covers them too. +3. **Refresh cadence.** Nightly, via the existing pg_cron job. If crews plan + the next day's route the evening before, nightly is fine; if they re-plan in + the field, the stats columns are the ones that go stale. +4. **`continuous_reading_count` cost at production scale.** It aggregates the + largest table in the schema. If the nightly refresh proves too slow, the + fallback is to source the three continuous columns from + `transducer_daily_data` instead. + +## 12. Out of scope + +- Any public counterpart. +- Write access. This is a read layer; edits happen in the Ocotillo UI. +- A curated `.qlr` / `.lyrx` artifact (§9.6). +- Springs and other non-well monitoring points. +- Per-field authorization. pygeoapi has no such hook; the layer boundary is the + authorization boundary. diff --git a/services/materialized_views.py b/services/materialized_views.py index 9b7e3740..75c5239d 100644 --- a/services/materialized_views.py +++ b/services/materialized_views.py @@ -19,5 +19,9 @@ # two pivot views above, at per-result grain. "ogc_water_chemistry", "ogc_internal_water_chemistry", + # Field-operations aggregates for the internal-only + # ogc_internal_water_well_field_operations layer (e1f2a3b4c5d6). The + # feature view itself is a plain view and needs no refresh. + "ogc_internal_water_well_field_operations_stats", "transducer_daily_data", ) diff --git a/tests/features/steps/ogc-cleanup-sprint1.py b/tests/features/steps/ogc-cleanup-sprint1.py index 5ccf0b22..53e91367 100644 --- a/tests/features/steps/ogc-cleanup-sprint1.py +++ b/tests/features/steps/ogc-cleanup-sprint1.py @@ -835,11 +835,24 @@ def step_then_schema_contains_relations_prefixed(context, prefix): assert matching, f"expected at least one relation prefixed {prefix!r}, found none" +# Relations that are internal-only by design, with no public counterpart at +# all -- not an accidental gap this check should catch. This layer publishes +# landowner contact details and staff-written notes; see +# docs/water-well-field-operations-layer.md section 3 for why it deliberately +# has no ogc_water_well_field_operations public twin. +INTERNAL_ONLY_NO_PUBLIC_COUNTERPART = { + "ogc_internal_water_well_field_operations", + "ogc_internal_water_well_field_operations_stats", +} + + @then("no ogc_internal_ relation is shared with the public /ogcapi endpoint") def step_then_no_internal_relation_shared_with_public(context): internal = {r for r in context.schema_relations if r.startswith("ogc_internal_")} assert internal, "no ogc_internal_ relations found in the schema" for relation in internal: + if relation in INTERNAL_ONLY_NO_PUBLIC_COUNTERPART: + continue public_equivalent = relation.replace("ogc_internal_", "ogc_", 1) assert public_equivalent in context.schema_relations, ( f"{relation} has no distinct public counterpart ({public_equivalent}) in " diff --git a/tests/test_cli_commands.py b/tests/test_cli_commands.py index 97534a60..9923fb38 100644 --- a/tests/test_cli_commands.py +++ b/tests/test_cli_commands.py @@ -72,10 +72,11 @@ def __exit__(self, exc_type, exc, tb): "REFRESH MATERIALIZED VIEW ogc_minor_chemistry_wells", "REFRESH MATERIALIZED VIEW ogc_water_chemistry", "REFRESH MATERIALIZED VIEW ogc_internal_water_chemistry", + "REFRESH MATERIALIZED VIEW ogc_internal_water_well_field_operations_stats", "REFRESH MATERIALIZED VIEW transducer_daily_data", ] assert commit_called["value"] is True - assert "Refreshed 10 materialized view(s)." in result.output + assert "Refreshed 11 materialized view(s)." in result.output def test_refresh_materialized_views_custom_and_concurrently( diff --git a/tests/test_ogc.py b/tests/test_ogc.py index 18c5fea7..5d6d3c83 100644 --- a/tests/test_ogc.py +++ b/tests/test_ogc.py @@ -818,12 +818,15 @@ def test_ogc_collections(ogc_client): # and latest_depth_to_water_wells repeats water_well_summary # (BDMS-977), and other_things is internal vocabulary (BDMS-979). The # backing relations are retained and still served on /ogcapi-internal. + # water_well_field_operations is internal-only for a stronger reason than + # the other four: it publishes landowner contact details. assert ids.isdisjoint( { "locations", "avg_tds_wells", "latest_depth_to_water_wells", "other_things", + "water_well_field_operations", } ) diff --git a/tests/test_ogc_water_well_field_operations.py b/tests/test_ogc_water_well_field_operations.py new file mode 100644 index 00000000..eb5c43a6 --- /dev/null +++ b/tests/test_ogc_water_well_field_operations.py @@ -0,0 +1,836 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""The internal-only water well field operations layer (e1f2a3b4c5d6). + +What is worth testing here is not that the columns exist but that the layer's +load-bearing rules hold: a well with no measurements still appears, a +permission with no record reads NULL rather than false, a history record +whose window has closed is not treated as current, and a currently-installed +sensor that is not a logger is still visible in the equipment columns even +though it does not count towards has_datalogger. + +See docs/water-well-field-operations-layer.md. +""" + +from datetime import date, datetime, timedelta +from importlib.util import find_spec + +import pytest +from fastapi.testclient import TestClient +from sqlalchemy import text + +from core.dependencies import ( + admin_function, + amp_admin_function, + amp_editor_function, + amp_viewer_function, + editor_function, + viewer_function, +) +from core.factory import create_api_app +from db import ( + AquiferSystem, + Deployment, + LexiconTerm, + Notes, + PermissionHistory, + Sensor, + StatusHistory, + Thing, + ThingAquiferAssociation, + WellScreen, +) +from db.engine import session_ctx +from tests import override_authentication + +pytestmark = pytest.mark.skipif( + find_spec("pygeoapi") is None, + reason="pygeoapi is not installed in this environment", +) + +VIEW = "ogc_internal_water_well_field_operations" +STATS_VIEW = "ogc_internal_water_well_field_operations_stats" + + +@pytest.fixture(scope="module") +def today(): + """The database's idea of today, not Python's. + + The view computes its `days_since_*` columns against CURRENT_DATE, which is + evaluated in the database session's timezone. That is UTC here while the + developer running the tests may not be, so a date read from Python is off + by one for part of every day. Read it from the same place the view does. + """ + with session_ctx() as session: + return session.execute(text("SELECT CURRENT_DATE")).scalar() + + +@pytest.fixture(scope="module") +def yesterday(today): + return today - timedelta(days=1) + + +@pytest.fixture(scope="module") +def last_year(today): + return today - timedelta(days=365) + + +@pytest.fixture(scope="module") +def ogc_client(): + app = create_api_app() + for dependency in ( + admin_function, + editor_function, + amp_admin_function, + amp_editor_function, + ): + app.dependency_overrides[dependency] = override_authentication( + default={"name": "foobar", "sub": "1234567890"} + ) + for dependency in (viewer_function, amp_viewer_function): + app.dependency_overrides[dependency] = override_authentication() + + with TestClient(app) as client: + yield client + + app.dependency_overrides = {} + + +def _row(session, thing_id, columns): + return session.execute( + text(f"SELECT {columns} FROM {VIEW} WHERE id = :thing_id"), + {"thing_id": thing_id}, + ).one() + + +def _refresh_stats(session): + session.execute(text(f"REFRESH MATERIALIZED VIEW {STATS_VIEW}")) + session.commit() + + +# ------------------------------------------------------------------ row set + + +def test_a_well_with_no_measurements_still_appears(water_well_thing): + # water_well_summary drops wells with no readings, because a summary of + # nothing says nothing. This layer must not: a well nobody has measured is + # exactly the well a crew needs to find. + with session_ctx() as session: + _refresh_stats(session) + row = _row( + session, + water_well_thing.id, + "name, manual_water_level_count, chemistry_sample_count, " + "continuous_reading_count, days_since_manual_water_level", + ) + + assert row.name == "Test Well" + assert row.manual_water_level_count == 0 + assert row.chemistry_sample_count == 0 + assert row.continuous_reading_count == 0 + # No reading means no elapsed time to report, not zero days since one. + assert row.days_since_manual_water_level is None + + +def test_latitude_and_longitude_match_the_geometry(water_well_thing): + with session_ctx() as session: + row = session.execute( + text( + "SELECT latitude, longitude, ST_Y(point) AS geom_y, " + f"ST_X(point) AS geom_x FROM {VIEW} WHERE id = :thing_id" + ), + {"thing_id": water_well_thing.id}, + ).one() + + # Decimal degrees on WGS 84, read off the same point the geometry carries. + assert row.latitude == row.geom_y + assert row.longitude == row.geom_x + assert -180 <= row.longitude <= 180 + assert -90 <= row.latitude <= 90 + + +# ------------------------------------------------------------- construction + + +def test_formation_completion_description_reads_the_lexicon_definition( + water_well_thing, +): + with session_ctx() as session: + term = LexiconTerm( + term="Test Formation XYZ", + definition="A test formation used only by this test", + ) + session.add(term) + session.commit() + + thing = session.get(Thing, water_well_thing.id) + thing.formation_completion_code = term.term + session.commit() + + row = _row( + session, + water_well_thing.id, + "formation_completion_code, formation_completion_description", + ) + assert row.formation_completion_code == "Test Formation XYZ" + assert ( + row.formation_completion_description + == "A test formation used only by this test" + ) + + thing.formation_completion_code = None + session.commit() + session.delete(term) + session.commit() + + +def test_formation_completion_description_is_null_without_a_code(water_well_thing): + with session_ctx() as session: + row = _row(session, water_well_thing.id, "formation_completion_description") + + assert row.formation_completion_description is None + + +def test_aquifer_system_name_is_comma_joined(water_well_thing): + with session_ctx() as session: + system_a = AquiferSystem( + name="Test Aquifer A", primary_aquifer_type="Unconfined multiple aquifers" + ) + system_b = AquiferSystem( + name="Test Aquifer B", primary_aquifer_type="Confined multiple aquifers" + ) + session.add_all([system_a, system_b]) + session.commit() + + associations = [ + ThingAquiferAssociation( + thing_id=water_well_thing.id, aquifer_system_id=system_a.id + ), + ThingAquiferAssociation( + thing_id=water_well_thing.id, aquifer_system_id=system_b.id + ), + ] + session.add_all(associations) + session.commit() + + row = _row(session, water_well_thing.id, "aquifer_system_name") + assert row.aquifer_system_name == "Test Aquifer A, Test Aquifer B" + + for association in associations: + session.delete(association) + session.delete(system_a) + session.delete(system_b) + session.commit() + + +def test_screens_list_every_interval_position_aligned( + water_well_thing, well_screen, second_well_screen +): + with session_ctx() as session: + row = _row( + session, + water_well_thing.id, + "screen_count, screen_depth_top, screen_depth_bottom, " + "screen_description", + ) + + # well_screen is 10-20ft, second_well_screen is 30-40ft, so + # screen_depth_top's ascending order puts well_screen first throughout. + assert row.screen_count == 2 + assert row.screen_depth_top == "10; 30" + assert row.screen_depth_bottom == "20; 40" + assert row.screen_description == ( + "Test well screen description; Test well screen description" + ) + + +def test_a_null_screen_field_leaves_an_empty_slot_not_a_dropped_position( + water_well_thing, well_screen +): + # A screen with no recorded bottom depth or description. Plain + # string_agg would drop those NULLs, shortening screen_depth_bottom and + # screen_description to one entry each while screen_depth_top still had + # two -- position 0 would then read as well_screen's bottom depth when + # it is actually the incomplete screen's. + with session_ctx() as session: + incomplete_screen = WellScreen( + thing_id=water_well_thing.id, + screen_depth_top=5.0, + screen_depth_bottom=None, + screen_description=None, + release_status="draft", + ) + session.add(incomplete_screen) + session.commit() + + row = _row( + session, + water_well_thing.id, + "screen_depth_top, screen_depth_bottom, screen_description", + ) + + # 5.0 sorts before well_screen's 10.0, so position 0 is the + # incomplete screen throughout -- an empty segment where its values + # are null, not a shorter list. + assert row.screen_depth_top == "5; 10" + assert row.screen_depth_bottom == "; 20" + assert row.screen_description == "; Test well screen description" + + session.delete(incomplete_screen) + session.commit() + + +# --------------------------------------------------------------- permissions + + +def test_permission_with_no_record_is_null_not_false(water_well_thing): + with session_ctx() as session: + row = _row( + session, + water_well_thing.id, + "may_measure_water_level, may_sample_water_chemistry, " + "may_install_datalogger", + ) + + # NULL means nobody has asked the landowner. Rendering it as "no" would + # tell a crew the well is off limits when the truth is unknown. + assert row.may_measure_water_level is None + assert row.may_sample_water_chemistry is None + assert row.may_install_datalogger is None + + +def test_permission_distinguishes_granted_from_refused( + water_well_thing, contact, last_year +): + with session_ctx() as session: + granted = PermissionHistory( + contact_id=contact.id, + target_id=water_well_thing.id, + target_table="thing", + permission_type="Water Level Sample", + permission_allowed=True, + start_date=last_year, + ) + refused = PermissionHistory( + contact_id=contact.id, + target_id=water_well_thing.id, + target_table="thing", + permission_type="Datalogger Installation", + permission_allowed=False, + start_date=last_year, + ) + session.add_all([granted, refused]) + session.commit() + + row = _row( + session, + water_well_thing.id, + "may_measure_water_level, may_install_datalogger, " + "may_sample_water_chemistry, permission_granted_by", + ) + + assert row.may_measure_water_level is True + assert row.may_install_datalogger is False + # Untouched permission type stays unknown rather than inheriting either. + assert row.may_sample_water_chemistry is None + assert row.permission_granted_by == contact.name + + session.delete(granted) + session.delete(refused) + session.commit() + + +def test_expired_permission_is_not_current( + water_well_thing, contact, last_year, yesterday +): + with session_ctx() as session: + expired = PermissionHistory( + contact_id=contact.id, + target_id=water_well_thing.id, + target_table="thing", + permission_type="Water Level Sample", + permission_allowed=True, + start_date=last_year, + end_date=yesterday, + ) + session.add(expired) + session.commit() + + row = _row(session, water_well_thing.id, "may_measure_water_level") + # A permission that ran out yesterday is not a permission today. This + # is the divergence from ogc_actively_monitored_wells, which ignores + # end_date entirely. + assert row.may_measure_water_level is None + + session.delete(expired) + session.commit() + + +# -------------------------------------------------------------------- status + + +def test_status_reads_the_current_record_not_the_latest( + water_well_thing, last_year, yesterday +): + with session_ctx() as session: + closed = StatusHistory( + target_id=water_well_thing.id, + target_table="thing", + status_type="Monitoring Status", + status_value="Currently monitored", + start_date=last_year, + end_date=yesterday, + reason="programme ended", + ) + open_status = StatusHistory( + target_id=water_well_thing.id, + target_table="thing", + status_type="Well Status", + status_value="Active, pumping well", + start_date=last_year, + ) + session.add_all([closed, open_status]) + session.commit() + + row = _row( + session, + water_well_thing.id, + "monitoring_status, well_status", + ) + + assert row.monitoring_status is None + assert row.well_status == "Active, pumping well" + + session.delete(closed) + session.delete(open_status) + session.commit() + + +def test_status_types_do_not_bleed_into_each_other(water_well_thing, last_year): + with session_ctx() as session: + monitoring = StatusHistory( + target_id=water_well_thing.id, + target_table="thing", + status_type="Monitoring Status", + status_value="Not currently monitored", + start_date=last_year, + reason="landowner asked us to stop", + ) + session.add(monitoring) + session.commit() + + row = _row( + session, + water_well_thing.id, + "monitoring_status, well_status, " + "open_status, datalogger_suitability_status", + ) + + assert row.monitoring_status == "Not currently monitored" + assert row.well_status is None + assert row.open_status is None + assert row.datalogger_suitability_status is None + + session.delete(monitoring) + session.commit() + + +# ---------------------------------------------------------------- datalogger + + +def test_has_datalogger_counts_only_logger_equipment( + water_well_thing, sensor_to_water_well_thing_deployment +): + with session_ctx() as session: + row = _row( + session, + water_well_thing.id, + "has_datalogger, datalogger_deployment_count, " + "sensor_type, serial_no, recording_interval", + ) + + assert row.has_datalogger is True + assert row.datalogger_deployment_count == 1 + # Single current sensor: aggregation degenerates to a bare value, no + # delimiter. + assert row.sensor_type == "Pressure Transducer" + assert row.serial_no == "123456" + assert row.recording_interval == "24" + + +def test_non_logger_equipment_does_not_make_a_well_instrumented( + water_well_thing, last_year +): + with session_ctx() as session: + barometer = Sensor( + name="Test Barometer", + sensor_type="Barometer", + sensor_status="In Service", + release_status="draft", + ) + session.add(barometer) + session.commit() + deployment = Deployment( + sensor_id=barometer.id, + thing_id=water_well_thing.id, + installation_date=last_year, + removal_date=None, + ) + session.add(deployment) + session.commit() + + row = _row( + session, + water_well_thing.id, + "has_datalogger, datalogger_deployment_count, sensor_type", + ) + # A barometer at the well is equipment, not a logger in the well -- + # but it must still be visible as installed equipment, unlike the + # old logger-only columns this replaces. + assert row.has_datalogger is False + assert row.datalogger_deployment_count == 0 + assert row.sensor_type == "Barometer" + + session.delete(deployment) + session.delete(barometer) + session.commit() + + +def test_installed_equipment_lists_every_sensor_position_aligned( + water_well_thing, sensor, sensor_to_water_well_thing_deployment, last_year +): + with session_ctx() as session: + barometer = Sensor( + name="Test Barometer", + sensor_type="Barometer", + model="BaroTroll", + serial_no="BT-002", + sensor_status="In Service", + release_status="draft", + ) + session.add(barometer) + session.commit() + barometer_deployment = Deployment( + sensor_id=barometer.id, + thing_id=water_well_thing.id, + installation_date=last_year, + removal_date=None, + recording_interval=60, + recording_interval_units="minute", + hanging_point_description="Strapped to fence post", + ) + session.add(barometer_deployment) + session.commit() + + row = _row( + session, + water_well_thing.id, + "sensor_type, model, serial_no, recording_interval, " + "recording_interval_units, hanging_point_desc, has_datalogger, " + "datalogger_deployment_count", + ) + + # Ordered alphabetically by sensor_type ("Barometer" < + # "Pressure Transducer"), and every column ordered the same way, so + # position 0 in each list describes the barometer and position 1 the + # transducer. + assert row.sensor_type == "Barometer; Pressure Transducer" + assert row.model == "BaroTroll; Model X" + assert row.serial_no == "BT-002; 123456" + assert row.recording_interval == "60; 24" + assert row.recording_interval_units == "minute; hour" + assert row.hanging_point_desc == "Strapped to fence post; hang 10" + # The non-logger sensor does not count towards the logger-only signal. + assert row.has_datalogger is True + assert row.datalogger_deployment_count == 1 + + session.delete(barometer_deployment) + session.delete(barometer) + session.commit() + + +def test_a_null_field_on_one_sensor_leaves_an_empty_slot_not_a_dropped_position( + water_well_thing, sensor, sensor_to_water_well_thing_deployment, last_year +): + # A camera has no recording interval. Plain string_agg would silently + # drop that NULL, shortening recording_interval's list to one entry while + # sensor_type still has two -- position 0 would then read as the + # camera's interval when it is actually the transducer's. The view + # COALESCEs to '' specifically so this doesn't happen. + with session_ctx() as session: + camera = Sensor( + name="Test Camera", + sensor_type="Camera", + model="Reconyx HC600", + serial_no=None, + sensor_status="In Service", + release_status="draft", + ) + session.add(camera) + session.commit() + camera_deployment = Deployment( + sensor_id=camera.id, + thing_id=water_well_thing.id, + installation_date=last_year, + removal_date=None, + recording_interval=None, + recording_interval_units=None, + hanging_point_description=None, + ) + session.add(camera_deployment) + session.commit() + + row = _row( + session, + water_well_thing.id, + "sensor_type, model, serial_no, recording_interval, " + "recording_interval_units, hanging_point_desc", + ) + + # "Camera" sorts before "Pressure Transducer", so position 0 is the + # camera throughout -- an empty segment where its value is null, not + # a shorter list. + assert row.sensor_type == "Camera; Pressure Transducer" + assert row.model == "Reconyx HC600; Model X" + assert row.serial_no == "; 123456" + assert row.recording_interval == "; 24" + assert row.recording_interval_units == "; hour" + assert row.hanging_point_desc == "; hang 10" + + session.delete(camera_deployment) + session.delete(camera) + session.commit() + + +def test_a_removed_logger_leaves_the_well_uninstrumented( + water_well_thing, sensor, sensor_to_water_well_thing_deployment, yesterday +): + with session_ctx() as session: + deployment = session.get(Deployment, sensor_to_water_well_thing_deployment.id) + deployment.removal_date = yesterday + session.commit() + + row = _row(session, water_well_thing.id, "has_datalogger") + assert row.has_datalogger is False + + deployment.removal_date = None + session.commit() + + +# ------------------------------------------------------------- measurements + + +def test_last_depth_to_water_uses_the_measuring_point_convention( + water_well_thing, groundwater_level_sample, today +): + from db import Observation + from tests import get_parameter_id + + with session_ctx() as session: + readings = [] + for day, value, measuring_point_height in ((1, 6.0, 1.0), (2, 9.0, 2.0)): + observation = Observation( + observation_datetime=datetime(2025, 1, day, 12, 0, 0), + sample_id=groundwater_level_sample.id, + parameter_id=get_parameter_id("groundwater level", "Field Parameter"), + release_status="public", + value=value, + unit="ft", + measuring_point_height=measuring_point_height, + groundwater_level_reason="Water level not affected", + ) + session.add(observation) + readings.append(observation) + session.commit() + _refresh_stats(session) + + row = _row( + session, + water_well_thing.id, + "manual_water_level_count, manual_water_level_first_date, " + "manual_water_level_last_date, last_depth_to_water_ft, " + "days_since_manual_water_level", + ) + + assert row.manual_water_level_count == 2 + assert row.manual_water_level_first_date == date(2025, 1, 1) + assert row.manual_water_level_last_date == date(2025, 1, 2) + # Latest reading is 9 ft from a measuring point 2 ft above ground. + assert abs(float(row.last_depth_to_water_ft) - 7.0) < 1e-9 + # Computed against today, not against the last refresh. + assert row.days_since_manual_water_level == (today - date(2025, 1, 2)).days + + for observation in readings: + session.delete(observation) + session.commit() + _refresh_stats(session) + + +def test_private_readings_are_counted_on_this_internal_layer( + water_well_thing, groundwater_level_sample +): + from db import Observation + from tests import get_parameter_id + + with session_ctx() as session: + observation = Observation( + observation_datetime=datetime(2025, 2, 1, 12, 0, 0), + sample_id=groundwater_level_sample.id, + parameter_id=get_parameter_id("groundwater level", "Field Parameter"), + release_status="private", + value=4.0, + unit="ft", + measuring_point_height=1.0, + groundwater_level_reason="Water level not affected", + ) + session.add(observation) + session.commit() + _refresh_stats(session) + + row = _row(session, water_well_thing.id, "manual_water_level_count") + # The internal mount is unfiltered by design; this layer has no public + # twin to keep in step with. + assert row.manual_water_level_count == 1 + + session.delete(observation) + session.commit() + _refresh_stats(session) + + +# ------------------------------------------------------ multi-valued columns + + +def test_multi_valued_columns_are_comma_joined_text( + water_well_thing, domestic_well_purpose, irrigation_well_purpose +): + with session_ctx() as session: + row = _row(session, water_well_thing.id, "well_purpose") + + # Text, not an array: this layer is exported to File Geodatabase and + # GeoPackage for offline field use, and neither format has a list type. + assert isinstance(row.well_purpose, str) + assert row.well_purpose == "Domestic, Irrigation" + + +def test_note_types_do_not_bleed_into_each_other(water_well_thing): + with session_ctx() as session: + water_note = Notes( + target_id=water_well_thing.id, + target_table="thing", + note_type="Water", + content="Well produces slightly sulfurous water", + ) + maintenance_note = Notes( + target_id=water_well_thing.id, + target_table="thing", + note_type="Maintenance", + content="Pump replaced 2024", + ) + session.add_all([water_note, maintenance_note]) + session.commit() + + row = _row( + session, + water_well_thing.id, + "water_notes, maintenance_notes, coordinate_notes, " "owner_comment_notes", + ) + + assert row.water_notes == "Well produces slightly sulfurous water" + assert row.maintenance_notes == "Pump replaced 2024" + # Untouched note types stay null rather than inheriting either note. + assert row.coordinate_notes is None + assert row.owner_comment_notes is None + + session.delete(water_note) + session.delete(maintenance_note) + session.commit() + + +def test_contacts_and_access_notes_are_published(water_well_thing, contact, phone): + with session_ctx() as session: + note = Notes( + target_id=water_well_thing.id, + target_table="thing", + note_type="Access", + content="Gate is locked, call ahead", + ) + session.add(note) + session.commit() + + row = _row( + session, + water_well_thing.id, + "contact_count, primary_contact_name, primary_contact_type, " + "primary_contact_role, primary_contact_phone, access_notes", + ) + + assert row.contact_count == 1 + assert row.primary_contact_name == contact.name + assert row.primary_contact_type == "Primary" + assert row.primary_contact_role == "Owner" + assert row.primary_contact_phone == phone.phone_number + assert row.access_notes == "Gate is locked, call ahead" + + session.delete(note) + session.commit() + + +# ------------------------------------------------------------------- mounts + + +def test_layer_is_served_on_the_internal_mount_only(ogc_client): + internal = ogc_client.get( + "/ogcapi-internal/collections/water_well_field_operations/items?limit=1" + ) + assert internal.status_code == 200 + assert internal.json()["type"] == "FeatureCollection" + + public = ogc_client.get( + "/ogcapi/collections/water_well_field_operations/items?limit=1" + ) + # The layer publishes landowner contact details; it must not exist at all + # on the anonymous mount. + assert public.status_code == 404 + + +def test_every_column_is_documented_on_the_internal_mount(ogc_client): + response = ogc_client.get( + "/ogcapi-internal/collections/water_well_field_operations/schema" + ) + assert response.status_code == 200 + + properties = response.json()["properties"] + gaps = [ + name + for name, prop in properties.items() + if name != "geometry" and not prop.get("description") + ] + assert not gaps, f"columns with no YAML entry: {gaps}" + + +def test_permission_columns_explain_their_null_meaning(ogc_client): + response = ogc_client.get( + "/ogcapi-internal/collections/water_well_field_operations/schema" + ) + properties = response.json()["properties"] + + # The three-valued meaning only reaches a consumer through this prose. + for column in ( + "may_measure_water_level", + "may_sample_water_chemistry", + "may_install_datalogger", + ): + assert "null" in properties[column]["description"].lower() diff --git a/tests/test_pygeoapi_mount.py b/tests/test_pygeoapi_mount.py index e267d180..2572874e 100644 --- a/tests/test_pygeoapi_mount.py +++ b/tests/test_pygeoapi_mount.py @@ -115,12 +115,15 @@ def test_loading_a_mount_restores_config_env_vars(): # Layers hidden from the public catalog but still served to staff GIS # clients on /ogcapi-internal: locations duplicates the thing-type layers # (BDMS-978), avg_tds_wells and latest_depth_to_water_wells are misleading -# or redundant (BDMS-977), other_things is internal vocabulary (BDMS-979). +# or redundant (BDMS-977), other_things is internal vocabulary (BDMS-979), +# and water_well_field_operations carries landowner contact details and +# staff-written access notes, so it has no public form at all. INTERNAL_ONLY_COLLECTIONS = { "locations", "avg_tds_wells", "latest_depth_to_water_wells", "other_things", + "water_well_field_operations", }