From adbf4258a69d96bef236d3435303449f06ff5c7c Mon Sep 17 00:00:00 2001 From: jakeross Date: Mon, 24 Aug 2026 22:01:51 -0700 Subject: [PATCH 01/13] docs(adr): decide the access-control engine count (ADR5) Answers PUB-D14 / PERM-D3 from the "Who May See What, and Who May Do What" whitepaper: two grant tables, one enforcement engine. Landowner publication consent and internal permission grants are stored separately, because they are decided by different people on different authority and revoked by different events. Both are evaluated by a single visibility layer and projected through a single field-allowlist chokepoint, because distributed enforcement has already drifted here: baba91fe5e83 fixed ogc_waterlevels publishing readings from draft and private wells -- with the well's name and coordinates attached -- while ogc_water_chemistry had gated on the parent thing since d9e0f1a2b3c4. Twelve views carry their own copy of the rule today. Leaves open, on purpose: data type granularity, whether field protection is removal or also transformation, where coarse group membership lives, what each audit event records, and who owns the never-public field list. Co-Authored-By: Claude Opus 5 --- ADR5.md | 231 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 231 insertions(+) create mode 100644 ADR5.md diff --git a/ADR5.md b/ADR5.md new file mode 100644 index 00000000..2b3f1d80 --- /dev/null +++ b/ADR5.md @@ -0,0 +1,231 @@ +# ADR5: One Enforcement Engine, Two Grant Tables + +## Status + +Proposed. Decides PUB-D14 / PERM-D3, the engine-count fork named in the +whitepaper *Who May See What, and Who May Do What* (Confluence, Ocotillo space). +Nothing is built yet. This record exists so the grant table and the visibility +layer are not designed twice. + +## Summary + +Ocotillo needs to answer one question everywhere: *may this principal exercise +this capability within this scope?* The whitepaper argues that publication +consent (a landowner's agreement about their well) and internal permission (an +institution's trust in a person) reduce to the same grammar, and then leaves +open whether they should therefore share one policy engine. + +**Decision: two tables, one engine.** Landowner publication consent and internal +permission grants are stored separately, with separate governance, separate +admin surfaces, and separate audit semantics. Both are evaluated by a single +visibility layer and projected through a single field-allowlist chokepoint. No +service answers "is this published" on its own. + +The split is about *who decides and on what authority*. The join is about *where +the answer is computed*. Those are different questions, and the current codebase +already shows what happens when the second one is answered "in each view." + +## Context + +### Enforcement is currently distributed, and it has already drifted + +Publication today is a `release_status` column plus a filter re-written into +every consuming view. There are twelve of them — six public collections +(`ogc_locations`, `ogc_waterlevels`, `ogc_water_chemistry`, +`ogc_actively_monitored_wells`, `ogc_latest_tds_wells`, `ogc_project_areas`) and +six internal mirrors that deliberately carry non-public rows for authenticated +staff GIS clients. Each carries its own copy of the rule in SQL. + +Migration `baba91fe5e83` is the drift, in the repository, with a date on it: +`ogc_waterlevels` filtered on the reading's own `release_status` and not the +parent well's, so a well marked `draft` or `private` still published its public +readings through OGC API - EDR — with the well's name and coordinates attached. +`ogc_water_chemistry` had required the parent thing to be public since +`d9e0f1a2b3c4`. Two views, two answers to the same question, and the difference +was invisible until someone read both. + +That is not a bug in a view. It is what distributed enforcement does over time, +and every new destination multiplies it. The engine-count decision is really a +decision about whether that class of defect stays possible. + +### Publication is currently a bulk act, not a consent record + +`data_migrations/migrations/20260714_0001_publish_project_areas.py` publishes by +setting `release_status='public'` across a selection. That is the coarse +mechanism the whitepaper describes: association publishes everything about a +record, and an owner who would share water levels but not chemistry cannot be +represented at all. + +### The grant shape already exists twice + +`db/permission.py`'s `Permission` was landowner field-access consent — +`contact_id`, `allow_sampling`, `allow_installation`, an optional date range, a +polymorphic target (`permissible_id` / `permissible_type`). It was also dead: +unregistered in `db/__init__.py`, with no table in any migration and no caller. +It has been deleted, along with `PermissionMixin`. `PermissionHistory` +generalizes the same idea into a typed, lexicon-controlled `permission_type` +with `permission_allowed`, a required `start_date`, an optional `end_date`, and +`target_id` / `target_table`; it is now `FieldAccessConsent`, table +`field_access_consent`. Neither decides what an API caller may see. + +Both are consent about physical site access. Neither is authorization. The shape +recurring independently is evidence the shape is right; it is not an argument +for merging those tables into the access-control model. + +### Principals are already more than users + +`/ogcapi-internal` is gated at the ASGI layer by `core/internal_ogc_auth.py`, +which accepts an Authentik JWT carrying `OGCInternal` **or** a static API key, +because ArcGIS Pro cannot present a bearer token. That key is a principal with +no user behind it, and today its scope is "everything the internal mount +serves." Any model that assumes principal == person is already wrong here. + +### Two axes in one column + +San Acacia data is public and provisional at once; `automated_ingestion/ +ocotillo/loader.py` says so in a comment and works around it. `ReleaseMixin` +gave one lexicon column defaulting to `draft`, and that lexicon lists `draft`, +`public`, `private`, `published`, and `archived` — visibility — as siblings of +`provisional` and `final` — review state. Two axes, one column, so a record +could hold only one of the two answers. + +`ReleaseMixin` now carries `data_maturity` alongside `release_status`. This is +not a new vocabulary: the lexicon category (`provisional`, `in review`, +`approved`, described in `core/lexicon.json` as "orthogonal to release_status") +and the column already existed on `transducer_observation`, and the split +generalizes them rather than inventing a parallel flag. Grants will need the +second axis, because some destinations want approved data only. + +## Decision + +### 1. Two tables + +**`permission_grant`** — internal authorization. Principal (user subject, role, +API key), capability (read, enter, correct, administer), scope (project, thing, +data type, field group), time bounds, `granted_by`, `granted_at`, `reason`. +Governed by data services staff. Answers "is this person trusted with this." + +**`publication_consent`** — landowner-facing publication. One row per +(thing, destination, data type), against a destination registry that holds the +anonymous public, NGWMN, and partner agencies. Carries the consenting contact, +the date it was recorded, and who recorded it. Governed by the data owner and by +whoever took the phone call. Answers "did the owner agree to this." + +Neither table gets a NULL-as-wildcard data type. A grant names its types, so a +data type added next year is never published by an existing row. + +### 2. One engine + +A single visibility layer resolves both tables into one answer per request, and +a single serialization chokepoint applies the per-audience field allowlist and +its transformation hooks (coordinate rounding is the known case). The +never-public field list is enforced there and nowhere else. + +Grants are read from the database at request time. They are never encoded into +token claims, and expiry is checked at use, because immediate revocation is a +promise made to landowners and a claim baked into a token outlives it. + +### 3. Both tables feed one append-only authorization audit log + +Grant, revocation, consent capture, publication-configuration change, membership +change. The log is shared even though the tables are not, because the question +after an incident — "who granted that, and when" — does not care which table the +row came from. `AuditMixin` and sqlalchemy-continuum cover data attribution and +data history; this is a separate structure, written from the application, with a +database-level backstop for writes that bypass it. + +## Why not the alternatives + +**One table.** Cheapest to enforce, and the enforcement argument is already won +by the shared engine, so the merge buys little. What it costs is governance: a +landowner's consent and a staff member's clearance would live in one table, one +admin screen, and one review path, decided by different people for different +reasons and with different consequences for being wrong. Revoking consent is a +phone call honored immediately; revoking clearance is an HR-shaped event. The +whitepaper's caution — that consent authoring and permission granting need +visible separation even if they share a surface — is easier to keep with a +schema that separates them than with a column that distinguishes them. + +**Two engines.** Cleanest governance separation, and it reintroduces exactly the +`baba91fe5e83` failure: two independent implementations of "is this published," +diverging quietly, with an outside party finding the difference first. Ocotillo +has one read path per record; it should have one answer. + +## What this does not decide + +- **Data type granularity** (PUB-U11). Water levels versus chemistry is + required. Per-analyte grants may be over-engineering. The admin screen decides + this, not the schema. +- **Whether field sensitivity is global or per destination**, and whether + protection is removal only or also transformation (PUB-U7 / PUB-U8). The + chokepoint supports both; which is configured is open. +- **What happens to rows already marked `release_status='provisional'`.** The + two axes now exist, but no data moves between them. Rewriting those rows as + a level plus `data_maturity='provisional'` needs someone to say what level + each of them meant, and data migrations have no CD path in this repo — they + are run by hand. +- **Whether `provisional` and `final` should leave the `release_status` + lexicon.** Removing them is what makes the split enforceable rather than + conventional, and it cannot happen before the row migration above. +- **Where coarse group membership lives**, Authentik or Ocotillo (PERM-D9). + Authentik roles become role principals with broad grants either way, and + nobody's access changes on the day the tables land. +- **What each audit event records** (PERM-U11 / PERM-U13). Before-and-after + capture forces an event-based write path; whether that rigidity is required + depends on a compliance driver nobody has yet identified. +- **Who owns the never-public field list.** A policy decision with a named + owner, and a prerequisite to anything shipping externally. Engineering + guarantees the list is enforced; it cannot decide what is on it. +- **Healy migration** (PUB-D13). Grandfathering wells as full consent preserves + today's behavior; re-consenting per data type honors the model. The data owner + chooses, explicitly, and it happens last. + +## Consequences + +**Good.** New destinations become a registry row plus grants, not a new pipeline +with its own copy of the rules. A field nobody approved is invisible outside the +Bureau by default, including fields added later. The correctness burden +concentrates in one layer that can be reviewed, tested, and monitored, instead +of in the next twelve views. A staff member can tell a well owner what is shared, +per kind of data, and revoke it from a screen. + +**Cost.** Two tables to keep coherent where one would do, and a shared engine +that must not leak one table's governance into the other's admin surface. The +visibility layer sits on the read path of every service, so its query shape +matters: grants are few and cacheable per principal, but an unbounded cache +silently defeats the immediate-revocation promise. Cache with short TTLs and +explicit invalidation on revoke. + +**Honesty requirement.** For a harvesting destination, revoking consent means +the data stops being offered. Copies already harvested live in someone else's +system. "Unpublish" means "stop offering," and that is what owners should be +told. + +## Sequencing implied by this decision + +1. This record (here). +2. **Done.** `permission_history` is `field_access_consent`, dead `Permission` + and `PermissionMixin` are gone, and `ReleaseMixin` carries `release_status` + (level) plus `data_maturity` (review state). +3. `permission_grant`, `publication_consent`, destination registry, and the + visibility layer, behind **one** service before it is behind all of them. +4. Field projection at the serialization chokepoint, with the never-public list. +5. Console administration. +6. Healy migration, after the data owner decides grandfathering. + +## References + +- Confluence, Ocotillo space: *Who May See What, and Who May Do What: A + scope-based, attribute-level access control system for Ocotillo* (whitepaper + this record answers), *Ocotillo Needs an Operator Console*, *Authentik + Access-Control Matrix Summary and Role Definitions*, *User Research Outcomes: + Ocotillo Permissions Interview with Ethan Mamer* +- [ADR3](ADR3.md): OGC API - EDR collections backed by publication-filtered views +- [ADR4](ADR4.md): the `domain/` layer the grant-evaluation rules belong in +- [db/field_access_consent.py](db/field_access_consent.py): landowner + field-access consent, and the in-house precedent for the grant shape +- [db/base.py](db/base.py): `ReleaseMixin`, `AuditMixin`, versioning wiring +- [alembic/versions/baba91fe5e83_gate_ogc_waterlevels_on_thing_release.py](alembic/versions/baba91fe5e83_gate_ogc_waterlevels_on_thing_release.py): + the drift this decision is meant to make impossible +- [core/internal_ogc_auth.py](core/internal_ogc_auth.py) and + [docs/internal-ogc-desktop-gis.md](docs/internal-ogc-desktop-gis.md): the API-key principal From b07d968bf3609dc3e34f5c8987d942d9676992cb Mon Sep 17 00:00:00 2001 From: jakeross Date: Mon, 24 Aug 2026 22:02:04 -0700 Subject: [PATCH 02/13] refactor(db): name field-access consent honestly, split release state Two vocabulary fixes from ADR5. Neither changes an API response shape. The table named permission_history records a landowner's consent to physical site access -- sampling, equipment installation -- which is a domain fact, not authorization. It is now field_access_consent (model FieldAccessConsent). The permission_type and permission_allowed columns keep their names, since permission_type is lexicon-backed and both are published on Thing responses under the unchanged `permissions` key. db/permission.py's Permission and base.py's PermissionMixin are deleted. They were dead: never imported in db/__init__.py, no `permission` table in any migration, no callers. release_status was one column carrying two axes -- its lexicon lists draft, public, private, published and archived next to provisional and final -- so a record could not be public and provisional at once, which San Acacia data is. ReleaseMixin now carries data_maturity alongside it. That is not a new vocabulary: the lexicon category (provisional, in review, approved) and the column already existed on transducer_observation, whose duplicate declaration is removed. Nullable, NULL meaning not stated. No data moves. Rows already marked release_status='provisional' are left alone; reassigning them across the two axes needs a decision per row and is tracked in ADR5. Migration e7c1a9f4b2d8 renames the table (with its primary key constraint and sequence, so the old name stops appearing in errors) and adds data_maturity to the 34 other ReleaseMixin tables and the 7 continuum version tables. Verified down and back up against ocotilloapi_test. Co-Authored-By: Claude Opus 5 --- CLAUDE.md | 19 ++ ..._field_access_consent_and_data_maturity.py | 170 ++++++++++++++++++ api/well_inventory.py | 2 +- db/__init__.py | 2 +- db/base.py | 47 +++-- db/contact.py | 6 +- ...ion_history.py => field_access_consent.py} | 47 +++-- db/notes.py | 2 + db/permission.py | 81 --------- db/thing.py | 8 +- db/transducer.py | 11 +- schemas/__init__.py | 6 +- ...ion_history.py => field_access_consent.py} | 5 +- schemas/thing.py | 8 +- services/sample_helper.py | 2 +- services/scoped_transfer.py | 10 +- services/util.py | 2 +- services/well_inventory_csv.py | 10 +- tests/features/environment.py | 16 +- .../steps/well-additional-information.py | 6 +- tests/test_thing.py | 1 + transfers/metrics.py | 4 +- transfers/permissions_transfer.py | 8 +- transfers/transfer_results_builder.py | 22 +-- transfers/transfer_results_specs.py | 4 +- 25 files changed, 309 insertions(+), 190 deletions(-) create mode 100644 alembic/versions/e7c1a9f4b2d8_field_access_consent_and_data_maturity.py rename db/{permission_history.py => field_access_consent.py} (60%) delete mode 100644 db/permission.py rename schemas/{permission_history.py => field_access_consent.py} (77%) diff --git a/CLAUDE.md b/CLAUDE.md index a5f9358c..816dba95 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -215,6 +215,25 @@ the pinned pygeoapi version — most sharply, `BaseProvider.fields` returns **`docs/ogc-field-descriptions.md`** before changing field metadata or upgrading pygeoapi. +### Access control and release state + +**`ADR5.md`** decides the shape of the access-control work: two grant tables +(internal permission vs landowner publication consent), one visibility layer, +one field-projection chokepoint. Nothing is built yet. + +Two vocabulary fixes from it have landed and matter when reading models: + +- **`db/field_access_consent.py`** (`FieldAccessConsent`, table + `field_access_consent`, formerly `PermissionHistory` / `permission_history`) + is a landowner's consent to *physical site access*. It is not authorization. + Its `permission_type` / `permission_allowed` columns keep their names, and + Thing responses still publish it under the `permissions` key. +- **`ReleaseMixin` has two axes**: `release_status` is the release *level* + (who may see it) and `data_maturity` is the review state (`provisional`, + `in review`, `approved`, NULL = not stated). The `release_status` lexicon + still lists `provisional` and `final` for historical rows; new code should + put review state in `data_maturity`. + ### Database Configuration The application supports two database modes (configured via `DB_DRIVER` in `.env`): diff --git a/alembic/versions/e7c1a9f4b2d8_field_access_consent_and_data_maturity.py b/alembic/versions/e7c1a9f4b2d8_field_access_consent_and_data_maturity.py new file mode 100644 index 00000000..63f75e24 --- /dev/null +++ b/alembic/versions/e7c1a9f4b2d8_field_access_consent_and_data_maturity.py @@ -0,0 +1,170 @@ +"""rename permission_history to field_access_consent; add data_maturity + +Two vocabulary fixes from ADR5, neither of which changes any API response +shape: + +1. ``permission_history`` records a landowner's consent to physical site + access. It is a domain fact, not authorization, and the name implied + otherwise. It becomes ``field_access_consent``. The ``permission_type`` + and ``permission_allowed`` columns keep their names -- ``permission_type`` + is lexicon-backed, and both are published in Thing responses. + +2. ``release_status`` was one column carrying two axes. Its lexicon lists + `draft`, `public`, `private`, `published`, `archived` (visibility) next to + `provisional` and `final` (review state), so a record could not be public + and provisional at once -- which San Acacia data is (see + automated_ingestion/ocotillo/loader.py). Release *level* stays in + ``release_status``; review state moves to ``data_maturity``. + + ``data_maturity`` is not a new vocabulary. The lexicon category + (`provisional`, `in review`, `approved`) and the column already existed on + ``transducer_observation``; this generalizes them onto every ReleaseMixin + table, so ``transducer_observation`` is deliberately absent from the list + below -- it already has the column, unchanged. + +``data_maturity`` is nullable and starts NULL, meaning "not stated". No +existing row changes meaning. Rows that carry `release_status='provisional'` +are left alone; splitting them onto the two axes is a data migration with its +own decision to make. + +Revision ID: e7c1a9f4b2d8 +Revises: c9d0e1f2a3b4 +Create Date: 2026-08-24 00:00:00.000000 + +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "e7c1a9f4b2d8" +down_revision: Union[str, Sequence[str], None] = "c9d0e1f2a3b4" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +# Every table backed by a model that mixes in ReleaseMixin, with +# permission_history already under its new name and transducer_observation +# omitted because it already carries data_maturity. +RELEASE_TABLES = ( + "address", + "analysis_method", + "aquifer_system", + "aquifer_type", + "asset", + "contact", + "data_provenance", + "deployment", + "email", + "field_access_consent", + "field_activity", + "field_event", + "field_event_participant", + "geologic_formation", + "group", + "location", + "measuring_point_history", + "monitoring_frequency_history", + "notes", + "observation", + "parameter", + "phone", + "regulatory_limit", + "sample", + "sensor", + "status_history", + "thing", + "thing_aquifer_association", + "thing_geologic_formation_association", + "thing_id_link", + "transducer_observation_block", + "well_casing_material", + "well_purpose", + "well_screen", +) + +# sqlalchemy-continuum mirrors every tracked column into the version table, so +# the models carrying __versioned__ need the new column there too. Version rows +# are written explicitly and historical rows have no answer, so it is nullable. +VERSION_TABLES = ( + "aquifer_system_version", + "geologic_formation_version", + "location_version", + "observation_version", + "parameter_version", + "regulatory_limit_version", + "thing_version", +) + + +def upgrade() -> None: + op.rename_table("permission_history", "field_access_consent") + # rename_table leaves the constraint and sequence names behind, which is + # cosmetic but makes the old name resurface in every error message. + op.execute( + sa.text( + "ALTER TABLE field_access_consent " + "RENAME CONSTRAINT permission_history_pkey TO field_access_consent_pkey" + ) + ) + op.execute( + sa.text( + "ALTER SEQUENCE IF EXISTS permission_history_id_seq " + "RENAME TO field_access_consent_id_seq" + ) + ) + + for table in RELEASE_TABLES: + op.add_column( + table, + sa.Column( + "data_maturity", + sa.String(length=100), + nullable=True, + comment="Review state; orthogonal to release_status", + ), + ) + op.create_foreign_key( + f"{table}_data_maturity_fkey", + table, + "lexicon_term", + ["data_maturity"], + ["term"], + onupdate="CASCADE", + ) + + for table in VERSION_TABLES: + op.add_column( + table, + sa.Column( + "data_maturity", + sa.String(length=100), + autoincrement=False, + nullable=True, + ), + ) + + +def downgrade() -> None: + for table in VERSION_TABLES: + op.drop_column(table, "data_maturity") + + for table in RELEASE_TABLES: + op.drop_constraint(f"{table}_data_maturity_fkey", table, type_="foreignkey") + op.drop_column(table, "data_maturity") + + op.execute( + sa.text( + "ALTER SEQUENCE IF EXISTS field_access_consent_id_seq " + "RENAME TO permission_history_id_seq" + ) + ) + op.execute( + sa.text( + "ALTER TABLE field_access_consent " + "RENAME CONSTRAINT field_access_consent_pkey TO permission_history_pkey" + ) + ) + op.rename_table("field_access_consent", "permission_history") diff --git a/api/well_inventory.py b/api/well_inventory.py index 46138a8f..64b37af4 100644 --- a/api/well_inventory.py +++ b/api/well_inventory.py @@ -45,7 +45,7 @@ # FieldEventParticipant, # FieldActivity, # Contact, -# PermissionHistory, +# FieldAccessConsent, # Thing, # ) # from schemas.thing import CreateWell diff --git a/db/__init__.py b/db/__init__.py index 4e2e7fb3..5eeb02fc 100644 --- a/db/__init__.py +++ b/db/__init__.py @@ -48,7 +48,7 @@ from db.notes import * from db.observation import * from db.parameter import * -from db.permission_history import * +from db.field_access_consent import * from db.publication import * from db.regulatory_limit import * from db.sample import * diff --git a/db/base.py b/db/base.py index e9b0d7f2..e909ce36 100644 --- a/db/base.py +++ b/db/base.py @@ -29,7 +29,7 @@ - `ReleaseMixin`: Adds a release status column referencing the `lexicon_term` table. - `AuditMixin`: Adds standard audit columns (created_at, created_by, updated_at, updated_by). 5. A simple `User` model for tracking user information in audit columns. -6. Polymorphic helper mixins (`StatusHistoryMixin`, `NotesMixin`, `DataProvenanceMixin`, `PermissionMixin`.) +6. Polymorphic helper mixins (`StatusHistoryMixin`, `NotesMixin`, `DataProvenanceMixin`.) which provide a clean, reusable way to add relationships to the polymorphic metadata tables. Any model that can have a status history (like Thing or Location) can simply inherit from the `StatusHistoryMixin` mixin. @@ -53,7 +53,6 @@ declared_attr, Mapped, mapped_column, - relationship, ) from sqlalchemy_continuum import make_versioned from sqlalchemy_searchable import make_searchable @@ -95,12 +94,32 @@ def pascal_to_snake(name): # ============= Common Mixins ============================================= class ReleaseMixin: - """Mixin to add release status to a model.""" + """Mixin to add release state to a model. + + Two axes, deliberately separate (ADR5): + + * ``release_status`` is the release *level* -- who may see the record at + all (draft, public, private, ...), lexicon-backed. + * ``data_maturity`` is how far through review the record is, on USGS + terms: provisional, in review, approved. It is orthogonal to the level: + San Acacia data is published and provisional at the same time, and one + column could not say both, because ``release_status``'s lexicon lists + those as siblings. + + ``data_maturity`` is nullable and NULL means not stated, which is honest + for the rows that predate it. The vocabulary and the column originated on + ``TransducerObservation``; this mixin is where it now lives so that every + released record can carry the second axis. + """ @declared_attr def release_status(self): return lexicon_term(default="draft") + @declared_attr + def data_maturity(self): + return lexicon_term(nullable=True) + class AuditMixin: """Mixin to add standard audit columns to a model.""" @@ -176,28 +195,6 @@ def properties(self): ) -# ============= Polymorphic Helper Mixins ============================================= - - -class PermissionMixin: - """ - Mixin for models that can have permissions (e.g., Thing, Location). - It automatically creates a polymorphic One-to-Many relationship to the - Permission table. - """ - - @declared_attr - def permissions(self): - # One-to-Many polymorphic relationship - return relationship( - "Permission", - primaryjoin=f"and_({self.__name__}.id==foreign(Permission.permissible_id), " - f"Permission.permissible_type=='{self.__name__}')", - lazy="selectin", - viewonly=True, - ) - - class User(Base): """Represents a user in the system.""" diff --git a/db/contact.py b/db/contact.py index 0fb59473..f1d68c41 100644 --- a/db/contact.py +++ b/db/contact.py @@ -27,7 +27,7 @@ from db.field import FieldEventParticipant, FieldEvent from db.thing import Thing from db.publication import Author, AuthorContactAssociation - from db.permission_history import PermissionHistory + from db.field_access_consent import FieldAccessConsent class ThingContactAssociation(Base, AutoBaseMixin): @@ -75,8 +75,8 @@ class Contact(Base, AutoBaseMixin, ReleaseMixin, NotesMixin): ) # One-To-Many: A Contact can grant many Permissions. - permissions: Mapped[List["PermissionHistory"]] = relationship( - "PermissionHistory", + permissions: Mapped[List["FieldAccessConsent"]] = relationship( + "FieldAccessConsent", back_populates="contact", cascade="all, delete, delete-orphan", ) diff --git a/db/permission_history.py b/db/field_access_consent.py similarity index 60% rename from db/permission_history.py rename to db/field_access_consent.py index 0e1526e2..3a0ef09d 100644 --- a/db/permission_history.py +++ b/db/field_access_consent.py @@ -1,10 +1,17 @@ """ -models/permission.py +db/field_access_consent.py -This model defines the `Permission` table, a polymorphic table that tracks -all legal and administrative agreements related to site access and activity. -Its purpose is to track who granted permission, what activities they authorized, -which entity the permission applies to, and for what period of time. +Landowner consent to physical site access, recorded per Thing or Location. + +This is a domain fact, not authorization. A row says a contact agreed to let +the Bureau do something at their well -- sample it, install equipment -- for +some period of time. Nothing here decides what an API caller may see or write; +that is the grant model described in ADR5, which is a separate table with +separate governance. + +The table was named `permission_history` until ADR5. The `permission_type` +and `permission_allowed` column names are kept because `permission_type` is +lexicon-backed and both are published in Thing responses as-is. """ from datetime import date @@ -21,10 +28,12 @@ from db.location import Location -class PermissionHistory(Base, AutoBaseMixin, ReleaseMixin): +class FieldAccessConsent(Base, AutoBaseMixin, ReleaseMixin): """ - Represents a specific grant of permission from a Contact for a - specific entity (e.g., a Thing or Location). + One consent record: a Contact agreed (or declined) to a type of field + activity at a specific entity (a Thing or a Location), over a date range. + + Not an access-control grant. See ADR5. """ # --- Foreign Keys --- @@ -52,14 +61,14 @@ class PermissionHistory(Base, AutoBaseMixin, ReleaseMixin): # They tell SQLAlchemy exactly how to find the specific parent record for a given child. _thing_target: Mapped["Thing"] = relationship( "Thing", - primaryjoin="and_(foreign(PermissionHistory.target_id) == Thing.id, " - "PermissionHistory.target_table == 'thing')", + primaryjoin="and_(foreign(FieldAccessConsent.target_id) == Thing.id, " + "FieldAccessConsent.target_table == 'thing')", viewonly=True, ) _location_target: Mapped["Location"] = relationship( "Location", - primaryjoin="and_(foreign(PermissionHistory.target_id) == Location.id, " - "PermissionHistory.target_table == 'location')", + primaryjoin="and_(foreign(FieldAccessConsent.target_id) == Location.id, " + "FieldAccessConsent.target_table == 'location')", viewonly=True, ) @@ -73,22 +82,22 @@ def target(self): return getattr(self, f"_{self.target_table}_target") -class PermissionHistoryMixin: +class FieldAccessConsentMixin: """ - Mixin for models that can have permissions (e.g., Thing, Location). + Mixin for models a landowner can consent about (e.g., Thing, Location). It automatically creates a polymorphic One-to-Many relationship to the - Permission table. + field_access_consent table. """ @declared_attr - def permission_history(cls): + def field_access_consent(cls): # One-to-Many polymorphic relationship return relationship( - "PermissionHistory", + "FieldAccessConsent", primaryjoin=( and_( - cls.id == foreign(PermissionHistory.target_id), - PermissionHistory.target_table == cls.__tablename__, + cls.id == foreign(FieldAccessConsent.target_id), + FieldAccessConsent.target_table == cls.__tablename__, ) ), lazy="selectin", diff --git a/db/notes.py b/db/notes.py index 3c238fbf..0a38b53f 100644 --- a/db/notes.py +++ b/db/notes.py @@ -110,6 +110,7 @@ def add_note( content: str, note_type: str, release_status: str = "draft", + data_maturity: str = None, created_by: str = None, ) -> Notes: """ @@ -123,6 +124,7 @@ def add_note( target_id=self.id, target_table=self.__class__.__tablename__, release_status=release_status, + data_maturity=data_maturity, ) def _get_notes(self, note_type: str) -> list[Notes]: diff --git a/db/permission.py b/db/permission.py deleted file mode 100644 index c4ea2c85..00000000 --- a/db/permission.py +++ /dev/null @@ -1,81 +0,0 @@ -""" -models/permission.py - -This model defines the `Permission` table, a polymorphic table that tracks -all legal and administrative agreements related to site access and activity. -Its purpose is to track who granted permission, what activities they authorized, -which entity the permission applies to, and for what period of time. -""" - -from typing import TYPE_CHECKING - -from sqlalchemy import ( - Integer, - ForeignKey, - String, - Boolean, - Date, - Text, -) -from sqlalchemy.orm import relationship, Mapped, mapped_column - -from db.base import Base, AutoBaseMixin, ReleaseMixin - -if TYPE_CHECKING: - from db.contact import Contact - from db.thing import Thing - from db.location import Location - - -class Permission(Base, AutoBaseMixin, ReleaseMixin): - """ - Represents a specific grant of permission from a Contact for a - specific entity (e.g., a Thing or Location). - """ - - # --- Foreign Keys --- - contact_id: Mapped[int] = mapped_column( - Integer, ForeignKey("contact.id"), nullable=False - ) - - # --- Columns --- - allow_sampling: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) - allow_installation: Mapped[bool] = mapped_column( - Boolean, nullable=False, default=False - ) - start_date: Mapped[Date] = mapped_column(Date, nullable=True) - end_date: Mapped[Date] = mapped_column(Date, nullable=True) - notes: Mapped[str] = mapped_column(Text, nullable=True) - - # --- Polymorphic Columns --- - permissible_id: Mapped[int] = mapped_column(Integer, nullable=False) - permissible_type: Mapped[str] = mapped_column(String(50), nullable=False) - - # --- Relationships --- - # Many-To-One: A Permission is granted by one Contact. - contact: Mapped["Contact"] = relationship("Contact", back_populates="permissions") - - # --- Polymorphic Parent Relationships (Internal) --- - # These are view-only relationships used by the 'target' property below. - # They tell SQLAlchemy exactly how to find the specific parent record for a given child. - _thing_target: Mapped["Thing"] = relationship( - "Thing", - primaryjoin="and_(foreign(Permission.permissible_id) == Thing.id, " - "Permission.permissible_type == 'Thing')", - viewonly=True, - ) - _location_target: Mapped["Location"] = relationship( - "Location", - primaryjoin="and_(foreign(Permission.permissible_id) == Location.id, " - "Permission.permissible_type == 'Location')", - viewonly=True, - ) - - @property - def target(self): - """ - A generic property to get the parent object (Thing, Location, etc.). - This is useful for simplifying application code by providing a single, - consistent way to access the parent of a polymorphic record. - """ - return getattr(self, f"_{self.permissible_type.lower()}_target") diff --git a/db/thing.py b/db/thing.py index 01aed09e..8f15cf06 100644 --- a/db/thing.py +++ b/db/thing.py @@ -30,7 +30,7 @@ ) from db.data_provenance import DataProvenanceMixin from db.measuring_point_history import MeasuringPointHistory -from db.permission_history import PermissionHistoryMixin +from db.field_access_consent import FieldAccessConsentMixin from db.status_history import StatusHistoryMixin from services.util import retrieve_latest_polymorphic_history_table_record @@ -63,7 +63,7 @@ class Thing( AutoBaseMixin, ReleaseMixin, StatusHistoryMixin, - PermissionHistoryMixin, + FieldAccessConsentMixin, DataProvenanceMixin, NotesMixin, ): @@ -598,8 +598,8 @@ def permissions(self) -> list: associated permissions, an empty list is returned instead of None to allow the API to serialize correctly (see schemas/thing.py). """ - if self.permission_history: - return self.permission_history + if self.field_access_consent: + return self.field_access_consent else: return [] diff --git a/db/transducer.py b/db/transducer.py index e129cfe6..699f7c65 100644 --- a/db/transducer.py +++ b/db/transducer.py @@ -180,14 +180,9 @@ class TransducerObservation(Base, AutoBaseMixin, ReleaseMixin): comment="Per-reading correction annotation; NULL means the value is as measured", ) - # How far through review this reading is, on USGS terms: provisional, - # in review, approved. Orthogonal to `release_status`, which says who may - # see it -- a reading can be public and provisional at once, which one - # column could not express because its lexicon lists those as siblings. - # - # Nullable because legacy rows predate it and nobody has established - # whether they are approved. NULL means not stated, which is honest. - data_maturity: Mapped[str] = lexicon_term(nullable=True) + # data_maturity (provisional / in review / approved) comes from + # ReleaseMixin, which is where the axis was generalized in ADR5. It started + # here; the semantics are unchanged. nma_waterlevelscontinuous_pressure_conddl_ms_cm: Mapped[float] = mapped_column( Float, nullable=True ) diff --git a/schemas/__init__.py b/schemas/__init__.py index 5a1d85af..de41b888 100644 --- a/schemas/__init__.py +++ b/schemas/__init__.py @@ -16,7 +16,7 @@ from datetime import datetime, timezone, date from typing import Annotated -from core.enums import ReleaseStatus +from core.enums import DataMaturity, ReleaseStatus from pydantic import ( BaseModel, ConfigDict, @@ -36,6 +36,9 @@ class ResourceNotFoundResponse(BaseModel): class BaseCreateModel(BaseModel): release_status: ReleaseStatus = "draft" + # Orthogonal to release_status: data can be published and provisional at + # the same time (ADR5). NULL means not stated. + data_maturity: DataMaturity | None = None @field_validator("release_status", mode="before") @classmethod @@ -113,6 +116,7 @@ class BaseResponseModel(BaseModel): id: int # every ORM model should have an id field created_at: UTCAwareDatetime release_status: ReleaseStatus + data_maturity: DataMaturity | None = None model_config = ConfigDict( from_attributes=True, diff --git a/schemas/permission_history.py b/schemas/field_access_consent.py similarity index 77% rename from schemas/permission_history.py rename to schemas/field_access_consent.py index d8f1f3ef..45d83d75 100644 --- a/schemas/permission_history.py +++ b/schemas/field_access_consent.py @@ -5,8 +5,11 @@ # ------ RESPONSE ---------- -class PermissionHistoryResponse(BaseModel): +class FieldAccessConsentResponse(BaseModel): """ + Landowner field-access consent, as published on a Thing. Not an + access-control grant (ADR5). + Even though permission_allowed and start_date are not-nullable in the database, they are nullable here to accommodate cases where no permission record exists for a given permission type. diff --git a/schemas/thing.py b/schemas/thing.py index c2798b5f..86942e37 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -35,7 +35,7 @@ from schemas.group import GroupResponse from schemas.location import LocationGeoJSONResponse from schemas.notes import NoteResponse, CreateNote -from schemas.permission_history import PermissionHistoryResponse +from schemas.field_access_consent import FieldAccessConsentResponse # -------- VALIDATE ---------- @@ -274,7 +274,7 @@ class WellResponse(BaseThingResponse): water_notes: list[NoteResponse] = [] construction_notes: list[NoteResponse] = [] contacts: list[WellContactSummaryResponse] = [] - permissions: list[PermissionHistoryResponse] + permissions: list[FieldAccessConsentResponse] formation_completion_code: FormationCode | None nma_formation_zone: str | None well_location_note: list[str] = [] @@ -299,7 +299,7 @@ def populate_well_casing_materials_with_strings(cls, well_casing_materials): return materials @field_validator("permissions", mode="before") - def populate_permission_history_with_latest_records(cls, permissions): + def populate_field_access_consent_with_latest_records(cls, permissions): """ Populate the permission history with the latest records for each type of permission. If multiple records exist for the same permission type @@ -326,7 +326,7 @@ def populate_permission_history_with_latest_records(cls, permissions): permissions_to_return.append(latest_record) else: permissions_to_return.append( - PermissionHistoryResponse( + FieldAccessConsentResponse( permission_type=permission_type, permission_allowed=None, start_date=None, diff --git a/services/sample_helper.py b/services/sample_helper.py index 0f25dd9a..6b4a5bb5 100644 --- a/services/sample_helper.py +++ b/services/sample_helper.py @@ -41,7 +41,7 @@ GroupThingAssociation.group ), THING_RESPONSE_BASE.selectinload(Thing.notes), - THING_RESPONSE_BASE.selectinload(Thing.permission_history), + THING_RESPONSE_BASE.selectinload(Thing.field_access_consent), THING_RESPONSE_BASE.selectinload(Thing.data_provenance), THING_RESPONSE_BASE.selectinload(Thing.status_history), ) diff --git a/services/scoped_transfer.py b/services/scoped_transfer.py index 39749de2..6dac0aaf 100644 --- a/services/scoped_transfer.py +++ b/services/scoped_transfer.py @@ -31,7 +31,7 @@ NMA_Chemistry_SampleInfo, Notes, Observation, - PermissionHistory, + FieldAccessConsent, Sample, Thing, ThingContactAssociation, @@ -1393,11 +1393,11 @@ def _execute_permissions(pointids: list[str]) -> ScopedFamilyResult: existing_permissions = { (target_id, contact_id, permission_type) for target_id, contact_id, permission_type in session.query( - PermissionHistory.target_id, - PermissionHistory.contact_id, - PermissionHistory.permission_type, + FieldAccessConsent.target_id, + FieldAccessConsent.contact_id, + FieldAccessConsent.permission_type, ) - .filter(PermissionHistory.target_table == "thing") + .filter(FieldAccessConsent.target_table == "thing") .all() } diff --git a/services/util.py b/services/util.py index dbf88f98..ad1b8b84 100644 --- a/services/util.py +++ b/services/util.py @@ -267,7 +267,7 @@ def retrieve_latest_polymorphic_history_table_record( DeclarativeBase | None The latest record from the specified polymorphic table with the defined type if it exists. """ - if polymorphic_relationship == "permission_history": + if polymorphic_relationship == "field_access_consent": type_field = "permission_type" elif polymorphic_relationship == "status_history": type_field = "status_type" diff --git a/services/well_inventory_csv.py b/services/well_inventory_csv.py index ccb2863b..c041373d 100644 --- a/services/well_inventory_csv.py +++ b/services/well_inventory_csv.py @@ -38,7 +38,7 @@ FieldEventParticipant, FieldActivity, Contact, - PermissionHistory, + FieldAccessConsent, Thing, ThingContactAssociation, Sample, @@ -425,11 +425,11 @@ def _make_well_permission( permission_type: str, permission_allowed: bool, start_date: date, -) -> PermissionHistory: +) -> FieldAccessConsent: """ - Makes a PermissionHistory record for the given well and contact. + Makes a FieldAccessConsent record for the given well and contact. If the contact has not been provided, but a permission is to be created, - no PermissionHistory record is created and a 400 error is raised. + no FieldAccessConsent record is created and a 400 error is raised. """ if contact is None: raise PydanticStyleException( @@ -444,7 +444,7 @@ def _make_well_permission( ], ) - permission = PermissionHistory( + permission = FieldAccessConsent( target_table="thing", target_id=well.id, contact=contact, diff --git a/tests/features/environment.py b/tests/features/environment.py index d3c1b47c..c0af8423 100644 --- a/tests/features/environment.py +++ b/tests/features/environment.py @@ -39,7 +39,7 @@ Deployment, TransducerObservationBlock, WellCasingMaterial, - PermissionHistory, + FieldAccessConsent, StatusHistory, ThingIdLink, WellPurpose, @@ -247,7 +247,7 @@ def add_contact(context, session): @add_context_object_container("permission_histories") -def add_permission_history( +def add_field_access_consent( context, session, contact_id, @@ -259,7 +259,7 @@ def add_permission_history( target_id, target_table, ): - permission_history = PermissionHistory( + field_access_consent = FieldAccessConsent( contact_id=contact_id, permission_type=permission_type, permission_allowed=permission_allowed, @@ -269,12 +269,12 @@ def add_permission_history( target_id=target_id, target_table=target_table, ) - session.add(permission_history) + session.add(field_access_consent) session.commit() - session.refresh(permission_history) + session.refresh(field_access_consent) - context.objects["permission_histories"].append(permission_history) - return permission_history + context.objects["permission_histories"].append(field_access_consent) + return field_access_consent @add_context_object_container("sensors") @@ -702,7 +702,7 @@ def before_all(context): "Water Level Sample", "Water Chemistry Sample", ]: - add_permission_history( + add_field_access_consent( context, session, contact_id=context.objects["contacts"][0].id, diff --git a/tests/features/steps/well-additional-information.py b/tests/features/steps/well-additional-information.py index c34f17b6..e3e88e24 100644 --- a/tests/features/steps/well-additional-information.py +++ b/tests/features/steps/well-additional-information.py @@ -14,7 +14,7 @@ def step_step_step(context): assert "permissions" in context.water_well_data permission_record = retrieve_latest_polymorphic_history_table_record( - context.objects["wells"][0], "permission_history", permission_type + context.objects["wells"][0], "field_access_consent", permission_type ) water_well_data_permissions = [ @@ -49,7 +49,7 @@ def step_then_the_response_should_include_whether_sampling_permission_is_granted assert "permissions" in context.water_well_data permission_record = retrieve_latest_polymorphic_history_table_record( - context.objects["wells"][0], "permission_history", permission_type + context.objects["wells"][0], "field_access_consent", permission_type ) water_well_data_permissions = [ @@ -84,7 +84,7 @@ def step_step_step_2(context): assert "permissions" in context.water_well_data permission_record = retrieve_latest_polymorphic_history_table_record( - context.objects["wells"][0], "permission_history", permission_type + context.objects["wells"][0], "field_access_consent", permission_type ) water_well_data_permissions = [ diff --git a/tests/test_thing.py b/tests/test_thing.py index 8c8859c4..3e1ebb3d 100644 --- a/tests/test_thing.py +++ b/tests/test_thing.py @@ -927,6 +927,7 @@ def test_get_water_wells_includes_contact_summary( "id": contact.id, "created_at": contact.created_at.astimezone(timezone.utc).strftime(DT_FMT), "release_status": contact.release_status, + "data_maturity": contact.data_maturity, "name": contact.name, "organization": contact.organization, "contact_type": contact.contact_type, diff --git a/transfers/metrics.py b/transfers/metrics.py index 456e9b48..0d306d73 100644 --- a/transfers/metrics.py +++ b/transfers/metrics.py @@ -34,7 +34,7 @@ TransducerObservation, Group, Asset, - PermissionHistory, + FieldAccessConsent, ThingGeologicFormationAssociation, NMA_Stratigraphy, NMA_FieldParameters, @@ -171,7 +171,7 @@ def weather_data_metrics(self, *args, **kw) -> None: self._handle_metrics(NMA_WeatherData, name="WeatherData", *args, **kw) def permissions_metrics(self, *args, **kw) -> None: - self._handle_metrics(PermissionHistory, *args, **kw) + self._handle_metrics(FieldAccessConsent, *args, **kw) def stratigraphy_metrics(self, *args, **kw) -> None: self._handle_metrics(ThingGeologicFormationAssociation, *args, **kw) diff --git a/transfers/permissions_transfer.py b/transfers/permissions_transfer.py index 346e9f14..6da629b6 100644 --- a/transfers/permissions_transfer.py +++ b/transfers/permissions_transfer.py @@ -4,7 +4,7 @@ from pandas import isna from sqlalchemy.orm import Session -from db import Thing, PermissionHistory, Contact, ThingContactAssociation +from db import Thing, FieldAccessConsent, Contact, ThingContactAssociation from transfers.util import read_csv, logger, replace_nans, chunk_by_size """ @@ -18,7 +18,7 @@ def _make_permission( wdf, well, contact_id, nma_field, permission_type -) -> PermissionHistory | None: +) -> FieldAccessConsent | None: values = wdf.loc[wdf["PointID"] == well.name, nma_field].values if len(values) == 0: @@ -27,7 +27,7 @@ def _make_permission( return None permission_allowed = bool(values[0]) - permission = PermissionHistory( + permission = FieldAccessConsent( contact_id=contact_id, permission_type=permission_type, permission_allowed=permission_allowed, @@ -48,7 +48,7 @@ def transfer_permissions(session: Session) -> None: The transferred wells and contacts need to be transferred first - to access the auto-generated well IDs - to know who gave permission to which well since contact_id is required for - PermissionHistory + FieldAccessConsent """ wdf = read_csv("WellData", dtype={"OSEWelltagID": str}) wdf = replace_nans(wdf) diff --git a/transfers/transfer_results_builder.py b/transfers/transfer_results_builder.py index 42e7c49b..edb41a81 100644 --- a/transfers/transfer_results_builder.py +++ b/transfers/transfer_results_builder.py @@ -7,7 +7,7 @@ import pandas as pd from sqlalchemy import select, func -from db import Deployment, PermissionHistory, Sensor, Thing, ThingContactAssociation +from db import Deployment, FieldAccessConsent, Sensor, Thing, ThingContactAssociation from db.engine import session_ctx from transfers.sensor_transfer import ( EQUIPMENT_TO_SENSOR_TYPE_MAP, @@ -210,14 +210,14 @@ def _permissions_destination_series(session) -> pd.Series: sql = ( select( Thing.name.label("point_id"), - PermissionHistory.permission_type.label("permission_type"), - PermissionHistory.permission_allowed.label("permission_allowed"), + FieldAccessConsent.permission_type.label("permission_type"), + FieldAccessConsent.permission_allowed.label("permission_allowed"), ) - .select_from(PermissionHistory) - .join(Thing, Thing.id == PermissionHistory.target_id) - .where(PermissionHistory.target_table == "thing") + .select_from(FieldAccessConsent) + .join(Thing, Thing.id == FieldAccessConsent.target_id) + .where(FieldAccessConsent.target_table == "thing") .where( - PermissionHistory.permission_type.in_( + FieldAccessConsent.permission_type.in_( ("Water Chemistry Sample", "Water Level Sample") ) ) @@ -370,10 +370,10 @@ def _build_permissions(self, spec: TransferComparisonSpec) -> TransferResult: destination_row_count = int( session.execute( select(func.count()) - .select_from(PermissionHistory) - .where(PermissionHistory.target_table == "thing") + .select_from(FieldAccessConsent) + .where(FieldAccessConsent.target_table == "thing") .where( - PermissionHistory.permission_type.in_( + FieldAccessConsent.permission_type.in_( ("Water Chemistry Sample", "Water Level Sample") ) ) @@ -402,7 +402,7 @@ def _build_permissions(self, spec: TransferComparisonSpec) -> TransferResult: transfer_name=spec.transfer_name, source_csv=spec.source_csv, source_key_column=spec.source_key_column, - destination_model="PermissionHistory", + destination_model="FieldAccessConsent", destination_key_column=spec.destination_key_column, source_row_count=source_row_count, agreed_transfer_row_count=agreed_transfer_row_count, diff --git a/transfers/transfer_results_specs.py b/transfers/transfer_results_specs.py index 5a23f40b..2a0071e5 100644 --- a/transfers/transfer_results_specs.py +++ b/transfers/transfer_results_specs.py @@ -28,7 +28,7 @@ NMA_view_NGWMN_WaterLevels, NMA_view_NGWMN_WellConstruction, Observation, - PermissionHistory, + FieldAccessConsent, Sensor, Thing, WellScreen, @@ -523,7 +523,7 @@ def _record_new_contact( PermissionsTransferResult, "WellData", "PointID|PermissionType|PermissionAllowed", - PermissionHistory, + FieldAccessConsent, "thing.name|permission_type|permission_allowed", option_field="transfer_permissions", ), From 3e78d712b6883b7938ba5b20a0d966df0b177ad9 Mon Sep 17 00:00:00 2001 From: jakeross Date: Mon, 24 Aug 2026 22:25:27 -0700 Subject: [PATCH 03/13] feat(access): add the grant tables and the visibility layer The storage and evaluation half of ADR5. No existing endpoint changes behavior: the layer goes behind one service before it goes behind all of them. Four tables. permission_grant holds internal authorization -- principal x capability x scope, one row per data type. publication_consent holds what a landowner agreed to publish about their well, per destination and data type, so "levels yes, chemistry no" is a row rather than institutional memory. destination is the registry those consent rows point at. authorization_audit is append-only and is written in the same transaction as every change, because the first question after an exposure is who granted that, and when. Two tables rather than one because consent and clearance are decided by different people on different authority and revoked by different events. One evaluator rather than two because migration baba91fe5e83 already showed what independent filtering costs. domain/access.py holds the rules as plain functions over plain values, so they are testable without a database and cannot be restated differently by a second caller. Three invariants live there: default deny, no wildcard data type, and expiry compared at use rather than swept by a job that can be missed. An unrecognized scope type denies instead of raising, so a row written by a newer version of the code never reads as permission to an older one. services/visibility.py is the only place that answers "may this principal" and "what does this destination get". api/access.py is its one tenant: grant and consent administration, the destination registry, and a /access/decision introspection route. The prefix is /access because /publication is already the bibliography. Writes are Admin, reads are Viewer. The five vocabularies are lexicon categories seeded from core/lexicon.json, not enums in code, so adding a destination kind is not a deploy. Data types are deliberately coarse -- water level, water chemistry, well construction, site metadata -- and there is deliberately no term meaning "all". Not included: no pre-existing endpoint consults the layer, so the OGC views still publish on release_status; field projection is the next step; grants are read per request with no cache, since an unbounded one would defeat the immediate-revocation promise; and Authentik roles have no broad day-one grants yet, so /access/decision currently says no to everyone. 52 tests: 32 with no database over the rules, 20 through the routes. Co-Authored-By: Claude Opus 5 --- ADR5.md | 6 +- CLAUDE.md | 20 +- .../79a3ab24627e_add_access_control_tables.py | 183 +++++++++ api/access.py | 371 ++++++++++++++++++ core/enums.py | 9 + core/initializers.py | 2 + core/lexicon.json | 137 ++++++- db/__init__.py | 6 +- db/authorization_audit.py | 75 ++++ db/destination.py | 55 +++ db/permission_grant.py | 98 +++++ db/publication_consent.py | 102 +++++ domain/access.py | 291 ++++++++++++++ schemas/access.py | 145 +++++++ services/access_admin.py | 268 +++++++++++++ services/visibility.py | 245 ++++++++++++ tests/test_access.py | 361 +++++++++++++++++ tests/test_domain_access.py | 250 ++++++++++++ 18 files changed, 2618 insertions(+), 6 deletions(-) create mode 100644 alembic/versions/79a3ab24627e_add_access_control_tables.py create mode 100644 api/access.py create mode 100644 db/authorization_audit.py create mode 100644 db/destination.py create mode 100644 db/permission_grant.py create mode 100644 db/publication_consent.py create mode 100644 domain/access.py create mode 100644 schemas/access.py create mode 100644 services/access_admin.py create mode 100644 services/visibility.py create mode 100644 tests/test_access.py create mode 100644 tests/test_domain_access.py diff --git a/ADR5.md b/ADR5.md index 2b3f1d80..6d4d7fb2 100644 --- a/ADR5.md +++ b/ADR5.md @@ -207,8 +207,10 @@ told. 2. **Done.** `permission_history` is `field_access_consent`, dead `Permission` and `PermissionMixin` are gone, and `ReleaseMixin` carries `release_status` (level) plus `data_maturity` (review state). -3. `permission_grant`, `publication_consent`, destination registry, and the - visibility layer, behind **one** service before it is behind all of them. +3. **Done.** `permission_grant`, `publication_consent`, `destination` and + `authorization_audit` exist; `services/visibility.py` is the single + evaluator; `api/access.py` is its one tenant. No existing endpoint routes + through it yet. 4. Field projection at the serialization chokepoint, with the never-public list. 5. Console administration. 6. Healy migration, after the data owner decides grandfathering. diff --git a/CLAUDE.md b/CLAUDE.md index 816dba95..3c793ebe 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -219,7 +219,25 @@ upgrading pygeoapi. **`ADR5.md`** decides the shape of the access-control work: two grant tables (internal permission vs landowner publication consent), one visibility layer, -one field-projection chokepoint. Nothing is built yet. +one field-projection chokepoint. + +The storage and the evaluator exist; the field projection does not. + +- **`services/visibility.py` is the only evaluator.** `may()` answers internal + authorization, `published_things()` answers "what does this destination + get". It loads rows and calls `domain/access.py`, which holds the rules and + touches no database. Do not filter by grant or consent anywhere else -- + migration `baba91fe5e83` is what distributed filtering already cost. +- **`api/access.py` (`/access`) is its only tenant.** Grants, destinations, + consent, and a `/access/decision` introspection route. No pre-existing + endpoint consults the layer yet, so release_status still governs what the + OGC views publish. The prefix is `/access`, not `/publication`, because + `api/publication.py` is the bibliography. +- **Default deny, no wildcards, expiry at use.** A grant with no matching row + is a no; a grant names its `data_type` (there is no term meaning "all"); and + nothing sweeps expired rows, so every check compares against the date asked + about. `services/access_admin.py` writes an `authorization_audit` row in the + same transaction as every change. Two vocabulary fixes from it have landed and matter when reading models: diff --git a/alembic/versions/79a3ab24627e_add_access_control_tables.py b/alembic/versions/79a3ab24627e_add_access_control_tables.py new file mode 100644 index 00000000..1787b1e4 --- /dev/null +++ b/alembic/versions/79a3ab24627e_add_access_control_tables.py @@ -0,0 +1,183 @@ +"""add the access-control tables + +The storage half of ADR5: two grant tables, one destination registry, one +append-only authorization audit log. Nothing reads them yet except +services/visibility.py and the /publication routes; no existing endpoint +changes behavior. + +* ``destination`` -- where published data is offered: the public web, a + harvester, a partner agency. +* ``permission_grant`` -- internal authorization, principal x capability x + scope, one row per data type. No NULL-as-wildcard: a grant names its type. +* ``publication_consent`` -- what a landowner agreed to publish about their + well, per destination and data type. One live row per combination; revoked + rows stay, which is why the unique index is partial. +* ``authorization_audit`` -- every grant, revocation and consent event, append + only, so "who granted that, and when" has an answer. + +The vocabularies (principal_type, capability, grant_scope_type, +access_data_type, destination_kind) are lexicon categories seeded from +core/lexicon.json by init_lexicon, not enum types, so adding a destination +kind is not a migration. + +Revision ID: 79a3ab24627e +Revises: e7c1a9f4b2d8 +Create Date: 2026-08-24 22:09:54.479541 + +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "79a3ab24627e" +down_revision: Union[str, Sequence[str], None] = "e7c1a9f4b2d8" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "authorization_audit", + sa.Column("event_type", sa.String(length=50), nullable=False), + sa.Column("actor", sa.String(length=255), nullable=False), + sa.Column("subject_table", sa.String(length=50), nullable=False), + sa.Column("subject_id", sa.Integer(), nullable=True), + sa.Column("detail", sa.JSON(), nullable=True), + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("timezone('UTC', now())"), + nullable=False, + ), + sa.Column("created_by_name", sa.String(length=255), nullable=True), + sa.Column("created_by_id", sa.String(length=255), nullable=True), + sa.Column("updated_by_name", sa.String(length=255), nullable=True), + sa.Column("updated_by_id", sa.String(length=255), nullable=True), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "destination", + sa.Column("slug", sa.String(length=50), nullable=False), + sa.Column("name", sa.String(length=255), nullable=False), + sa.Column("destination_kind", sa.String(length=100), nullable=False), + sa.Column("description", sa.Text(), nullable=True), + sa.Column("active", sa.Boolean(), nullable=False), + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("timezone('UTC', now())"), + nullable=False, + ), + sa.Column("created_by_name", sa.String(length=255), nullable=True), + sa.Column("created_by_id", sa.String(length=255), nullable=True), + sa.Column("updated_by_name", sa.String(length=255), nullable=True), + sa.Column("updated_by_id", sa.String(length=255), nullable=True), + sa.ForeignKeyConstraint( + ["destination_kind"], ["lexicon_term.term"], onupdate="CASCADE" + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("slug"), + ) + op.create_table( + "permission_grant", + sa.Column("principal_type", sa.String(length=100), nullable=False), + sa.Column("principal_id", sa.String(length=255), nullable=False), + sa.Column("capability", sa.String(length=100), nullable=False), + sa.Column("scope_type", sa.String(length=100), nullable=False), + sa.Column("scope_id", sa.Integer(), nullable=True), + sa.Column("data_type", sa.String(length=100), nullable=False), + sa.Column("starts_at", sa.Date(), nullable=False), + sa.Column("ends_at", sa.Date(), nullable=True), + sa.Column("granted_by", sa.String(length=255), nullable=False), + sa.Column("reason", sa.Text(), nullable=True), + sa.Column("revoked_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("revoked_by", sa.String(length=255), nullable=True), + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("timezone('UTC', now())"), + nullable=False, + ), + sa.Column("created_by_name", sa.String(length=255), nullable=True), + sa.Column("created_by_id", sa.String(length=255), nullable=True), + sa.Column("updated_by_name", sa.String(length=255), nullable=True), + sa.Column("updated_by_id", sa.String(length=255), nullable=True), + sa.ForeignKeyConstraint( + ["capability"], ["lexicon_term.term"], onupdate="CASCADE" + ), + sa.ForeignKeyConstraint( + ["data_type"], ["lexicon_term.term"], onupdate="CASCADE" + ), + sa.ForeignKeyConstraint( + ["principal_type"], ["lexicon_term.term"], onupdate="CASCADE" + ), + sa.ForeignKeyConstraint( + ["scope_type"], ["lexicon_term.term"], onupdate="CASCADE" + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + "ix_permission_grant_principal", + "permission_grant", + ["principal_type", "principal_id", "capability"], + unique=False, + ) + op.create_table( + "publication_consent", + sa.Column("thing_id", sa.Integer(), nullable=False), + sa.Column("destination_id", sa.Integer(), nullable=False), + sa.Column("data_type", sa.String(length=100), nullable=False), + sa.Column("contact_id", sa.Integer(), nullable=True), + sa.Column("recorded_by", sa.String(length=255), nullable=False), + sa.Column("notes", sa.Text(), nullable=True), + sa.Column("starts_at", sa.Date(), nullable=False), + sa.Column("ends_at", sa.Date(), nullable=True), + sa.Column("revoked_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("revoked_by", sa.String(length=255), nullable=True), + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("timezone('UTC', now())"), + nullable=False, + ), + sa.Column("created_by_name", sa.String(length=255), nullable=True), + sa.Column("created_by_id", sa.String(length=255), nullable=True), + sa.Column("updated_by_name", sa.String(length=255), nullable=True), + sa.Column("updated_by_id", sa.String(length=255), nullable=True), + sa.ForeignKeyConstraint( + ["contact_id"], + ["contact.id"], + ), + sa.ForeignKeyConstraint( + ["data_type"], ["lexicon_term.term"], onupdate="CASCADE" + ), + sa.ForeignKeyConstraint( + ["destination_id"], + ["destination.id"], + ), + sa.ForeignKeyConstraint(["thing_id"], ["thing.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + "uq_publication_consent_live", + "publication_consent", + ["thing_id", "destination_id", "data_type"], + unique=True, + postgresql_where=sa.text("revoked_at IS NULL"), + ) + + +def downgrade() -> None: + op.drop_index("uq_publication_consent_live", table_name="publication_consent") + op.drop_table("publication_consent") + op.drop_index("ix_permission_grant_principal", table_name="permission_grant") + op.drop_table("permission_grant") + op.drop_table("destination") + op.drop_table("authorization_audit") diff --git a/api/access.py b/api/access.py new file mode 100644 index 00000000..3ab6fb22 --- /dev/null +++ b/api/access.py @@ -0,0 +1,371 @@ +# =============================================================================== +# Copyright 2025 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +api/access.py + +The first tenant of the ADR5 visibility layer. + +One router, deliberately small: + +* ``/access/grant`` administers internal grants, and ``/access/decision`` + answers "may I, right now?" from ``services/visibility.py`` rather than from + anything local. +* ``/access/destination`` and ``/access/consent`` administer destinations and + landowner consent, and ``/access/destination/{slug}/thing`` answers "what + does this destination get". + +The prefix is ``/access`` rather than ``/publication`` because +``api/publication.py`` is already the bibliography -- citations, not consent. + +Nothing here changes what an existing endpoint returns. The layer is proved +end to end behind one service before it is put behind all of them (ADR5, A.6). +Every route is authorized: administration is Admin, reading is Viewer. +""" + +from datetime import date + +from fastapi import APIRouter, Query +from sqlalchemy import select +from starlette.status import HTTP_201_CREATED + +from core.dependencies import ( + admin_dependency, + session_dependency, + viewer_dependency, +) +from db.destination import Destination +from db.permission_grant import PermissionGrant +from db.publication_consent import PublicationConsent +from schemas.access import ( + AccessDecision, + CreateDestination, + CreatePermissionGrant, + CreatePublicationConsent, + DestinationResponse, + PermissionGrantResponse, + PublicationConsentResponse, + PublishedThing, +) +from services.access_admin import ( + AlreadyRevoked, + actor_from_payload, + create_grant, + record_consent, + register_destination, + revoke_consent, + revoke_grant, +) +from services.exceptions_helper import PydanticStyleException +from services.visibility import ( + destination_by_slug, + may, + principals_from_payload, + published_things, +) + +router = APIRouter(prefix="/access", tags=["access control"]) + + +def _not_found(what: str, value): + return PydanticStyleException( + status_code=404, + detail=[ + { + "loc": ["path", what], + "msg": f"No {what} {value!r}.", + "type": "value_error", + "input": value, + } + ], + ) + + +def _invalid(field: str, message: str, value): + return PydanticStyleException( + status_code=422, + detail=[ + { + "loc": ["body", field], + "msg": message, + "type": "value_error", + "input": value, + } + ], + ) + + +# ============= Permission grants ============================================= + + +@router.post( + "/grant", summary="Create a permission grant", status_code=HTTP_201_CREATED +) +def create_permission_grant( + payload: CreatePermissionGrant, + session: session_dependency, + user: admin_dependency, +) -> PermissionGrantResponse: + """Grant a principal one capability over one data type within one scope. + + The grant names its data type; there is no wildcard, so a data type added + later is never covered by this row. + """ + try: + grant = create_grant( + session, + actor_from_payload(user), + principal_type=payload.principal_type.value, + principal_id=payload.principal_id, + capability=payload.capability.value, + scope_type=payload.scope_type.value, + scope_id=payload.scope_id, + data_type=payload.data_type.value, + starts_at=payload.starts_at, + ends_at=payload.ends_at, + reason=payload.reason, + ) + except ValueError as exception: + raise _invalid("scope_id", str(exception), payload.scope_id) + + return PermissionGrantResponse.model_validate(grant) + + +@router.post( + "/grant/{grant_id}/revocation", + summary="Revoke a permission grant", + status_code=HTTP_201_CREATED, +) +def revoke_permission_grant( + grant_id: int, + session: session_dependency, + user: admin_dependency, +) -> PermissionGrantResponse: + """Revoke now. Effective at the next read, not at the next token refresh.""" + grant = session.get(PermissionGrant, grant_id) + if grant is None: + raise _not_found("grant_id", grant_id) + + try: + grant = revoke_grant(session, actor_from_payload(user), grant) + except AlreadyRevoked as exception: + raise _invalid("grant_id", str(exception), grant_id) + + return PermissionGrantResponse.model_validate(grant) + + +@router.get("/grant", summary="List grants held by a principal") +def get_permission_grants( + session: session_dependency, + user: admin_dependency, + principal_id: str = Query(description="Authentik subject, role, or key label"), + include_revoked: bool = Query(default=False), +) -> list[PermissionGrantResponse]: + statement = select(PermissionGrant).where( + PermissionGrant.principal_id == principal_id + ) + if not include_revoked: + statement = statement.where(PermissionGrant.revoked_at.is_(None)) + + return [ + PermissionGrantResponse.model_validate(row) + for row in session.execute(statement).scalars() + ] + + +@router.get("/decision", summary="Ask the visibility layer about yourself") +def get_access_decision( + session: session_dependency, + user: viewer_dependency, + capability: str = Query(), + data_type: str = Query(), + thing_id: int = Query(default=None), + on_date: date = Query(default=None), +) -> AccessDecision: + """May the caller do this? Answered by the one visibility layer. + + Default deny: an unrecognized capability or a caller the token says + nothing about gets False rather than an error, because a question this + layer cannot answer is not a yes. + """ + principals = principals_from_payload(user) + return AccessDecision( + allowed=may( + session, + principals, + capability=capability, + data_type=data_type, + thing_id=thing_id, + on_date=on_date, + ), + capability=capability, + data_type=data_type, + thing_id=thing_id, + principals=[f"{kind}:{identifier}" for kind, identifier in principals], + ) + + +# ============= Destinations ============================================= + + +@router.post( + "/destination", summary="Register a destination", status_code=HTTP_201_CREATED +) +def create_destination( + payload: CreateDestination, + session: session_dependency, + user: admin_dependency, +) -> DestinationResponse: + existing = destination_by_slug(session, payload.slug) + if existing is not None: + raise PydanticStyleException( + status_code=409, + detail=[ + { + "loc": ["body", "slug"], + "msg": f"Destination {payload.slug!r} already exists.", + "type": "value_error", + "input": payload.slug, + } + ], + ) + + destination = register_destination( + session, + actor_from_payload(user), + slug=payload.slug, + name=payload.name, + destination_kind=payload.destination_kind.value, + description=payload.description, + ) + return DestinationResponse.model_validate(destination) + + +@router.get("/destination", summary="List destinations") +def get_destinations( + session: session_dependency, + user: viewer_dependency, +) -> list[DestinationResponse]: + return [ + DestinationResponse.model_validate(row) + for row in session.execute(select(Destination).order_by(Destination.slug)) + .scalars() + .all() + ] + + +@router.get("/destination/{slug}/thing", summary="What this destination may read") +def get_published_things( + slug: str, + session: session_dependency, + user: viewer_dependency, + data_type: str = Query(default=None), + on_date: date = Query(default=None), +) -> list[PublishedThing]: + """The destination's view, computed from consent rows at request time. + + A retired destination gets an empty list, and so does one nobody has + consented to yet: default deny, with no separate "unpublished" state to + keep in sync. + """ + destination = destination_by_slug(session, slug) + if destination is None: + raise _not_found("slug", slug) + + return [ + PublishedThing(**entry) + for entry in published_things( + session, destination, data_type=data_type, on_date=on_date + ) + ] + + +# ============= Publication consent ============================================= + + +@router.post( + "/consent", summary="Record publication consent", status_code=HTTP_201_CREATED +) +def create_publication_consent( + payload: CreatePublicationConsent, + session: session_dependency, + user: admin_dependency, +) -> PublicationConsentResponse: + """Record that an owner agreed to publish one data type to one destination.""" + destination = destination_by_slug(session, payload.destination_slug) + if destination is None: + raise _not_found("destination_slug", payload.destination_slug) + + try: + consent = record_consent( + session, + actor_from_payload(user), + thing_id=payload.thing_id, + destination_id=destination.id, + data_type=payload.data_type.value, + starts_at=payload.starts_at, + ends_at=payload.ends_at, + contact_id=payload.contact_id, + notes=payload.notes, + ) + except ValueError as exception: + raise _invalid("ends_at", str(exception), payload.ends_at) + + return PublicationConsentResponse.model_validate(consent) + + +@router.post( + "/consent/{consent_id}/revocation", + summary="Withdraw publication consent", + status_code=HTTP_201_CREATED, +) +def revoke_publication_consent( + consent_id: int, + session: session_dependency, + user: admin_dependency, +) -> PublicationConsentResponse: + """Stop offering it. Copies already harvested are not recalled.""" + consent = session.get(PublicationConsent, consent_id) + if consent is None: + raise _not_found("consent_id", consent_id) + + try: + consent = revoke_consent(session, actor_from_payload(user), consent) + except AlreadyRevoked as exception: + raise _invalid("consent_id", str(exception), consent_id) + + return PublicationConsentResponse.model_validate(consent) + + +@router.get("/consent", summary="List consent for a thing") +def get_publication_consent( + session: session_dependency, + user: viewer_dependency, + thing_id: int = Query(), + include_revoked: bool = Query(default=False), +) -> list[PublicationConsentResponse]: + statement = select(PublicationConsent).where( + PublicationConsent.thing_id == thing_id + ) + if not include_revoked: + statement = statement.where(PublicationConsent.revoked_at.is_(None)) + + return [ + PublicationConsentResponse.model_validate(row) + for row in session.execute(statement).scalars() + ] + + +# ============= EOF ============================================= diff --git a/core/enums.py b/core/enums.py index 79027212..675d7fe1 100644 --- a/core/enums.py +++ b/core/enums.py @@ -83,4 +83,13 @@ Lithology: type[Enum] = build_enum_from_lexicon_category("lithology") FormationCode: type[Enum] = build_enum_from_lexicon_category("formation_code") NoteType: type[Enum] = build_enum_from_lexicon_category("note_type") + +# Access control (ADR5). Vocabularies rather than hardcoded literals so a new +# destination kind is a lexicon entry, not a deploy. There is deliberately no +# access_data_type meaning "all". +PrincipalType: type[Enum] = build_enum_from_lexicon_category("principal_type") +Capability: type[Enum] = build_enum_from_lexicon_category("capability") +GrantScopeType: type[Enum] = build_enum_from_lexicon_category("grant_scope_type") +AccessDataType: type[Enum] = build_enum_from_lexicon_category("access_data_type") +DestinationKind: type[Enum] = build_enum_from_lexicon_category("destination_kind") # ============= EOF ============================================= diff --git a/core/initializers.py b/core/initializers.py index 9f419caa..a35506f4 100644 --- a/core/initializers.py +++ b/core/initializers.py @@ -227,7 +227,9 @@ def register_api_routes(app): from api.geothermal import router as geothermal_router from api.chemisty import router as chemistry_router from api.gis_artifacts import router as gis_artifacts_router + from api.access import router as access_router + app.include_router(access_router) app.include_router(asset_router) app.include_router(chemistry_router) app.include_router(author_router) diff --git a/core/lexicon.json b/core/lexicon.json index 40291d69..10da407b 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -247,6 +247,26 @@ { "name": "data_maturity", "description": "How far through review a measurement is, on USGS terms. Orthogonal to release_status, which controls visibility rather than trust." + }, + { + "name": "principal_type", + "description": "What kind of thing an access grant is made to: a person, a role, or an API key. Destinations are not principals -- publishing is recorded as consent (ADR5)." + }, + { + "name": "capability", + "description": "What a principal may do: read, enter measurements, correct records, administer grants (ADR5)." + }, + { + "name": "grant_scope_type", + "description": "How far a grant reaches: the whole institution, one group/project, or one thing (ADR5)." + }, + { + "name": "access_data_type", + "description": "The kind of data a grant or a publication consent names. Deliberately coarse, and deliberately without a term meaning 'all' (ADR5, PUB-U11)." + }, + { + "name": "destination_kind", + "description": "What sort of place published data is offered to: the public web, a harvesting agency, a partner with a standing connection (ADR5)." } ], "terms": [ @@ -1859,7 +1879,8 @@ }, { "categories": [ - "activity_type" + "activity_type", + "access_data_type" ], "term": "water chemistry", "definition": "water chemistry" @@ -8508,6 +8529,118 @@ ], "term": "in review", "definition": "Under review and not yet approved. Intermediate state from the USGS Aquarius approval levels used for continuous records." + }, + { + "categories": [ + "principal_type" + ], + "term": "user", + "definition": "A person, referenced by their Authentik subject identifier." + }, + { + "categories": [ + "principal_type" + ], + "term": "role", + "definition": "An Authentik group, carrying the stable institutional part of access." + }, + { + "categories": [ + "principal_type" + ], + "term": "api key", + "definition": "A key issued to a client that has no person behind it." + }, + { + "categories": [ + "capability" + ], + "term": "read", + "definition": "May see the data." + }, + { + "categories": [ + "capability" + ], + "term": "enter", + "definition": "May record new measurements." + }, + { + "categories": [ + "capability" + ], + "term": "correct", + "definition": "May revise existing records." + }, + { + "categories": [ + "capability" + ], + "term": "administer", + "definition": "May grant and revoke access." + }, + { + "categories": [ + "grant_scope_type" + ], + "term": "global", + "definition": "Reaches every thing. Used for the broad role grants that preserve today's access." + }, + { + "categories": [ + "grant_scope_type" + ], + "term": "group", + "definition": "Reaches the things in one group -- a project, a network." + }, + { + "categories": [ + "grant_scope_type" + ], + "term": "thing", + "definition": "Reaches exactly one thing." + }, + { + "categories": [ + "access_data_type" + ], + "term": "water level", + "definition": "Depth-to-water readings, manual and continuous." + }, + { + "categories": [ + "access_data_type" + ], + "term": "well construction", + "definition": "Casing, screen, borehole and completion detail." + }, + { + "categories": [ + "access_data_type" + ], + "term": "site metadata", + "definition": "Where the thing is and what it is: name, location, type, status." + }, + { + "categories": [ + "destination_kind" + ], + "term": "public web", + "definition": "Offered to anyone, unauthenticated." + }, + { + "categories": [ + "destination_kind" + ], + "term": "harvester", + "definition": "A machine consumer that pulls on a schedule, such as NGWMN." + }, + { + "categories": [ + "destination_kind" + ], + "term": "partner agency", + "definition": "A named organization reading through a standing connection." } ] -} \ No newline at end of file +} diff --git a/db/__init__.py b/db/__init__.py index 5eeb02fc..c4475b45 100644 --- a/db/__init__.py +++ b/db/__init__.py @@ -31,13 +31,16 @@ from db.aquifer_system import * from db.aquifer_type import * from db.asset import * +from db.authorization_audit import * from db.base import * from db.base import Base from db.collabnet import * from db.contact import * from db.data_provenance import * from db.deployment import * +from db.destination import * from db.field import * +from db.field_access_consent import * from db.geochronology import * from db.geologic_formation import * from db.geothermal import * @@ -48,8 +51,9 @@ from db.notes import * from db.observation import * from db.parameter import * -from db.field_access_consent import * +from db.permission_grant import * from db.publication import * +from db.publication_consent import * from db.regulatory_limit import * from db.sample import * from db.sensor import * diff --git a/db/authorization_audit.py b/db/authorization_audit.py new file mode 100644 index 00000000..1eb22708 --- /dev/null +++ b/db/authorization_audit.py @@ -0,0 +1,75 @@ +# =============================================================================== +# Copyright 2025 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +db/authorization_audit.py + +The append-only log of authorization events: grants, revocations, consent +captured or withdrawn, destinations registered or retired. + +ADR5 asks for this from the first commit, for one reason: when something is +exposed that should not have been, the first question is never "what was the +value". It is "who granted that, and when". + +Append only. Nothing in the application updates or deletes a row here, and a +database-level backstop for writes that bypass the application is still owed +(ADR5, 4.3). ``AuditMixin.created_at`` is the event time; there is no second +timestamp column to disagree with it. + +Separate from sqlalchemy-continuum's versioning, which records data history. +Authorization changes are a different, higher-value target. +""" + +from typing import Optional + +from sqlalchemy import JSON, Integer, String +from sqlalchemy.orm import Mapped, mapped_column + +from db.base import Base, AutoBaseMixin + +# Event types. Strings rather than lexicon terms: the log must be able to +# record an event whose vocabulary row was itself just deleted. +GRANT_CREATED = "grant.created" +GRANT_REVOKED = "grant.revoked" +CONSENT_RECORDED = "consent.recorded" +CONSENT_REVOKED = "consent.revoked" +DESTINATION_REGISTERED = "destination.registered" + + +class AuthorizationAudit(Base, AutoBaseMixin): + """One authorization event, as it happened.""" + + event_type: Mapped[str] = mapped_column(String(50), nullable=False) + + # Who did it, as the identifier the application had at the time. + actor: Mapped[str] = mapped_column(String(255), nullable=False) + + # What it happened to: table name and row id, kept as loose values so a + # deleted row does not take its own audit trail with it. + subject_table: Mapped[str] = mapped_column(String(50), nullable=False) + subject_id: Mapped[Optional[int]] = mapped_column(Integer, nullable=True) + + # The event's own fields, whatever they were. What each event should + # record is open (PERM-U11); recording the payload keeps the question + # answerable later rather than losing it now. + detail: Mapped[Optional[dict]] = mapped_column(JSON, nullable=True) + + def __str__(self): + return ( + f"{self.event_type} {self.subject_table}:{self.subject_id} by {self.actor}" + ) + + +# ============= EOF ============================================= diff --git a/db/destination.py b/db/destination.py new file mode 100644 index 00000000..c035fad1 --- /dev/null +++ b/db/destination.py @@ -0,0 +1,55 @@ +# =============================================================================== +# Copyright 2025 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +db/destination.py + +The registry of places data goes: the anonymous public web, a federal +harvester, a partner agency's standing connection. + +A destination is not a user and not a role. It is the other half of a +publication consent row -- the "to whom" a landowner agreed to. Adding a new +one is a registry entry plus a set of consent rows, which is the point of +having a registry at all (ADR5, Part IV). +""" + +from typing import Optional + +from sqlalchemy import String, Text, Boolean +from sqlalchemy.orm import Mapped, mapped_column + +from db.base import Base, AutoBaseMixin, lexicon_term + + +class Destination(Base, AutoBaseMixin): + """One place published data is offered to.""" + + # Stable, URL-safe handle. Routes and consent records refer to a + # destination by slug rather than id so a fixture, a config file, and an + # operator all name it the same way. + slug: Mapped[str] = mapped_column(String(50), nullable=False, unique=True) + name: Mapped[str] = mapped_column(String(255), nullable=False) + destination_kind: Mapped[str] = lexicon_term(nullable=False) + description: Mapped[Optional[str]] = mapped_column(Text, nullable=True) + + # Retiring a destination stops it being offered without deleting the + # consent history that says what was once agreed to. + active: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) + + def __str__(self): + return self.slug + + +# ============= EOF ============================================= diff --git a/db/permission_grant.py b/db/permission_grant.py new file mode 100644 index 00000000..daa19c7c --- /dev/null +++ b/db/permission_grant.py @@ -0,0 +1,98 @@ +# =============================================================================== +# Copyright 2025 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +db/permission_grant.py + +Internal authorization: may this principal exercise this capability within +this scope? (ADR5, Part III.) + +This is institutional trust, decided per person by data services staff. The +landowner's half -- what an owner agreed to publish about their well -- is +``db/publication_consent.py``, a separate table with separate governance and +the same grammar. Both are evaluated by ``services/visibility.py``. + +Invariants, enforced in ``domain/access.py`` before a row is written: + +* ``data_type`` is never null and there is no term meaning "all", so a data + type added later is not covered by an existing grant. +* A ``global`` grant carries no ``scope_id``; a ``group`` or ``thing`` grant + requires one. +* Expiry is read at use. Nothing sweeps this table, so a missed job cannot + leave a grant standing past its end date. +""" + +from datetime import date, datetime +from typing import Optional + +from sqlalchemy import DateTime, Index, Integer, String, Text +from sqlalchemy.orm import Mapped, mapped_column + +from db.base import Base, AutoBaseMixin, lexicon_term + + +class PermissionGrant(Base, AutoBaseMixin): + """One grant: principal x capability x scope, over one data type.""" + + # --- Principal --- + # principal_id is a stable identifier whose meaning depends on the type: + # an Authentik subject for `user`, a group name for `role`, a key label + # for `api key`. It is a string, not a foreign key, because Authentik owns + # identity and Ocotillo owns authorization (ADR5, A.4). + principal_type: Mapped[str] = lexicon_term(nullable=False) + principal_id: Mapped[str] = mapped_column(String(255), nullable=False) + + # --- What and where --- + capability: Mapped[str] = lexicon_term(nullable=False) + scope_type: Mapped[str] = lexicon_term(nullable=False) + scope_id: Mapped[Optional[int]] = mapped_column(Integer, nullable=True) + data_type: Mapped[str] = lexicon_term(nullable=False) + + # --- When --- + starts_at: Mapped[date] = mapped_column(nullable=False) + ends_at: Mapped[Optional[date]] = mapped_column(nullable=True) + + # --- On whose authority --- + # "Who granted that, and when" is the first question after an incident, so + # it is a column rather than something to reconstruct from the audit log. + granted_by: Mapped[str] = mapped_column(String(255), nullable=False) + reason: Mapped[Optional[str]] = mapped_column(Text, nullable=True) + + # --- Revocation --- + # Revoking sets these rather than deleting the row: the record that access + # once existed is the point. Effective at the next read, never backdated. + revoked_at: Mapped[Optional[datetime]] = mapped_column( + DateTime(timezone=True), nullable=True + ) + revoked_by: Mapped[Optional[str]] = mapped_column(String(255), nullable=True) + + __table_args__ = ( + # The read path always starts from "who is asking". + Index( + "ix_permission_grant_principal", + "principal_type", + "principal_id", + "capability", + ), + ) + + def __str__(self): + return ( + f"{self.principal_type}:{self.principal_id} may {self.capability} " + f"{self.data_type} ({self.scope_type})" + ) + + +# ============= EOF ============================================= diff --git a/db/publication_consent.py b/db/publication_consent.py new file mode 100644 index 00000000..2ce59594 --- /dev/null +++ b/db/publication_consent.py @@ -0,0 +1,102 @@ +# =============================================================================== +# Copyright 2025 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +db/publication_consent.py + +The landowner's half of ADR5: did the owner of this well agree to publish this +data type to this destination? + +Kept apart from ``db/permission_grant.py`` on purpose. Both reduce to the same +grammar and both are evaluated by ``services/visibility.py``, but they are +decided by different people on different authority and revoked by different +events -- a phone call here, an HR-shaped event there. One engine, two tables. + +An owner willing to share water levels but not chemistry is one row, not a +workaround. +""" + +from datetime import date, datetime +from typing import TYPE_CHECKING, Optional + +from sqlalchemy import DateTime, ForeignKey, Index, Integer, String, Text, text +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from db.base import Base, AutoBaseMixin, lexicon_term + +if TYPE_CHECKING: + from db.contact import Contact + from db.destination import Destination + from db.thing import Thing + + +class PublicationConsent(Base, AutoBaseMixin): + """One consent: this thing's data type is offered to this destination.""" + + # --- What is published, and where --- + thing_id: Mapped[int] = mapped_column( + Integer, ForeignKey("thing.id", ondelete="CASCADE"), nullable=False + ) + destination_id: Mapped[int] = mapped_column( + Integer, ForeignKey("destination.id"), nullable=False + ) + data_type: Mapped[str] = lexicon_term(nullable=False) + + # --- Who agreed --- + # Nullable because the Bureau owns some of the wells it monitors, and + # inventing a consenting contact for those would be a lie. NULL means the + # decision was institutional; `recorded_by` still says who made it. + contact_id: Mapped[Optional[int]] = mapped_column( + Integer, ForeignKey("contact.id"), nullable=True + ) + recorded_by: Mapped[str] = mapped_column(String(255), nullable=False) + notes: Mapped[Optional[str]] = mapped_column(Text, nullable=True) + + # --- When --- + starts_at: Mapped[date] = mapped_column(nullable=False) + ends_at: Mapped[Optional[date]] = mapped_column(nullable=True) + + # --- Revocation --- + # "Unpublish" means "stop offering". Copies a harvester already took live + # in someone else's system, and owners are told so rather than promised a + # recall the Bureau does not have (ADR5, 3.6). + revoked_at: Mapped[Optional[datetime]] = mapped_column( + DateTime(timezone=True), nullable=True + ) + revoked_by: Mapped[Optional[str]] = mapped_column(String(255), nullable=True) + + # --- Relationships --- + thing: Mapped["Thing"] = relationship("Thing", viewonly=True) + destination: Mapped["Destination"] = relationship("Destination", viewonly=True) + contact: Mapped[Optional["Contact"]] = relationship("Contact", viewonly=True) + + __table_args__ = ( + # One live row per (thing, destination, data type). Revoked rows stay + # for the history, so the constraint has to ignore them. + Index( + "uq_publication_consent_live", + "thing_id", + "destination_id", + "data_type", + unique=True, + postgresql_where=text("revoked_at IS NULL"), + ), + ) + + def __str__(self): + return f"thing {self.thing_id} -> destination {self.destination_id} ({self.data_type})" + + +# ============= EOF ============================================= diff --git a/domain/access.py b/domain/access.py new file mode 100644 index 00000000..68397b75 --- /dev/null +++ b/domain/access.py @@ -0,0 +1,291 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Access-control rules from ADR5, as plain functions over plain values. + +Two questions, one grammar: + +* May this principal exercise this capability within this scope? + (``permission_grant`` -- institutional trust, decided per person.) +* Did the owner agree to publish this data type of this well to this + destination? (``publication_consent`` -- landowner consent, decided per well.) + +Two record types with separate governance, evaluated the same way. The rules +live here so they can be exercised without a database; ``services/visibility.py`` +loads the rows and asks. + +Three invariants this module exists to keep: + +* **Default deny.** No grant means no. An empty sequence answers False, and + every helper returns False rather than raising when it cannot say yes. +* **No wildcards.** A grant names its data type. There is no term meaning + "all", so a data type added next year is never covered by an existing row. +* **Expiry is checked at use.** Nothing sweeps expired rows; every check + compares against the date it is asked about. +""" + +from dataclasses import dataclass, field +from datetime import date, datetime + +# Scope types. A grant reaches everything below its scope, so `global` covers +# every thing, `group` covers the things in one group (a project or network), +# and `thing` covers exactly one. +SCOPE_GLOBAL = "global" +SCOPE_GROUP = "group" +SCOPE_THING = "thing" +SCOPE_TYPES = frozenset({SCOPE_GLOBAL, SCOPE_GROUP, SCOPE_THING}) + +# Capabilities. +CAPABILITY_READ = "read" +CAPABILITY_ENTER = "enter" +CAPABILITY_CORRECT = "correct" +CAPABILITY_ADMINISTER = "administer" +CAPABILITIES = frozenset( + {CAPABILITY_READ, CAPABILITY_ENTER, CAPABILITY_CORRECT, CAPABILITY_ADMINISTER} +) + +# Principal types. A destination is not here: publishing to one is recorded as +# consent, not as a grant, which is the two-table half of ADR5. +PRINCIPAL_USER = "user" +PRINCIPAL_ROLE = "role" +PRINCIPAL_API_KEY = "api_key" +PRINCIPAL_TYPES = frozenset({PRINCIPAL_USER, PRINCIPAL_ROLE, PRINCIPAL_API_KEY}) + + +class AccessRuleError(ValueError): + """Base for rule violations. A ValueError, per ADR4.""" + + +class UnknownScopeType(AccessRuleError): + pass + + +class UnknownCapability(AccessRuleError): + pass + + +class UnknownPrincipalType(AccessRuleError): + pass + + +class MissingDataType(AccessRuleError): + pass + + +class ScopeIdMismatch(AccessRuleError): + pass + + +class BackwardsDateRange(AccessRuleError): + pass + + +@dataclass(frozen=True) +class Grant: + """One row of ``permission_grant``, as plain values.""" + + principal_type: str + principal_id: str + capability: str + scope_type: str + scope_id: int | None + data_type: str + starts_at: date + ends_at: date | None = None + revoked_at: datetime | None = None + + +@dataclass(frozen=True) +class Consent: + """One row of ``publication_consent``, as plain values.""" + + thing_id: int + destination_id: int + data_type: str + starts_at: date + ends_at: date | None = None + revoked_at: datetime | None = None + + +@dataclass(frozen=True) +class AccessRequest: + """What a caller is asking for. + + ``principals`` is every identity the caller presents at once -- their user + subject and each role they hold -- because a grant may name any of them. + ``group_ids`` are the groups the target thing belongs to, which is what + makes a project-scoped grant reach it. + """ + + capability: str + data_type: str + principals: tuple[tuple[str, str], ...] = () + thing_id: int | None = None + group_ids: tuple[int, ...] = field(default_factory=tuple) + + +def validate_grant( + principal_type: str, + capability: str, + scope_type: str, + scope_id: int | None, + data_type: str | None, + starts_at: date, + ends_at: date | None, +) -> None: + """Reject a grant that could not be evaluated honestly. + + Raised before a row is written, so the invariants hold in the table rather + than in the reader. + """ + if principal_type not in PRINCIPAL_TYPES: + raise UnknownPrincipalType( + f"'{principal_type}' is not a principal type " + f"({', '.join(sorted(PRINCIPAL_TYPES))})." + ) + if capability not in CAPABILITIES: + raise UnknownCapability( + f"'{capability}' is not a capability ({', '.join(sorted(CAPABILITIES))})." + ) + if scope_type not in SCOPE_TYPES: + raise UnknownScopeType( + f"'{scope_type}' is not a scope type ({', '.join(sorted(SCOPE_TYPES))})." + ) + if scope_type == SCOPE_GLOBAL and scope_id is not None: + raise ScopeIdMismatch("A global grant names no scope_id.") + if scope_type != SCOPE_GLOBAL and scope_id is None: + raise ScopeIdMismatch(f"A {scope_type}-scoped grant needs a scope_id.") + if not data_type: + # The no-wildcard rule. There is deliberately no term meaning "all": + # a blanket grant is what published data nobody had agreed to publish. + raise MissingDataType( + "A grant names its data type. There is no wildcard, so a new data " + "type is never covered by an existing grant." + ) + require_forward_range(starts_at, ends_at) + + +def require_forward_range(starts_at: date, ends_at: date | None) -> None: + if ends_at is not None and starts_at is not None and ends_at < starts_at: + raise BackwardsDateRange( + f"end date {ends_at.isoformat()} precedes start date " + f"{starts_at.isoformat()}." + ) + + +def is_active( + starts_at: date | None, + ends_at: date | None, + revoked_at: datetime | None, + on_date: date, +) -> bool: + """Whether a row is in force on ``on_date``. + + Revocation wins immediately and is not backdated: a revoked row is dead + from the moment it is revoked, which is the promise made to a landowner who + calls to change their mind. + """ + if revoked_at is not None: + return False + if starts_at is not None and on_date < starts_at: + return False + if ends_at is not None and on_date > ends_at: + return False + return True + + +def scope_covers( + scope_type: str, + scope_id: int | None, + thing_id: int | None, + group_ids: tuple[int, ...] = (), +) -> bool: + """Whether a grant's scope reaches the thing being asked about.""" + if scope_type == SCOPE_GLOBAL: + return True + if scope_type == SCOPE_THING: + return thing_id is not None and scope_id == thing_id + if scope_type == SCOPE_GROUP: + return scope_id in set(group_ids) + # An unrecognized scope type denies rather than raises: a row written by a + # newer version of the code must not read as permission to an older one. + return False + + +def grant_covers(grant: Grant, request: AccessRequest, on_date: date) -> bool: + """Whether one grant answers one request. Every axis must match.""" + if (grant.principal_type, grant.principal_id) not in request.principals: + return False + if grant.capability != request.capability: + return False + if grant.data_type != request.data_type: + return False + if not is_active(grant.starts_at, grant.ends_at, grant.revoked_at, on_date): + return False + return scope_covers( + grant.scope_type, grant.scope_id, request.thing_id, request.group_ids + ) + + +def any_grant_allows(grants, request: AccessRequest, on_date: date) -> bool: + """Default deny: an empty sequence is a no.""" + return any(grant_covers(grant, request, on_date) for grant in grants) + + +def consent_covers( + consent: Consent, + thing_id: int, + destination_id: int, + data_type: str, + on_date: date, +) -> bool: + """Whether one consent row publishes this data type of this well here.""" + return ( + consent.thing_id == thing_id + and consent.destination_id == destination_id + and consent.data_type == data_type + and is_active(consent.starts_at, consent.ends_at, consent.revoked_at, on_date) + ) + + +def any_consent_publishes( + consents, + thing_id: int, + destination_id: int, + data_type: str, + on_date: date, +) -> bool: + """Default deny, for the publication half.""" + return any( + consent_covers(consent, thing_id, destination_id, data_type, on_date) + for consent in consents + ) + + +def published_thing_ids(consents, destination_id: int, data_type: str, on_date: date): + """Thing ids a destination may read for one data type, in first-seen order.""" + seen = [] + for consent in consents: + if consent.thing_id in seen: + continue + if consent_covers( + consent, consent.thing_id, destination_id, data_type, on_date + ): + seen.append(consent.thing_id) + return seen + + +# ============= EOF ============================================= diff --git a/schemas/access.py b/schemas/access.py new file mode 100644 index 00000000..4a5fd86d --- /dev/null +++ b/schemas/access.py @@ -0,0 +1,145 @@ +# =============================================================================== +# Copyright 2025 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +schemas/access.py + +Request and response shapes for the ADR5 access-control routes: destinations, +permission grants, publication consent. + +These do not inherit ``BaseCreateModel`` / ``BaseResponseModel``. Those carry +``release_status`` and ``data_maturity``, which describe released *data*. A +grant is not data anybody releases -- it is the rule about who sees it. +""" + +from datetime import date + +from pydantic import BaseModel, ConfigDict, Field + +from core.enums import ( + AccessDataType, + Capability, + DestinationKind, + GrantScopeType, + PrincipalType, +) +from schemas import UTCAwareDatetime + + +# ------ DESTINATION ---------- +class CreateDestination(BaseModel): + slug: str = Field(max_length=50, examples=["ngwmn"]) + name: str = Field( + max_length=255, examples=["National Ground-Water Monitoring Network"] + ) + destination_kind: DestinationKind + description: str | None = None + + +class DestinationResponse(BaseModel): + model_config = ConfigDict(from_attributes=True) + + id: int + slug: str + name: str + destination_kind: DestinationKind + description: str | None + active: bool + + +# ------ PERMISSION GRANT ---------- +class CreatePermissionGrant(BaseModel): + """One grant. Every axis is named; there is no wildcard data type.""" + + principal_type: PrincipalType + principal_id: str = Field(max_length=255) + capability: Capability + scope_type: GrantScopeType + # Null for a global grant, required for a group- or thing-scoped one. The + # rule is enforced in domain/access.py rather than here, so it holds for + # every writer, not only this route. + scope_id: int | None = None + data_type: AccessDataType + starts_at: date + ends_at: date | None = None + reason: str | None = None + + +class PermissionGrantResponse(BaseModel): + model_config = ConfigDict(from_attributes=True) + + id: int + principal_type: PrincipalType + principal_id: str + capability: Capability + scope_type: GrantScopeType + scope_id: int | None + data_type: AccessDataType + starts_at: date + ends_at: date | None + granted_by: str + reason: str | None + revoked_at: UTCAwareDatetime | None + revoked_by: str | None + + +class AccessDecision(BaseModel): + """The visibility layer's answer, and what was asked.""" + + allowed: bool + capability: Capability + data_type: AccessDataType + thing_id: int | None + principals: list[str] + + +# ------ PUBLICATION CONSENT ---------- +class CreatePublicationConsent(BaseModel): + thing_id: int + destination_slug: str = Field(max_length=50) + data_type: AccessDataType + starts_at: date + ends_at: date | None = None + # Null when the Bureau owns the well: the decision was institutional, and + # inventing a consenting contact would be a lie. + contact_id: int | None = None + notes: str | None = None + + +class PublicationConsentResponse(BaseModel): + model_config = ConfigDict(from_attributes=True) + + id: int + thing_id: int + destination_id: int + data_type: AccessDataType + contact_id: int | None + recorded_by: str + notes: str | None + starts_at: date + ends_at: date | None + revoked_at: UTCAwareDatetime | None + revoked_by: str | None + + +class PublishedThing(BaseModel): + """One thing as a destination sees it: which data types it may read.""" + + thing_id: int + name: str | None + data_types: list[AccessDataType] + + +# ============= EOF ============================================= diff --git a/services/access_admin.py b/services/access_admin.py new file mode 100644 index 00000000..2d94946b --- /dev/null +++ b/services/access_admin.py @@ -0,0 +1,268 @@ +# =============================================================================== +# Copyright 2025 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +services/access_admin.py + +Writes to the access-control tables: register a destination, grant, revoke, +record a landowner's consent, withdraw it. + +Every write lands an ``authorization_audit`` row in the same transaction. That +is the point of the module: there is no path that changes who may see what and +leaves no trace, because the question after an incident is "who granted that, +and when" (ADR5, 4.3). + +Rules live in ``domain/access.py`` and are checked before the row is written, +so an unevaluable grant -- no data type, a global grant with a scope id -- +cannot reach the table. Domain errors subclass ValueError; the routes turn +them into 422s. +""" + +from datetime import date, datetime, timezone + +from db.authorization_audit import ( + AuthorizationAudit, + CONSENT_RECORDED, + CONSENT_REVOKED, + DESTINATION_REGISTERED, + GRANT_CREATED, + GRANT_REVOKED, +) +from db.destination import Destination +from db.permission_grant import PermissionGrant +from db.publication_consent import PublicationConsent +from domain.access import require_forward_range, validate_grant + +# Used when the caller's token carries no subject -- the development bypass, +# or a test override. Recorded rather than left null: "we do not know who" +# is itself worth knowing when reading the log back. +UNKNOWN_ACTOR = "unknown" + + +class AlreadyRevoked(ValueError): + """Raised when revoking something that is already revoked.""" + + +def actor_from_payload(payload) -> str: + """The identifier to record as having done this.""" + if not isinstance(payload, dict): + return UNKNOWN_ACTOR + return str(payload.get("sub") or payload.get("preferred_username") or UNKNOWN_ACTOR) + + +def _audit(session, actor, event_type, subject_table, subject_id, detail): + session.add( + AuthorizationAudit( + event_type=event_type, + actor=actor, + subject_table=subject_table, + subject_id=subject_id, + detail=detail, + ) + ) + + +def register_destination( + session, + actor: str, + slug: str, + name: str, + destination_kind: str, + description: str = None, +) -> Destination: + destination = Destination( + slug=slug, + name=name, + destination_kind=destination_kind, + description=description, + active=True, + ) + session.add(destination) + session.flush() + _audit( + session, + actor, + DESTINATION_REGISTERED, + Destination.__tablename__, + destination.id, + {"slug": slug, "name": name, "destination_kind": destination_kind}, + ) + session.commit() + session.refresh(destination) + return destination + + +def create_grant( + session, + actor: str, + principal_type: str, + principal_id: str, + capability: str, + scope_type: str, + scope_id: int, + data_type: str, + starts_at: date, + ends_at: date = None, + reason: str = None, +) -> PermissionGrant: + validate_grant( + principal_type=principal_type, + capability=capability, + scope_type=scope_type, + scope_id=scope_id, + data_type=data_type, + starts_at=starts_at, + ends_at=ends_at, + ) + + grant = PermissionGrant( + principal_type=principal_type, + principal_id=principal_id, + capability=capability, + scope_type=scope_type, + scope_id=scope_id, + data_type=data_type, + starts_at=starts_at, + ends_at=ends_at, + granted_by=actor, + reason=reason, + ) + session.add(grant) + session.flush() + _audit( + session, + actor, + GRANT_CREATED, + PermissionGrant.__tablename__, + grant.id, + { + "principal_type": principal_type, + "principal_id": principal_id, + "capability": capability, + "scope_type": scope_type, + "scope_id": scope_id, + "data_type": data_type, + "starts_at": starts_at.isoformat(), + "ends_at": ends_at.isoformat() if ends_at else None, + "reason": reason, + }, + ) + session.commit() + session.refresh(grant) + return grant + + +def revoke_grant(session, actor: str, grant: PermissionGrant) -> PermissionGrant: + """Revoke now. Effective at the next read, never backdated.""" + if grant.revoked_at is not None: + raise AlreadyRevoked(f"grant {grant.id} was already revoked.") + + grant.revoked_at = datetime.now(timezone.utc) + grant.revoked_by = actor + _audit( + session, + actor, + GRANT_REVOKED, + PermissionGrant.__tablename__, + grant.id, + { + "principal_type": grant.principal_type, + "principal_id": grant.principal_id, + "capability": grant.capability, + "data_type": grant.data_type, + }, + ) + session.commit() + session.refresh(grant) + return grant + + +def record_consent( + session, + actor: str, + thing_id: int, + destination_id: int, + data_type: str, + starts_at: date, + ends_at: date = None, + contact_id: int = None, + notes: str = None, +) -> PublicationConsent: + """Record that an owner agreed to publish this data type here.""" + require_forward_range(starts_at, ends_at) + + consent = PublicationConsent( + thing_id=thing_id, + destination_id=destination_id, + data_type=data_type, + contact_id=contact_id, + recorded_by=actor, + notes=notes, + starts_at=starts_at, + ends_at=ends_at, + ) + session.add(consent) + session.flush() + _audit( + session, + actor, + CONSENT_RECORDED, + PublicationConsent.__tablename__, + consent.id, + { + "thing_id": thing_id, + "destination_id": destination_id, + "data_type": data_type, + "contact_id": contact_id, + "starts_at": starts_at.isoformat(), + "ends_at": ends_at.isoformat() if ends_at else None, + }, + ) + session.commit() + session.refresh(consent) + return consent + + +def revoke_consent( + session, actor: str, consent: PublicationConsent +) -> PublicationConsent: + """Stop offering this. + + Not a recall: copies a harvester already took live in someone else's + system, and the owner should be told that rather than promised otherwise. + """ + if consent.revoked_at is not None: + raise AlreadyRevoked(f"consent {consent.id} was already revoked.") + + consent.revoked_at = datetime.now(timezone.utc) + consent.revoked_by = actor + _audit( + session, + actor, + CONSENT_REVOKED, + PublicationConsent.__tablename__, + consent.id, + { + "thing_id": consent.thing_id, + "destination_id": consent.destination_id, + "data_type": consent.data_type, + }, + ) + session.commit() + session.refresh(consent) + return consent + + +# ============= EOF ============================================= diff --git a/services/visibility.py b/services/visibility.py new file mode 100644 index 00000000..23e7619f --- /dev/null +++ b/services/visibility.py @@ -0,0 +1,245 @@ +# =============================================================================== +# Copyright 2025 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +services/visibility.py + +The one place that answers "what may this principal see, and what has this +destination been given" (ADR5, Part IV). + +Every consumer asks here. Not the public API filtering one way and a harvester +view another: three services filtering independently give three subtly +different answers to "is this published", and an outside party finds the +difference first. That is not hypothetical in this repository -- migration +baba91fe5e83 fixed exactly that between two OGC views. + +This module loads rows and hands them to ``domain/access.py``, which holds the +rules. It decides nothing itself. + +Grants are read at request time and never cached here. ADR5 A.5 allows a short +per-principal cache with explicit invalidation on revoke; an unbounded one +would silently defeat the immediate-revocation promise, so the first version +has none. +""" + +from datetime import date + +from sqlalchemy import select + +from db.destination import Destination +from db.group import GroupThingAssociation +from db.permission_grant import PermissionGrant +from db.publication_consent import PublicationConsent +from db.thing import Thing +from domain.access import ( + AccessRequest, + Consent, + Grant, + PRINCIPAL_ROLE, + PRINCIPAL_USER, + any_grant_allows, + consent_covers, +) + +# Claims carrying the caller's identity and their Authentik groups. +SUBJECT_CLAIM = "sub" +GROUPS_CLAIM = "groups" + + +def principals_from_payload(payload) -> tuple[tuple[str, str], ...]: + """Every identity a caller presents, as (principal_type, principal_id). + + A caller is their subject *and* each role they hold, because a grant may + name any of them. + + The development bypass returns ``True`` rather than a token payload, and an + anonymous caller has no payload at all. Both yield no principals, which + means default deny: the bypass turns off authentication, not authorization. + """ + if not isinstance(payload, dict): + return () + + principals = [] + subject = payload.get(SUBJECT_CLAIM) + if subject: + principals.append((PRINCIPAL_USER, str(subject))) + for group in payload.get(GROUPS_CLAIM) or []: + principals.append((PRINCIPAL_ROLE, str(group))) + return tuple(principals) + + +def group_ids_for_thing(session, thing_id: int) -> tuple[int, ...]: + """The groups a thing belongs to, which is what a project grant reaches.""" + if thing_id is None: + return () + rows = session.execute( + select(GroupThingAssociation.group_id).where( + GroupThingAssociation.thing_id == thing_id + ) + ).all() + return tuple(row[0] for row in rows) + + +def load_grants(session, principals: tuple[tuple[str, str], ...]) -> list[Grant]: + """Live and expired grants for these principals, as domain values. + + Expired and revoked rows are loaded rather than filtered in SQL so the + date rule lives in one place -- ``domain.access.is_active`` -- instead of + being restated as a WHERE clause that can drift from it. + """ + if not principals: + return [] + + principal_ids = {principal_id for _, principal_id in principals} + rows = session.execute( + select(PermissionGrant).where(PermissionGrant.principal_id.in_(principal_ids)) + ).scalars() + + return [ + Grant( + principal_type=row.principal_type, + principal_id=row.principal_id, + capability=row.capability, + scope_type=row.scope_type, + scope_id=row.scope_id, + data_type=row.data_type, + starts_at=row.starts_at, + ends_at=row.ends_at, + revoked_at=row.revoked_at, + ) + for row in rows + ] + + +def may( + session, + principals: tuple[tuple[str, str], ...], + capability: str, + data_type: str, + thing_id: int = None, + on_date: date = None, +) -> bool: + """May these principals do this, to this data type, at this thing? + + Default deny. No principals, no grants, or nothing matching is a no. + """ + request = AccessRequest( + capability=capability, + data_type=data_type, + principals=tuple(principals), + thing_id=thing_id, + group_ids=group_ids_for_thing(session, thing_id), + ) + return any_grant_allows( + load_grants(session, request.principals), request, on_date or date.today() + ) + + +def destination_by_slug(session, slug: str) -> Destination | None: + return session.execute( + select(Destination).where(Destination.slug == slug) + ).scalar_one_or_none() + + +def _consent_rows(session, destination_id: int, data_type: str = None): + statement = select(PublicationConsent).where( + PublicationConsent.destination_id == destination_id + ) + if data_type: + statement = statement.where(PublicationConsent.data_type == data_type) + return session.execute(statement).scalars().all() + + +def published_data_types( + session, destination: Destination, thing_id: int, on_date: date = None +) -> list[str]: + """Data types this destination may read for one thing.""" + if not destination.active: + return [] + + on_date = on_date or date.today() + return sorted( + { + row.data_type + for row in _consent_rows(session, destination.id) + if row.thing_id == thing_id + and consent_covers( + Consent( + thing_id=row.thing_id, + destination_id=row.destination_id, + data_type=row.data_type, + starts_at=row.starts_at, + ends_at=row.ends_at, + revoked_at=row.revoked_at, + ), + thing_id, + destination.id, + row.data_type, + on_date, + ) + } + ) + + +def published_things( + session, destination: Destination, data_type: str = None, on_date: date = None +) -> list[dict]: + """What this destination gets: one entry per thing, with its data types. + + A retired destination gets nothing, without the caller having to remember + to check. + """ + if not destination.active: + return [] + + on_date = on_date or date.today() + by_thing: dict[int, set] = {} + for row in _consent_rows(session, destination.id, data_type): + covered = consent_covers( + Consent( + thing_id=row.thing_id, + destination_id=row.destination_id, + data_type=row.data_type, + starts_at=row.starts_at, + ends_at=row.ends_at, + revoked_at=row.revoked_at, + ), + row.thing_id, + destination.id, + row.data_type, + on_date, + ) + if covered: + by_thing.setdefault(row.thing_id, set()).add(row.data_type) + + if not by_thing: + return [] + + names = dict( + session.execute( + select(Thing.id, Thing.name).where(Thing.id.in_(by_thing.keys())) + ).all() + ) + return [ + { + "thing_id": thing_id, + "name": names.get(thing_id), + "data_types": sorted(data_types), + } + for thing_id, data_types in sorted(by_thing.items()) + ] + + +# ============= EOF ============================================= diff --git a/tests/test_access.py b/tests/test_access.py new file mode 100644 index 00000000..3f39571c --- /dev/null +++ b/tests/test_access.py @@ -0,0 +1,361 @@ +# =============================================================================== +# Copyright 2025 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""The ADR5 visibility layer, end to end through its first tenant. + +These exercise the promise the model is for: an owner who shares water levels +but not chemistry, and a withdrawal that takes effect on the next read. +""" + +from datetime import date, timedelta + +import pytest +from sqlalchemy import delete, select + +from core.dependencies import admin_function, viewer_function +from main import app +from db.authorization_audit import AuthorizationAudit +from db.destination import Destination +from db.engine import session_ctx +from db.permission_grant import PermissionGrant +from db.publication_consent import PublicationConsent +from tests import client, override_authentication + +ADMIN_PAYLOAD = {"sub": "test-admin", "groups": ["Admin"]} +SLUG = "test-harvester" +TODAY = date.today() + + +@pytest.fixture(autouse=True) +def override_authentication_dependency_fixture(): + app.dependency_overrides[admin_function] = override_authentication( + default=ADMIN_PAYLOAD + ) + app.dependency_overrides[viewer_function] = override_authentication( + default=ADMIN_PAYLOAD + ) + + yield + + app.dependency_overrides = {} + + +@pytest.fixture +def destination(): + response = client.post( + "/access/destination", + json={ + "slug": SLUG, + "name": "Test Harvester", + "destination_kind": "harvester", + "description": "Stands in for NGWMN.", + }, + ) + assert response.status_code == 201, response.text + created = response.json() + + yield created + + with session_ctx() as session: + session.execute( + delete(PublicationConsent).where( + PublicationConsent.destination_id == created["id"] + ) + ) + session.execute(delete(Destination).where(Destination.id == created["id"])) + session.execute( + delete(AuthorizationAudit).where( + AuthorizationAudit.actor == ADMIN_PAYLOAD["sub"] + ) + ) + session.commit() + + +@pytest.fixture +def grants(): + created = [] + + yield created + + with session_ctx() as session: + for grant_id in created: + session.execute( + delete(PermissionGrant).where(PermissionGrant.id == grant_id) + ) + session.execute( + delete(AuthorizationAudit).where( + AuthorizationAudit.actor == ADMIN_PAYLOAD["sub"] + ) + ) + session.commit() + + +def consent_to(thing_id, data_type, **overrides): + payload = { + "thing_id": thing_id, + "destination_slug": SLUG, + "data_type": data_type, + "starts_at": TODAY.isoformat(), + } + payload.update(overrides) + return client.post("/access/consent", json=payload) + + +def published(**params): + response = client.get(f"/access/destination/{SLUG}/thing", params=params) + assert response.status_code == 200, response.text + return response.json() + + +# ------ destinations ---------- + + +def test_register_a_destination(destination): + assert destination["slug"] == SLUG + assert destination["active"] is True + + +def test_a_duplicate_slug_is_a_conflict(destination): + response = client.post( + "/access/destination", + json={"slug": SLUG, "name": "Again", "destination_kind": "harvester"}, + ) + assert response.status_code == 409 + + +def test_an_unknown_destination_is_a_404(): + response = client.get("/access/destination/nobody-registered-this/thing") + assert response.status_code == 404 + + +# ------ the case the model exists for ---------- + + +def test_levels_yes_chemistry_no(destination, water_well_thing): + """The kitchen-table promise: per data type, on one well.""" + assert consent_to(water_well_thing.id, "water level").status_code == 201 + + entries = published() + assert entries == [ + { + "thing_id": water_well_thing.id, + "name": water_well_thing.name, + "data_types": ["water level"], + } + ] + assert published(data_type="water chemistry") == [] + + +def test_nothing_is_published_without_consent(destination, water_well_thing): + """Default deny. A registered destination starts with nothing.""" + assert published() == [] + + +def test_withdrawal_takes_effect_on_the_next_read(destination, water_well_thing): + consent_id = consent_to(water_well_thing.id, "water level").json()["id"] + assert len(published()) == 1 + + revocation = client.post(f"/access/consent/{consent_id}/revocation") + assert revocation.status_code == 201 + assert revocation.json()["revoked_by"] == ADMIN_PAYLOAD["sub"] + + assert published() == [] + + +def test_consent_cannot_be_withdrawn_twice(destination, water_well_thing): + consent_id = consent_to(water_well_thing.id, "water level").json()["id"] + client.post(f"/access/consent/{consent_id}/revocation") + + assert client.post(f"/access/consent/{consent_id}/revocation").status_code == 422 + + +def test_withdrawn_consent_is_kept_for_the_record(destination, water_well_thing): + consent_id = consent_to(water_well_thing.id, "water level").json()["id"] + client.post(f"/access/consent/{consent_id}/revocation") + + live = client.get("/access/consent", params={"thing_id": water_well_thing.id}) + assert live.json() == [] + + history = client.get( + "/access/consent", + params={"thing_id": water_well_thing.id, "include_revoked": True}, + ) + assert [row["id"] for row in history.json()] == [consent_id] + + +def test_expired_consent_stops_publishing(destination, water_well_thing): + yesterday = TODAY - timedelta(days=1) + consent_to( + water_well_thing.id, + "water level", + starts_at=(TODAY - timedelta(days=30)).isoformat(), + ends_at=yesterday.isoformat(), + ) + + assert published() == [] + assert len(published(on_date=yesterday.isoformat())) == 1 + + +def test_consent_to_an_unknown_destination_is_a_404(water_well_thing): + response = client.post( + "/access/consent", + json={ + "thing_id": water_well_thing.id, + "destination_slug": "not-registered", + "data_type": "water level", + "starts_at": TODAY.isoformat(), + }, + ) + assert response.status_code == 404 + + +# ------ grants ---------- + + +def make_grant(grants, **overrides): + payload = { + "principal_type": "user", + "principal_id": ADMIN_PAYLOAD["sub"], + "capability": "read", + "scope_type": "global", + "scope_id": None, + "data_type": "water level", + "starts_at": TODAY.isoformat(), + "reason": "test", + } + payload.update(overrides) + response = client.post("/access/grant", json=payload) + if response.status_code == 201: + grants.append(response.json()["id"]) + return response + + +def decision(**params): + response = client.get("/access/decision", params=params) + assert response.status_code == 200, response.text + return response.json() + + +def test_a_grant_answers_only_for_its_data_type(grants): + assert make_grant(grants).status_code == 201 + + assert decision(capability="read", data_type="water level")["allowed"] is True + assert decision(capability="read", data_type="water chemistry")["allowed"] is False + + +def test_no_grant_is_a_no(grants): + assert decision(capability="read", data_type="water level")["allowed"] is False + + +def test_a_grant_answers_only_for_its_capability(grants): + make_grant(grants) + + assert decision(capability="correct", data_type="water level")["allowed"] is False + + +def test_revoking_a_grant_is_effective_immediately(grants): + grant_id = make_grant(grants).json()["id"] + assert decision(capability="read", data_type="water level")["allowed"] is True + + assert client.post(f"/access/grant/{grant_id}/revocation").status_code == 201 + + assert decision(capability="read", data_type="water level")["allowed"] is False + + +def test_a_thing_scoped_grant_stops_at_that_thing(grants, water_well_thing): + make_grant(grants, scope_type="thing", scope_id=water_well_thing.id) + + covered = decision( + capability="read", data_type="water level", thing_id=water_well_thing.id + ) + other = decision( + capability="read", data_type="water level", thing_id=water_well_thing.id + 9999 + ) + assert covered["allowed"] is True + assert other["allowed"] is False + + +def test_a_global_grant_with_a_scope_id_is_rejected(grants): + """The rule lives in domain/access.py; the route surfaces it as a 422.""" + assert make_grant(grants, scope_id=7).status_code == 422 + + +def test_a_grant_naming_no_data_type_cannot_be_written(grants): + """No wildcards. Pydantic rejects it before the service is reached.""" + assert make_grant(grants, data_type=None).status_code == 422 + + +def test_listing_grants_hides_revoked_ones_by_default(grants): + grant_id = make_grant(grants).json()["id"] + client.post(f"/access/grant/{grant_id}/revocation") + + live = client.get("/access/grant", params={"principal_id": ADMIN_PAYLOAD["sub"]}) + assert live.json() == [] + + history = client.get( + "/access/grant", + params={"principal_id": ADMIN_PAYLOAD["sub"], "include_revoked": True}, + ) + assert [row["id"] for row in history.json()] == [grant_id] + + +# ------ audit ---------- + + +def audit_events(): + with session_ctx() as session: + return [ + row.event_type + for row in session.execute( + select(AuthorizationAudit) + .where(AuthorizationAudit.actor == ADMIN_PAYLOAD["sub"]) + .order_by(AuthorizationAudit.id) + ).scalars() + ] + + +def test_every_authorization_change_is_logged(destination, grants, water_well_thing): + consent_id = consent_to(water_well_thing.id, "water level").json()["id"] + client.post(f"/access/consent/{consent_id}/revocation") + grant_id = make_grant(grants).json()["id"] + client.post(f"/access/grant/{grant_id}/revocation") + + assert audit_events() == [ + "destination.registered", + "consent.recorded", + "consent.revoked", + "grant.created", + "grant.revoked", + ] + + +def test_the_log_records_who_and_what(destination, water_well_thing): + consent_to(water_well_thing.id, "water level") + + with session_ctx() as session: + entry = ( + session.execute( + select(AuthorizationAudit) + .where(AuthorizationAudit.event_type == "consent.recorded") + .order_by(AuthorizationAudit.id.desc()) + ) + .scalars() + .first() + ) + + assert entry.actor == ADMIN_PAYLOAD["sub"] + assert entry.subject_table == "publication_consent" + assert entry.detail["thing_id"] == water_well_thing.id + assert entry.detail["data_type"] == "water level" diff --git a/tests/test_domain_access.py b/tests/test_domain_access.py new file mode 100644 index 00000000..35362b2c --- /dev/null +++ b/tests/test_domain_access.py @@ -0,0 +1,250 @@ +# =============================================================================== +# Copyright 2025 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""Access-control rules (ADR5). No database, no fixtures.""" + +from datetime import date, datetime, timezone + +import pytest + +from domain.access import ( + AccessRequest, + BackwardsDateRange, + Consent, + Grant, + MissingDataType, + ScopeIdMismatch, + UnknownCapability, + UnknownPrincipalType, + UnknownScopeType, + any_consent_publishes, + any_grant_allows, + consent_covers, + grant_covers, + is_active, + scope_covers, + validate_grant, +) + +TODAY = date(2026, 8, 24) +STUDENT = ("user", "authentik-sub-1") +EDITOR_ROLE = ("role", "Editor") + + +def a_grant(**overrides): + fields = { + "principal_type": "user", + "principal_id": "authentik-sub-1", + "capability": "read", + "scope_type": "thing", + "scope_id": 7, + "data_type": "water level", + "starts_at": date(2026, 1, 1), + "ends_at": None, + "revoked_at": None, + } + fields.update(overrides) + return Grant(**fields) + + +def a_request(**overrides): + fields = { + "capability": "read", + "data_type": "water level", + "principals": (STUDENT,), + "thing_id": 7, + "group_ids": (), + } + fields.update(overrides) + return AccessRequest(**fields) + + +# ------ default deny ---------- + + +def test_no_grants_is_a_no(): + assert any_grant_allows([], a_request(), TODAY) is False + + +def test_no_principals_is_a_no(): + assert any_grant_allows([a_grant()], a_request(principals=()), TODAY) is False + + +def test_a_matching_grant_allows(): + assert any_grant_allows([a_grant()], a_request(), TODAY) is True + + +# ------ every axis has to match ---------- + + +@pytest.mark.parametrize( + "override", + [ + {"capability": "correct"}, + {"data_type": "water chemistry"}, + {"principal_id": "somebody-else"}, + {"scope_id": 8}, + ], +) +def test_one_axis_off_denies(override): + assert grant_covers(a_grant(**override), a_request(), TODAY) is False + + +def test_principal_type_is_part_of_identity(): + """A role named the same as a subject is not the same principal.""" + grant = a_grant(principal_type="role", principal_id="authentik-sub-1") + assert grant_covers(grant, a_request(), TODAY) is False + + +def test_a_role_grant_covers_a_caller_holding_that_role(): + grant = a_grant(principal_type="role", principal_id="Editor") + request = a_request(principals=(STUDENT, EDITOR_ROLE)) + assert grant_covers(grant, request, TODAY) is True + + +# ------ scope ---------- + + +def test_global_scope_reaches_everything(): + assert scope_covers("global", None, thing_id=999, group_ids=()) is True + + +def test_group_scope_reaches_a_member_thing(): + assert scope_covers("group", 3, thing_id=7, group_ids=(3, 4)) is True + + +def test_group_scope_stops_at_a_non_member(): + assert scope_covers("group", 3, thing_id=7, group_ids=(4,)) is False + + +def test_thing_scope_needs_a_thing(): + assert scope_covers("thing", 7, thing_id=None, group_ids=()) is False + + +def test_an_unknown_scope_type_denies_rather_than_raising(): + """A row written by newer code must not read as permission to older code.""" + assert scope_covers("watershed", 1, thing_id=7, group_ids=()) is False + + +# ------ time ---------- + + +def test_a_grant_is_dead_before_it_starts(): + assert is_active(date(2026, 9, 1), None, None, TODAY) is False + + +def test_a_grant_is_dead_after_it_ends(): + assert is_active(date(2026, 1, 1), date(2026, 8, 23), None, TODAY) is False + + +def test_a_grant_is_live_on_its_last_day(): + assert is_active(date(2026, 1, 1), TODAY, None, TODAY) is True + + +def test_revocation_beats_the_date_range(): + revoked = datetime(2026, 8, 20, tzinfo=timezone.utc) + assert is_active(date(2026, 1, 1), None, revoked, TODAY) is False + + +def test_expiry_is_checked_at_use_not_swept(): + """The same row answers differently on different days, with no job run.""" + semester = a_grant(starts_at=date(2026, 1, 1), ends_at=date(2026, 5, 15)) + assert grant_covers(semester, a_request(), date(2026, 5, 15)) is True + assert grant_covers(semester, a_request(), date(2026, 5, 16)) is False + + +# ------ validation, before a row is written ---------- + + +def test_a_grant_without_a_data_type_is_rejected(): + """The no-wildcard rule: NULL does not mean 'all'.""" + with pytest.raises(MissingDataType): + validate_grant("user", "read", "global", None, None, TODAY, None) + + +def test_a_global_grant_carries_no_scope_id(): + with pytest.raises(ScopeIdMismatch): + validate_grant("user", "read", "global", 7, "water level", TODAY, None) + + +def test_a_thing_grant_needs_a_scope_id(): + with pytest.raises(ScopeIdMismatch): + validate_grant("user", "read", "thing", None, "water level", TODAY, None) + + +def test_an_unknown_capability_is_rejected(): + with pytest.raises(UnknownCapability): + validate_grant("user", "delete", "global", None, "water level", TODAY, None) + + +def test_an_unknown_principal_type_is_rejected(): + with pytest.raises(UnknownPrincipalType): + validate_grant("robot", "read", "global", None, "water level", TODAY, None) + + +def test_an_unknown_scope_type_is_rejected(): + with pytest.raises(UnknownScopeType): + validate_grant("user", "read", "watershed", 1, "water level", TODAY, None) + + +def test_a_backwards_date_range_is_rejected(): + with pytest.raises(BackwardsDateRange): + validate_grant( + "user", "read", "global", None, "water level", TODAY, date(2026, 1, 1) + ) + + +def test_validation_errors_are_value_errors(): + """The importers and the routes both rely on this (ADR4).""" + with pytest.raises(ValueError): + validate_grant("user", "read", "global", None, None, TODAY, None) + + +# ------ publication consent ---------- + + +def a_consent(**overrides): + fields = { + "thing_id": 7, + "destination_id": 2, + "data_type": "water level", + "starts_at": date(2026, 1, 1), + "ends_at": None, + "revoked_at": None, + } + fields.update(overrides) + return Consent(**fields) + + +def test_consent_publishes_the_type_it_names(): + assert consent_covers(a_consent(), 7, 2, "water level", TODAY) is True + + +def test_consent_to_levels_is_not_consent_to_chemistry(): + """The case the whole design exists for.""" + assert consent_covers(a_consent(), 7, 2, "water chemistry", TODAY) is False + + +def test_consent_is_per_destination(): + assert consent_covers(a_consent(), 7, 3, "water level", TODAY) is False + + +def test_withdrawn_consent_stops_being_offered_immediately(): + withdrawn = a_consent(revoked_at=datetime(2026, 8, 24, tzinfo=timezone.utc)) + assert any_consent_publishes([withdrawn], 7, 2, "water level", TODAY) is False + + +def test_nothing_published_without_a_consent_row(): + assert any_consent_publishes([], 7, 2, "water level", TODAY) is False From ceda4e45802fee1643e5d1d6452ce7e62c39b26f Mon Sep 17 00:00:00 2001 From: jakeross Date: Mon, 24 Aug 2026 22:44:30 -0700 Subject: [PATCH 04/13] feat(access): project published records field by field The attribute-level half of ADR5. Fields are published by allowlist, per audience, at one chokepoint, because the Bureau's promises are field-shaped: the owner's name and phone number sit on the same record as the water levels the owner agreed to share. core/field-allowlists.yml holds the allowlists, keyed by destination kind with a per-slug override that replaces rather than extends. An audience with no entry receives an empty record, not the whole one, so a column added next year is invisible until someone lists it deliberately. domain/field_projection.py holds the rules over plain dicts. services/field_projection.py parses the configuration, validates it in full on first read, and turns a model row into the record an audience receives. It is called from services/visibility.py, below the routes: api/access.py never projects anything itself, so a new route or output format cannot skip the rule by forgetting to call it. Validation is at load rather than at request time. An unknown field name, a transform on a field nobody publishes, or an allowlist naming a never-public field raises immediately, because the alternative is discovering a typo by noticing data that should have been there -- or data that should not have been. The never-public list wins over every allowlist and is applied twice, at load and again at projection. It currently covers provenance columns, legacy AMPAPI primary keys, and the free-text location and coordinate note columns, which are where gate codes, lock combinations and candid landowner notes have landed. Adding to that list is safe at any time; removing from it is a policy decision with a named owner, and ADR5 records that nobody has been named yet. Protection includes transformation, not only removal. `round` is the first transform: the same well reaches a harvester at four decimal places and the public web at two. Rounded rather than withheld, so protecting a landowner does not mean dropping the well off the map. /access/destination/{slug}/thing now returns the projected record and location alongside the consented data types. Not covered: the ogc_* views still select their own columns in SQL and gate on release_status, so nothing on this path applies to them yet; only thing and location are projectable entities; and field rules between internal roles -- contact information being AMP-only -- are not implemented. Documented in docs/access-field-projection.md. 22 tests, most needing no database. Co-Authored-By: Claude Opus 5 --- ADR5.md | 6 +- CLAUDE.md | 8 ++ core/field-allowlists.yml | 128 +++++++++++++++++++++ docs/access-field-projection.md | 96 ++++++++++++++++ domain/field_projection.py | 144 ++++++++++++++++++++++++ schemas/access.py | 12 +- services/field_projection.py | 164 +++++++++++++++++++++++++++ services/visibility.py | 71 +++++++++--- tests/test_access.py | 101 +++++++++++++++-- tests/test_field_projection.py | 189 ++++++++++++++++++++++++++++++++ 10 files changed, 895 insertions(+), 24 deletions(-) create mode 100644 core/field-allowlists.yml create mode 100644 docs/access-field-projection.md create mode 100644 domain/field_projection.py create mode 100644 services/field_projection.py create mode 100644 tests/test_field_projection.py diff --git a/ADR5.md b/ADR5.md index 6d4d7fb2..5ec810bc 100644 --- a/ADR5.md +++ b/ADR5.md @@ -211,7 +211,11 @@ told. `authorization_audit` exist; `services/visibility.py` is the single evaluator; `api/access.py` is its one tenant. No existing endpoint routes through it yet. -4. Field projection at the serialization chokepoint, with the never-public list. +4. **Done for the payloads the visibility layer builds.** + `services/field_projection.py` is the chokepoint, `core/field-allowlists.yml` + holds the per-audience allowlists and the never-public list, and coordinate + rounding is the first transform. The OGC views still select their own + columns in SQL and do not pass through it. 5. Console administration. 6. Healy migration, after the data owner decides grandfathering. diff --git a/CLAUDE.md b/CLAUDE.md index 3c793ebe..b0c89530 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -233,6 +233,14 @@ The storage and the evaluator exist; the field projection does not. endpoint consults the layer yet, so release_status still governs what the OGC views publish. The prefix is `/access`, not `/publication`, because `api/publication.py` is the bibliography. +- **Fields are published by allowlist, per audience.** + `core/field-allowlists.yml` says what each audience receives; + `services/field_projection.py` applies it below the routes, so a new route + cannot skip it. An audience with no entry gets an empty record, and the + `never_public` block overrides every allowlist. Protection includes + transformation -- public coordinates are rounded, not withheld. The OGC + views do **not** pass through this yet. Read + **`docs/access-field-projection.md`** before touching the allowlists. - **Default deny, no wildcards, expiry at use.** A grant with no matching row is a no; a grant names its `data_type` (there is no term meaning "all"); and nothing sweeps expired rows, so every check compares against the date asked diff --git a/core/field-allowlists.yml b/core/field-allowlists.yml new file mode 100644 index 00000000..acd1505b --- /dev/null +++ b/core/field-allowlists.yml @@ -0,0 +1,128 @@ +# Per-audience field allowlists for published records (ADR5, 3.5 and A.2). +# +# Two rules this file exists to keep: +# +# 1. Omission produces silence. Only fields listed here reach an audience, so +# a column added next year is invisible until someone adds it deliberately. +# An audience with no entry receives nothing -- default deny, not "all". +# +# 2. `never_public` wins over every allowlist. Engineering guarantees these +# fields are enforced everywhere; what belongs on the list is a policy +# decision with a named data owner, and that owner has not been named yet +# (ADR5, "What this does not decide"). Adding a field here is safe at any +# time. Removing one is not an engineering call. +# +# Enforced by services/field_projection.py, which validates the whole file at +# load: an unknown field, a transform on a field nobody publishes, or an +# allowlist naming a never-public field raises rather than degrading quietly. +# +# Audiences are matched by destination slug first, then by destination kind. +# `round` reduces coordinate precision: 2 decimal places is roughly a +# kilometre, 4 is roughly ten metres. + +never_public: + thing: + # Provenance columns. Who typed a record is internal, always. + - created_by_id + - created_by_name + - updated_by_id + - updated_by_name + # Legacy AMPAPI primary keys. Publishing them invites outside systems to + # key on identifiers the migration is meant to retire. + - nma_pk_welldata + - nma_pk_location + - search_vector + location: + - created_by_id + - created_by_name + - updated_by_id + - updated_by_name + - nma_pk_location + # Free text written by staff for staff. The permissions interview was + # explicit that gate codes, lock combinations and candid landowner notes + # must never leave the Bureau, and this is where such text has landed. + - nma_location_notes + - nma_coordinate_notes + +audiences: + # Per-destination-kind defaults. A newly registered destination inherits the + # rules for its kind; nothing inherits "everything". + by_kind: + public web: + thing: + fields: + - id + - name + - thing_type + - well_depth + - hole_depth + - well_completion_date + location: + fields: + - id + - latitude + - longitude + - elevation + - county + - state + # Protect the landowner without dropping the well off the map. + transforms: + latitude: + round: 2 + longitude: + round: 2 + + harvester: + thing: + fields: + - id + - name + - thing_type + - well_depth + - hole_depth + - well_casing_depth + - well_completion_date + location: + fields: + - id + - latitude + - longitude + - elevation + - county + - state + - quad_name + # A harvester plotting hydrographs needs the well in the right place; + # ten metres is close enough for that and not close enough to walk to. + transforms: + latitude: + round: 4 + longitude: + round: 4 + + partner agency: + thing: + fields: + - id + - name + - thing_type + - well_depth + - hole_depth + - well_casing_diameter + - well_casing_depth + - well_completion_date + - well_construction_method + location: + fields: + - id + - latitude + - longitude + - elevation + - county + - state + - quad_name + - description + + # Per-destination overrides, keyed by slug. An entry here replaces the rules + # for that destination's kind rather than adding to them, so what one partner + # receives is readable in one place. + by_slug: {} diff --git a/docs/access-field-projection.md b/docs/access-field-projection.md new file mode 100644 index 00000000..f3c3fea0 --- /dev/null +++ b/docs/access-field-projection.md @@ -0,0 +1,96 @@ +# Field projection: what an audience actually receives + +Read this before editing `core/field-allowlists.yml`, adding a projectable +entity, or wiring a new consumer into published payloads. + +The decision behind it is [ADR5](../ADR5.md), sections 3.5 and A.2. + +## The rule + +Fields are published by **allowlist, per audience, at one chokepoint**. + +- Only fields named for an audience appear in that audience's payload. + Omission produces silence, not leakage: a column added next year is + invisible to everyone outside the Bureau until someone lists it. +- An audience with no entry receives an **empty record**, not the whole one. +- Protection includes **transformation**, not only removal. A public record can + carry a coordinate rounded to protect the landowner while the precise value + stays internal. +- The **never-public list wins** over every allowlist, and is applied twice: + when the configuration loads, and again when a record is projected. + +## Where the pieces are + +| Piece | Path | Job | +| --- | --- | --- | +| Rules | `domain/field_projection.py` | Omit, transform, validate. Plain dicts; no database, no config parsing. | +| Configuration + chokepoint | `services/field_projection.py` | Parse and validate the YAML, build a record from a model row, project it. | +| Configuration | `core/field-allowlists.yml` | The allowlists and the never-public list. | +| Only current consumer | `services/visibility.py` (`published_things`) | Builds every published payload through `project_entity`. | + +`api/access.py` never projects anything itself. That is the point: the +projection sits *below* the routes, so a new route or a new output format +cannot skip it by forgetting to call it. + +## Adding a field for an audience + +1. Add it to that audience's `fields` list in `core/field-allowlists.yml`, + under `audiences.by_kind.` or `audiences.by_slug.`. +2. Run the tests. The configuration is validated on load, so a typo, a field + the entity does not have, or a field on the never-public list fails + immediately rather than quietly withholding or exposing data. + +A `by_slug` entry **replaces** the rules for that destination's kind rather +than extending them, so what one partner receives is readable in one place. + +## Adding a transform + +Transforms live in `TRANSFORMS` in `domain/field_projection.py` and are named +in the YAML as `field: {transform: argument}`. Only `round` exists today: + +```yaml +transforms: + latitude: + round: 2 +``` + +Two decimal places is roughly a kilometre; four is roughly ten metres. A +transform on a field that is not in the same allowlist raises at load, because +it would never run. + +## The never-public list + +`never_public` in the YAML is the list no configuration can override. +Engineering guarantees that whatever is listed is enforced everywhere. +Engineering **cannot decide what belongs on it** — that needs a named data +owner, and ADR5 records that nobody has been named yet. + +What that means in practice: + +- **Adding** a field to `never_public` is safe at any time and needs no + ceremony. +- **Removing** one is not an engineering call. If a removal seems necessary, + the question goes to the data owner, not into a pull request. + +Currently listed: provenance columns (`created_by_*`, `updated_by_*`), legacy +AMPAPI primary keys, and the free-text location and coordinate note columns — +the permissions interview was explicit that gate codes, lock combinations and +candid landowner notes must never leave the Bureau, and that is the text those +columns have collected. + +## What this does *not* cover yet + +The OGC collections do **not** go through the projection. `ogc_*` views select +their own column lists in SQL and gate on `release_status`, exactly the +distributed filtering ADR5 argues against. Bringing them under the chokepoint +is later work, and until it happens the guarantees on this page apply only to +payloads built by `services/visibility.py`. + +Only `thing` and `location` are projectable entities. Adding another means +adding it to `ENTITY_MODELS` (and `DERIVED_FIELDS` if the payload carries +values that are not columns, the way `latitude` and `longitude` stand in for +the PostGIS `point`). + +Field rules are per *audience* today. ADR5 also asks for field rules between +internal roles — contact information being AMP-only is one — and that case is +not implemented. diff --git a/domain/field_projection.py b/domain/field_projection.py new file mode 100644 index 00000000..8ddb09f2 --- /dev/null +++ b/domain/field_projection.py @@ -0,0 +1,144 @@ +# =============================================================================== +# Copyright 2025 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Field projection: what an audience actually receives, field by field (ADR5, 3.5). + +Record-level grants are not enough, because the Bureau's promises are +field-shaped. The owner's name and phone number sit on the same well record as +the water levels the owner agreed to share. + +The rule is an allowlist, per audience: + +* **Omission produces silence, not leakage.** Only fields explicitly approved + for an audience appear in that audience's payload, so a column added next + year is invisible until someone decides otherwise. +* **Protection includes transformation.** A public record can carry a + coordinate rounded to protect the landowner while the precise value stays + internal. Dropping the well from the map entirely is not the only option. +* **The never-public list wins over any allowlist.** Engineering guarantees a + listed field is enforced everywhere; a named data owner decides what is on + the list. A configuration that names one is a mistake, and it is raised at + load rather than honored at request time. + +Plain values only: dicts in, dicts out. ``services/field_projection.py`` reads +the configuration and supplies them. +""" + +from dataclasses import dataclass, field as dataclass_field + + +class FieldProjectionError(ValueError): + """Base for projection configuration problems.""" + + +class NeverPublicFieldAllowed(FieldProjectionError): + """An allowlist named a field no configuration may expose.""" + + +class UnknownField(FieldProjectionError): + """An allowlist named a field the entity does not have.""" + + +class UnknownTransform(FieldProjectionError): + """A transform nobody implements.""" + + +@dataclass(frozen=True) +class EntityProjection: + """The rule for one entity, for one audience.""" + + fields: frozenset + # field name -> (transform name, argument), e.g. "latitude" -> ("round", 2) + transforms: dict = dataclass_field(default_factory=dict) + + +def round_to(value, places: int): + """Reduce coordinate precision. Two decimal places is roughly a kilometre.""" + if value is None: + return None + return round(float(value), places) + + +# Transformations a configuration may ask for. Anything else raises at load, +# so a typo cannot silently degrade to "publish the value untouched". +TRANSFORMS = {"round": round_to} + + +def validate_projection( + entity: str, + fields, + transforms: dict, + known_fields, + never_public, +) -> None: + """Reject a projection that could not be honored, before it is used.""" + unknown = sorted(set(fields) - set(known_fields)) + if unknown: + raise UnknownField( + f"{entity} has no field(s) {', '.join(unknown)}. " + "An allowlist naming a field that does not exist is a typo, and a " + "typo in an allowlist silently withholds data." + ) + + forbidden = sorted(set(fields) & set(never_public)) + if forbidden: + raise NeverPublicFieldAllowed( + f"{entity}.{forbidden[0]} is on the never-public list and cannot " + "be added to an audience. Removing it from that list is a policy " + "decision with a named owner, not a configuration change." + ) + + for field_name, (transform_name, _) in transforms.items(): + if field_name not in fields: + raise UnknownField( + f"{entity}.{field_name} has a transform but is not in the " + "allowlist, so the transform would never run." + ) + if transform_name not in TRANSFORMS: + raise UnknownTransform( + f"'{transform_name}' is not a transform " + f"({', '.join(sorted(TRANSFORMS))})." + ) + + +def project(record: dict, projection: EntityProjection, never_public=frozenset()): + """One record, as this audience receives it. + + Default deny: an audience with no rule gets an empty dict, not the record. + The never-public list is applied here as well as at load, so a field on it + stays out even if a projection was built without validation. + """ + if projection is None: + return {} + + projected = {} + for field_name, value in record.items(): + if field_name not in projection.fields: + continue + if field_name in never_public: + continue + + transform = projection.transforms.get(field_name) + if transform is not None: + transform_name, argument = transform + value = TRANSFORMS[transform_name](value, argument) + + projected[field_name] = value + + return projected + + +# ============= EOF ============================================= diff --git a/schemas/access.py b/schemas/access.py index 4a5fd86d..082a0e81 100644 --- a/schemas/access.py +++ b/schemas/access.py @@ -135,11 +135,19 @@ class PublicationConsentResponse(BaseModel): class PublishedThing(BaseModel): - """One thing as a destination sees it: which data types it may read.""" + """One thing as a destination sees it. + + ``thing_id`` and ``data_types`` are the envelope for staff reading this + route; ``properties`` and ``location`` are what the destination itself + receives, already projected through the per-audience allowlist. A field + nobody approved for this audience is absent rather than null, and a + coordinate may arrive rounded. + """ thing_id: int - name: str | None data_types: list[AccessDataType] + properties: dict + location: dict # ============= EOF ============================================= diff --git a/services/field_projection.py b/services/field_projection.py new file mode 100644 index 00000000..2222a103 --- /dev/null +++ b/services/field_projection.py @@ -0,0 +1,164 @@ +# =============================================================================== +# Copyright 2025 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +services/field_projection.py + +Loads ``core/field-allowlists.yml`` and turns a database row into the record an +audience actually receives. + +This is the chokepoint from ADR5 A.2. Every published payload is built here, so +a new route or a new output format cannot skip the rule by omission -- it sits +below them. The rules themselves are in ``domain/field_projection.py``. + +The whole file is validated the first time it is read: an unknown field name, a +transform on a field nobody publishes, or an allowlist naming a never-public +field raises. Failing at load rather than at request time is deliberate; the +alternative is discovering a typo by noticing data that should have been there, +or worse, data that should not have been. +""" + +from functools import lru_cache +from pathlib import Path + +import yaml + +from db.location import Location +from db.thing import Thing +from domain.field_projection import EntityProjection, project, validate_projection + +CONFIG_PATH = Path(__file__).parent.parent / "core" / "field-allowlists.yml" + +THING = "thing" +LOCATION = "location" + +# Fields a projection may name, per entity. Model columns plus the derived +# values a payload carries instead of raw storage: `point` is a PostGIS +# geometry nobody consumes directly, so `latitude` and `longitude` stand in +# for it and can be rounded independently. +DERIVED_FIELDS = { + THING: frozenset(), + LOCATION: frozenset({"latitude", "longitude"}), +} +ENTITY_MODELS = {THING: Thing, LOCATION: Location} + + +def known_fields(entity: str) -> frozenset: + model = ENTITY_MODELS[entity] + columns = {column.name for column in model.__table__.columns} + return frozenset(columns | DERIVED_FIELDS[entity]) + + +@lru_cache(maxsize=1) +def _configuration(path: str = None) -> dict: + """Parse and validate the allowlists once per process.""" + raw = yaml.safe_load(Path(path or CONFIG_PATH).read_text(encoding="utf-8")) + + never_public = { + entity: frozenset(fields or ()) + for entity, fields in (raw.get("never_public") or {}).items() + } + + audiences = {} + for keyed_by in ("by_kind", "by_slug"): + for audience, entities in ( + (raw.get("audiences") or {}).get(keyed_by) or {} + ).items(): + for entity, rule in (entities or {}).items(): + projection = _build_projection(entity, rule, never_public) + audiences[(keyed_by, audience, entity)] = projection + + return {"never_public": never_public, "audiences": audiences} + + +def _build_projection(entity: str, rule, never_public: dict) -> EntityProjection: + if entity not in ENTITY_MODELS: + raise KeyError( + f"'{entity}' is not a projectable entity " + f"({', '.join(sorted(ENTITY_MODELS))})." + ) + + # A bare list is the common case; the mapping form adds transforms. + if isinstance(rule, list): + fields, raw_transforms = rule, {} + else: + fields = (rule or {}).get("fields") or [] + raw_transforms = (rule or {}).get("transforms") or {} + + transforms = {} + for field_name, spec in raw_transforms.items(): + # One transform per field: {"round": 2} -> ("round", 2). + ((transform_name, argument),) = spec.items() + transforms[field_name] = (transform_name, argument) + + validate_projection( + entity=entity, + fields=fields, + transforms=transforms, + known_fields=known_fields(entity), + never_public=never_public.get(entity, frozenset()), + ) + return EntityProjection(fields=frozenset(fields), transforms=transforms) + + +def projection_for(destination, entity: str) -> EntityProjection | None: + """The rule for this destination, or None -- which means nothing is sent. + + A per-destination entry replaces its kind's rules rather than extending + them, so what one audience receives is readable in one place. + """ + configuration = _configuration()["audiences"] + by_slug = configuration.get(("by_slug", destination.slug, entity)) + if by_slug is not None: + return by_slug + return configuration.get(("by_kind", destination.destination_kind, entity)) + + +def never_public_fields(entity: str) -> frozenset: + return _configuration()["never_public"].get(entity, frozenset()) + + +def thing_record(thing) -> dict: + """Every stored field of a thing, before projection.""" + return { + column.name: getattr(thing, column.name) + for column in Thing.__table__.columns + if column.name != "search_vector" + } + + +def location_record(location) -> dict: + """Every stored field of a location, with the geometry as lat/lon.""" + record = { + column.name: getattr(location, column.name) + for column in Location.__table__.columns + if column.name != "point" + } + latitude, longitude = location.latlon + record["latitude"] = latitude + record["longitude"] = longitude + return record + + +def project_entity(destination, entity: str, record: dict) -> dict: + """Apply this destination's rule to one record.""" + return project( + record, + projection_for(destination, entity), + never_public=never_public_fields(entity), + ) + + +# ============= EOF ============================================= diff --git a/services/visibility.py b/services/visibility.py index 23e7619f..57becb33 100644 --- a/services/visibility.py +++ b/services/visibility.py @@ -26,7 +26,8 @@ baba91fe5e83 fixed exactly that between two OGC views. This module loads rows and hands them to ``domain/access.py``, which holds the -rules. It decides nothing itself. +rules, and to ``services/field_projection.py``, which decides field by field +what an audience receives. It decides nothing itself. Grants are read at request time and never cached here. ADR5 A.5 allows a short per-principal cache with explicit invalidation on revoke; an unbounded one @@ -40,9 +41,17 @@ from db.destination import Destination from db.group import GroupThingAssociation +from db.location import Location, LocationThingAssociation from db.permission_grant import PermissionGrant from db.publication_consent import PublicationConsent from db.thing import Thing +from services.field_projection import ( + LOCATION, + THING, + location_record, + project_entity, + thing_record, +) from domain.access import ( AccessRequest, Consent, @@ -227,19 +236,53 @@ def published_things( if not by_thing: return [] - names = dict( - session.execute( - select(Thing.id, Thing.name).where(Thing.id.in_(by_thing.keys())) - ).all() - ) - return [ - { - "thing_id": thing_id, - "name": names.get(thing_id), - "data_types": sorted(data_types), - } - for thing_id, data_types in sorted(by_thing.items()) - ] + things = { + thing.id: thing + for thing in session.execute( + select(Thing).where(Thing.id.in_(by_thing.keys())) + ).scalars() + } + locations = _current_locations(session, by_thing.keys()) + + entries = [] + for thing_id, data_types in sorted(by_thing.items()): + thing = things.get(thing_id) + if thing is None: + # Consent outliving its thing is a data problem, not a reason to + # publish a record nobody can check. + continue + + location = locations.get(thing_id) + entries.append( + { + "thing_id": thing_id, + "data_types": sorted(data_types), + # Field by field, per audience, at the one chokepoint. A field + # nobody approved for this destination is absent, and a + # coordinate may arrive rounded rather than exact. + "properties": project_entity(destination, THING, thing_record(thing)), + "location": ( + project_entity(destination, LOCATION, location_record(location)) + if location is not None + else {} + ), + } + ) + + return entries + + +def _current_locations(session, thing_ids) -> dict: + """The location each thing sits at now, keyed by thing id.""" + rows = session.execute( + select(LocationThingAssociation.thing_id, Location) + .join(Location, Location.id == LocationThingAssociation.location_id) + .where( + LocationThingAssociation.thing_id.in_(thing_ids), + LocationThingAssociation.effective_end.is_(None), + ) + ).all() + return {thing_id: location for thing_id, location in rows} # ============= EOF ============================================= diff --git a/tests/test_access.py b/tests/test_access.py index 3f39571c..b499c50f 100644 --- a/tests/test_access.py +++ b/tests/test_access.py @@ -83,6 +83,36 @@ def destination(): session.commit() +@pytest.fixture +def public_destination(): + response = client.post( + "/access/destination", + json={ + "slug": "test-public-web", + "name": "Test Public Web", + "destination_kind": "public web", + }, + ) + assert response.status_code == 201, response.text + created = response.json() + + yield created + + with session_ctx() as session: + session.execute( + delete(PublicationConsent).where( + PublicationConsent.destination_id == created["id"] + ) + ) + session.execute(delete(Destination).where(Destination.id == created["id"])) + session.execute( + delete(AuthorizationAudit).where( + AuthorizationAudit.actor == ADMIN_PAYLOAD["sub"] + ) + ) + session.commit() + + @pytest.fixture def grants(): created = [] @@ -148,16 +178,73 @@ def test_levels_yes_chemistry_no(destination, water_well_thing): assert consent_to(water_well_thing.id, "water level").status_code == 201 entries = published() - assert entries == [ - { - "thing_id": water_well_thing.id, - "name": water_well_thing.name, - "data_types": ["water level"], - } - ] + assert len(entries) == 1 + assert entries[0]["thing_id"] == water_well_thing.id + assert entries[0]["data_types"] == ["water level"] + assert entries[0]["properties"]["name"] == water_well_thing.name + assert published(data_type="water chemistry") == [] +# ------ field projection ---------- + + +def test_a_destination_receives_only_its_allowlisted_fields( + destination, water_well_thing +): + consent_to(water_well_thing.id, "water level") + properties = published()[0]["properties"] + + # Named for the harvester audience in core/field-allowlists.yml. + assert set(properties) == { + "id", + "name", + "thing_type", + "well_depth", + "hole_depth", + "well_casing_depth", + "well_completion_date", + } + + +def test_never_public_fields_reach_nobody(destination, water_well_thing): + consent_to(water_well_thing.id, "water level") + entry = published()[0] + + for column in ("created_by_id", "created_by_name", "nma_pk_welldata"): + assert column not in entry["properties"] + for column in ("nma_location_notes", "nma_coordinate_notes"): + assert column not in entry["location"] + + +def test_coordinates_are_rounded_for_the_audience( + destination, public_destination, water_well_thing +): + """The same well, two audiences, two precisions -- not published or hidden.""" + consent_to(water_well_thing.id, "water level") + client.post( + "/access/consent", + json={ + "thing_id": water_well_thing.id, + "destination_slug": public_destination["slug"], + "data_type": "water level", + "starts_at": TODAY.isoformat(), + }, + ) + + harvester_location = published()[0]["location"] + public_response = client.get( + f"/access/destination/{public_destination['slug']}/thing" + ) + public_location = public_response.json()[0]["location"] + + assert harvester_location["latitude"] == round(harvester_location["latitude"], 4) + assert public_location["latitude"] == round(public_location["latitude"], 2) + assert public_location["latitude"] != harvester_location["latitude"] + # Rounded, not withheld: the well still appears on the public map. + assert public_location["latitude"] is not None + + def test_nothing_is_published_without_consent(destination, water_well_thing): """Default deny. A registered destination starts with nothing.""" assert published() == [] diff --git a/tests/test_field_projection.py b/tests/test_field_projection.py new file mode 100644 index 00000000..1f9514c7 --- /dev/null +++ b/tests/test_field_projection.py @@ -0,0 +1,189 @@ +# =============================================================================== +# Copyright 2025 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""Field projection (ADR5, 3.5). + +The rule tests need no database. The loader tests import the models to know +what fields exist, but never connect. +""" + +import pytest + +from domain.field_projection import ( + EntityProjection, + NeverPublicFieldAllowed, + UnknownField, + UnknownTransform, + project, + round_to, + validate_projection, +) +from services.field_projection import ( + LOCATION, + THING, + _build_projection, + _configuration, + known_fields, + never_public_fields, + projection_for, +) + +RECORD = { + "id": 1, + "name": "Test Well", + "well_depth": 120.0, + "created_by_id": "authentik-sub-1", + "nma_pk_welldata": 4321, +} + + +class FakeDestination: + def __init__(self, slug, destination_kind): + self.slug = slug + self.destination_kind = destination_kind + + +# ------ the allowlist ---------- + + +def test_only_listed_fields_survive(): + projection = EntityProjection(fields=frozenset({"id", "name"})) + assert project(RECORD, projection) == {"id": 1, "name": "Test Well"} + + +def test_a_field_nobody_listed_is_absent_not_null(): + """Omission produces silence: the key is gone, not set to None.""" + projection = EntityProjection(fields=frozenset({"id"})) + assert "well_depth" not in project(RECORD, projection) + + +def test_no_projection_publishes_nothing(): + """Default deny. An audience with no rule receives an empty record.""" + assert project(RECORD, None) == {} + + +def test_a_new_field_is_invisible_until_someone_lists_it(): + projection = EntityProjection(fields=frozenset({"id", "name"})) + with_new_column = dict(RECORD, gate_code="1234") + assert "gate_code" not in project(with_new_column, projection) + + +# ------ never-public ---------- + + +def test_never_public_beats_an_allowlist_that_asked_for_it(): + """Belt and braces: enforced at load, and again at projection time.""" + projection = EntityProjection(fields=frozenset({"id", "created_by_id"})) + projected = project(RECORD, projection, never_public=frozenset({"created_by_id"})) + assert projected == {"id": 1} + + +def test_configuring_a_never_public_field_raises_at_load(): + with pytest.raises(NeverPublicFieldAllowed): + validate_projection( + entity=THING, + fields=["id", "created_by_id"], + transforms={}, + known_fields=known_fields(THING), + never_public=frozenset({"created_by_id"}), + ) + + +def test_provenance_columns_are_never_public(): + assert "created_by_id" in never_public_fields(THING) + assert "updated_by_name" in never_public_fields(THING) + + +def test_staff_written_location_notes_are_never_public(): + """Gate codes and candid landowner notes have landed in these columns.""" + assert "nma_location_notes" in never_public_fields(LOCATION) + assert "nma_coordinate_notes" in never_public_fields(LOCATION) + + +# ------ transformation ---------- + + +def test_a_coordinate_can_be_rounded_rather_than_dropped(): + projection = EntityProjection( + fields=frozenset({"latitude"}), transforms={"latitude": ("round", 2)} + ) + assert project({"latitude": 33.809712}, projection) == {"latitude": 33.81} + + +def test_rounding_leaves_a_missing_coordinate_missing(): + assert round_to(None, 2) is None + + +def test_an_untransformed_field_passes_through(): + projection = EntityProjection( + fields=frozenset({"latitude", "elevation"}), + transforms={"latitude": ("round", 2)}, + ) + projected = project({"latitude": 33.809712, "elevation": 2464.9}, projection) + assert projected["elevation"] == 2464.9 + + +# ------ configuration validation ---------- + + +def test_an_unknown_field_raises(): + """A typo in an allowlist silently withholds data, so it fails loudly.""" + with pytest.raises(UnknownField): + _build_projection(THING, ["id", "welll_depth"], {}) + + +def test_an_unknown_entity_raises(): + with pytest.raises(KeyError): + _build_projection("borehole", ["id"], {}) + + +def test_an_unknown_transform_raises(): + with pytest.raises(UnknownTransform): + _build_projection( + LOCATION, + {"fields": ["latitude"], "transforms": {"latitude": {"fuzz": 2}}}, + {}, + ) + + +def test_a_transform_on_an_unpublished_field_raises(): + with pytest.raises(UnknownField): + _build_projection( + LOCATION, {"fields": ["id"], "transforms": {"latitude": {"round": 2}}}, {} + ) + + +def test_the_shipped_configuration_is_valid(): + """Loading validates every audience; this asserts it stays that way.""" + assert _configuration()["audiences"] + + +# ------ audience lookup ---------- + + +def test_a_destination_inherits_the_rules_for_its_kind(): + projection = projection_for(FakeDestination("ngwmn", "harvester"), THING) + assert "name" in projection.fields + + +def test_an_unregistered_kind_receives_nothing(): + assert projection_for(FakeDestination("mystery", "carrier pigeon"), THING) is None + + +def test_the_public_web_gets_a_coarser_coordinate_than_a_harvester(): + public = projection_for(FakeDestination("web", "public web"), LOCATION) + harvester = projection_for(FakeDestination("ngwmn", "harvester"), LOCATION) + assert public.transforms["latitude"] == ("round", 2) + assert harvester.transforms["latitude"] == ("round", 4) From 48ad79792f7ff11ebcd11b1c0bce2bf7e6d50262 Mon Sep 17 00:00:00 2001 From: jakeross Date: Mon, 24 Aug 2026 22:54:53 -0700 Subject: [PATCH 05/13] feat(access): seed the day-one role baseline An empty grant table denies everyone, which is correct and useless: /access/decision said no to the people who administer the system. This writes down the access the Authentik roles already had, so nobody's access changes on the day the visibility layer starts being consulted (ADR5, 5.2). 68 grants: seven roles x their capabilities x four data types, all globally scoped. The mapping mirrors core/dependencies.py -- Viewer reads, Editor also enters and corrects, Admin also administers, the AMP family the same, and OGCInternal reads because the desktop-GIS mount has never written. Lexicon* is not seeded: it gates vocabulary, not data. AMP.Staging is not seeded because it gates a workbench that ships dark, and granting access to something still being validated is the one outcome nobody intended. The seeder will not resurrect a grant somebody revoked. Its skip check counts revoked rows as already-seeded, because narrowing this baseline is the whole point of writing it down, and a seeder that quietly undid a deliberate revocation would be worse than no seeder. Every row it writes carries granted_by = system:day-one-baseline and a reason naming the ADR, so a grant that exists because of institutional history is distinguishable from one somebody weighed. Each is audited as grant.created. Exposed as `oco seed-access-grants`, which previews by default and needs --apply to write. Grants are security state; creating them as a side effect of a process start would be the kind of thing nobody remembers is happening. It has not been run in any environment, so dev, staging and production still deny everyone until somebody runs it there. Co-Authored-By: Claude Opus 5 --- ADR5.md | 9 +- CLAUDE.md | 6 ++ cli/cli.py | 38 +++++++ services/access_seed.py | 178 +++++++++++++++++++++++++++++++ tests/test_access_seed.py | 217 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 446 insertions(+), 2 deletions(-) create mode 100644 services/access_seed.py create mode 100644 tests/test_access_seed.py diff --git a/ADR5.md b/ADR5.md index 5ec810bc..454b9133 100644 --- a/ADR5.md +++ b/ADR5.md @@ -216,8 +216,13 @@ told. holds the per-audience allowlists and the never-public list, and coordinate rounding is the first transform. The OGC views still select their own columns in SQL and do not pass through it. -5. Console administration. -6. Healy migration, after the data owner decides grandfathering. +5. **Per environment, by hand:** `oco seed-access-grants` writes the day-one + role baseline (ADR5, 5.2) -- one global grant per Authentik role, + capability and data type, so nobody's access changes when the layer starts + being consulted. Until it runs, that environment denies everyone. + Idempotent, and it does not resurrect a revoked seeded grant. +6. Console administration. +7. Healy migration, after the data owner decides grandfathering. ## References diff --git a/CLAUDE.md b/CLAUDE.md index b0c89530..976793b6 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -241,6 +241,12 @@ The storage and the evaluator exist; the field projection does not. transformation -- public coordinates are rounded, not withheld. The OGC views do **not** pass through this yet. Read **`docs/access-field-projection.md`** before touching the allowlists. +- **The role baseline is seeded by hand, per environment.** + `oco seed-access-grants` writes one global grant per (Authentik role, + capability, data type) so today's roles keep today's access; it previews by + default and needs `--apply` to write. Idempotent, and it will not resurrect + a seeded grant somebody revoked, because narrowing the baseline is the point. + Until it is run in an environment, `/access/decision` denies everyone there. - **Default deny, no wildcards, expiry at use.** A grant with no matching row is a no; a grant names its `data_type` (there is no term meaning "all"); and nothing sweeps expired rows, so every check compares against the date asked diff --git a/cli/cli.py b/cli/cli.py index 7a7332e9..80651d78 100644 --- a/cli/cli.py +++ b/cli/cli.py @@ -1431,6 +1431,44 @@ def import_project_area_boundaries_command( ) +@cli.command("seed-access-grants") +def seed_access_grants( + apply: bool = typer.Option( + False, + "--apply", + help="Write the grants. Without this the command only shows the plan.", + ), + verbose: bool = typer.Option( + False, "--verbose", help="List every grant, not just the counts." + ), +): + """Give the Authentik roles the access they already had (ADR5, 5.2). + + An empty grant table means default deny, which is correct and useless: the + visibility layer says no to everyone until the roles that exist today are + written down as role principals. + + Previews by default. Idempotent, and it will not resurrect a seeded grant + somebody revoked. + """ + from db.engine import session_ctx + from services.access_seed import seed_role_grants + + with session_ctx() as session: + plan = seed_role_grants(session, apply=apply) + + verb = "Created" if apply else "Would create" + typer.echo(f"{verb} {len(plan.created)} grant(s).") + typer.echo(f"Left {len(plan.skipped)} existing grant(s) alone.") + + if verbose: + for entry in plan.created: + typer.echo(f" + {plan.describe(entry)}") + + if plan.created and not apply: + typer.echo("Nothing was written. Re-run with --apply.") + + if __name__ == "__main__": cli() diff --git a/services/access_seed.py b/services/access_seed.py new file mode 100644 index 00000000..fdf4e3b0 --- /dev/null +++ b/services/access_seed.py @@ -0,0 +1,178 @@ +# =============================================================================== +# Copyright 2025 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +services/access_seed.py + +The day-one baseline from ADR5, 5.2: existing Authentik roles become role +principals holding broad grants, so nobody's access changes on the day the +grant table starts being consulted. + +Without this the grant table is empty, and an empty grant table means default +deny -- correct, and useless: ``/access/decision`` says no to everyone, +including the people who administer the system. + +Three properties this seeder has to have: + +* **Idempotent.** Running it twice creates nothing the second time. +* **Not a resurrection.** A seeded grant somebody revoked stays revoked. The + skip check looks at every seeded row, not only the live ones, so re-running + after a deliberate revocation does not quietly undo it. +* **Marked.** Every row it writes carries ``granted_by`` set to the seed actor + and a reason naming the ADR, so a grant that exists because of institutional + history is distinguishable from one somebody decided on. + +The mapping mirrors the role families in ``core/dependencies.py``. It is a +starting point, not a statement about what each role should have: narrowing it +is the point of the whole exercise, and every narrowing is a revocation +somebody makes deliberately. +""" + +from dataclasses import dataclass, field +from datetime import date + +from sqlalchemy import select + +from core.enums import AccessDataType +from db.permission_grant import PermissionGrant +from domain.access import ( + CAPABILITY_ADMINISTER, + CAPABILITY_CORRECT, + CAPABILITY_ENTER, + CAPABILITY_READ, + PRINCIPAL_ROLE, + SCOPE_GLOBAL, +) +from services.access_admin import create_grant + +# Recorded as the granting actor. Not a person, and deliberately obvious in +# the audit log: these grants exist because of what access already was, not +# because someone weighed them. +SEED_ACTOR = "system:day-one-baseline" +SEED_REASON = ( + "Day-one baseline (ADR5, 5.2): preserves the access this Authentik role " + "already had before grants were consulted. Narrow it deliberately." +) + +READ_ONLY = (CAPABILITY_READ,) +EDIT = (CAPABILITY_READ, CAPABILITY_ENTER, CAPABILITY_CORRECT) +FULL = EDIT + (CAPABILITY_ADMINISTER,) + +# Authentik group -> capabilities, mirroring core/dependencies.py. The tiers +# nest within a family, so an Admin's row set is a superset of an Editor's. +# +# Lexicon* is absent: it gates vocabulary, not data. AMP.Staging is absent +# because it gates a workbench that ships dark, and seeding it would be the +# one thing nobody intended -- granting access to something still being +# validated. +ROLE_BASELINE = { + "Viewer": READ_ONLY, + "Editor": EDIT, + "Admin": FULL, + "AMPViewer": READ_ONLY, + "AMPEditor": EDIT, + "AMPAdmin": FULL, + # The desktop-GIS mount reads; it has never written. + "OGCInternal": READ_ONLY, +} + + +@dataclass +class SeedPlan: + """What seeding would do, or did.""" + + created: list = field(default_factory=list) + skipped: list = field(default_factory=list) + + def describe(self, entry) -> str: + role, capability, data_type = entry + return f"role:{role} may {capability} {data_type} (global)" + + +def data_types() -> tuple: + """Every access data type there is, named one by one. + + No wildcard: this is a list of rows, so a data type added later is not + covered until somebody seeds or grants it. + """ + return tuple(member.value for member in AccessDataType) + + +def planned_entries() -> list: + return [ + (role, capability, data_type) + for role, capabilities in ROLE_BASELINE.items() + for capability in capabilities + for data_type in data_types() + ] + + +def _already_seeded(session) -> set: + """Every (role, capability, data type) this seeder has ever written. + + Revoked rows count. A grant somebody took away is not re-created by + running the seeder again. + """ + rows = session.execute( + select( + PermissionGrant.principal_id, + PermissionGrant.capability, + PermissionGrant.data_type, + ).where( + PermissionGrant.principal_type == PRINCIPAL_ROLE, + PermissionGrant.granted_by == SEED_ACTOR, + ) + ).all() + return {(role, capability, data_type) for role, capability, data_type in rows} + + +def seed_role_grants(session, starts_at: date = None, apply: bool = True) -> SeedPlan: + """Create the missing baseline grants. Safe to run repeatedly. + + With ``apply=False`` nothing is written and the plan describes what would + be. Grants are security state, so the CLI previews by default. + """ + starts_at = starts_at or date.today() + seeded = _already_seeded(session) + + plan = SeedPlan() + for entry in planned_entries(): + if entry in seeded: + plan.skipped.append(entry) + continue + + plan.created.append(entry) + if not apply: + continue + + role, capability, data_type = entry + create_grant( + session, + SEED_ACTOR, + principal_type=PRINCIPAL_ROLE, + principal_id=role, + capability=capability, + scope_type=SCOPE_GLOBAL, + scope_id=None, + data_type=data_type, + starts_at=starts_at, + ends_at=None, + reason=SEED_REASON, + ) + + return plan + + +# ============= EOF ============================================= diff --git a/tests/test_access_seed.py b/tests/test_access_seed.py new file mode 100644 index 00000000..f88037a8 --- /dev/null +++ b/tests/test_access_seed.py @@ -0,0 +1,217 @@ +# =============================================================================== +# Copyright 2025 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""The day-one role baseline (ADR5, 5.2). + +An empty grant table denies everyone, including the people who run the system. +These cover the seeder that writes today's access down, and the property that +matters more than convenience: it never gives back what someone took away. +""" + +from datetime import date, datetime, timezone + +import pytest +from sqlalchemy import delete, select + +from core.dependencies import admin_function, viewer_function +from db.authorization_audit import AuthorizationAudit +from db.engine import session_ctx +from db.permission_grant import PermissionGrant +from main import app +from services.access_seed import ( + ROLE_BASELINE, + SEED_ACTOR, + planned_entries, + seed_role_grants, +) +from tests import client, override_authentication + +VIEWER_PAYLOAD = {"sub": "test-viewer", "groups": ["Viewer"]} + + +@pytest.fixture(autouse=True) +def seeded_grants(): + app.dependency_overrides[admin_function] = override_authentication( + default=VIEWER_PAYLOAD + ) + app.dependency_overrides[viewer_function] = override_authentication( + default=VIEWER_PAYLOAD + ) + + yield + + app.dependency_overrides = {} + with session_ctx() as session: + session.execute( + delete(PermissionGrant).where(PermissionGrant.granted_by == SEED_ACTOR) + ) + session.execute( + delete(AuthorizationAudit).where(AuthorizationAudit.actor == SEED_ACTOR) + ) + session.commit() + + +def seeded_rows(session): + return ( + session.execute( + select(PermissionGrant).where(PermissionGrant.granted_by == SEED_ACTOR) + ) + .scalars() + .all() + ) + + +# ------ the seeder ---------- + + +def test_seeding_writes_one_row_per_role_capability_and_data_type(): + with session_ctx() as session: + plan = seed_role_grants(session) + assert len(plan.created) == len(planned_entries()) + assert len(seeded_rows(session)) == len(planned_entries()) + + +def test_seeding_twice_creates_nothing_the_second_time(): + with session_ctx() as session: + seed_role_grants(session) + again = seed_role_grants(session) + + assert again.created == [] + assert len(again.skipped) == len(planned_entries()) + + +def test_a_preview_writes_nothing(): + with session_ctx() as session: + plan = seed_role_grants(session, apply=False) + assert plan.created + assert seeded_rows(session) == [] + + +def test_a_revoked_baseline_grant_is_not_resurrected(): + """Narrowing the baseline is the point; re-running must not undo it.""" + with session_ctx() as session: + seed_role_grants(session) + grant = ( + session.execute( + select(PermissionGrant).where( + PermissionGrant.granted_by == SEED_ACTOR, + PermissionGrant.principal_id == "Viewer", + PermissionGrant.data_type == "water chemistry", + ) + ) + .scalars() + .one() + ) + grant.revoked_at = datetime.now(timezone.utc) + grant.revoked_by = "someone-who-decided" + session.commit() + + plan = seed_role_grants(session) + + assert plan.created == [] + + +def test_every_seeded_grant_is_marked_as_history_not_judgement(): + with session_ctx() as session: + seed_role_grants(session) + rows = seeded_rows(session) + + assert all(row.granted_by == SEED_ACTOR for row in rows) + assert all("ADR5" in row.reason for row in rows) + assert all(row.scope_type == "global" and row.scope_id is None for row in rows) + + +def test_seeding_is_audited(): + with session_ctx() as session: + seed_role_grants(session) + events = ( + session.execute( + select(AuthorizationAudit).where(AuthorizationAudit.actor == SEED_ACTOR) + ) + .scalars() + .all() + ) + + assert len(events) == len(planned_entries()) + assert {event.event_type for event in events} == {"grant.created"} + + +def test_a_viewer_gets_read_and_nothing_else(): + """The tiers nest within a family; the baseline has to say so.""" + assert ROLE_BASELINE["Viewer"] == ("read",) + assert set(ROLE_BASELINE["Admin"]) > set(ROLE_BASELINE["Editor"]) + assert set(ROLE_BASELINE["Editor"]) > set(ROLE_BASELINE["Viewer"]) + + +def test_the_dark_workbench_group_is_not_seeded(): + """AMP.Staging gates a workbench still being validated. Seeding it would + grant exactly the access nobody intended.""" + assert "AMP.Staging" not in ROLE_BASELINE + assert not any(role.startswith("Lexicon") for role in ROLE_BASELINE) + + +# ------ what it is for ---------- + + +def test_decision_says_no_before_seeding(): + response = client.get( + "/access/decision", params={"capability": "read", "data_type": "water level"} + ) + assert response.json()["allowed"] is False + + +def test_decision_says_yes_to_a_role_holder_after_seeding(): + with session_ctx() as session: + seed_role_grants(session) + + allowed = client.get( + "/access/decision", params={"capability": "read", "data_type": "water level"} + ).json() + assert allowed["allowed"] is True + assert "role:Viewer" in allowed["principals"] + + +def test_a_viewer_still_cannot_correct_after_seeding(): + with session_ctx() as session: + seed_role_grants(session) + + response = client.get( + "/access/decision", + params={"capability": "correct", "data_type": "water level"}, + ) + assert response.json()["allowed"] is False + + +def test_the_baseline_is_global_so_it_reaches_any_thing(): + with session_ctx() as session: + seed_role_grants(session) + + response = client.get( + "/access/decision", + params={ + "capability": "read", + "data_type": "water level", + "thing_id": 999999, + }, + ) + assert response.json()["allowed"] is True + + +def test_seeded_grants_start_today_not_retroactively(): + with session_ctx() as session: + seed_role_grants(session) + rows = seeded_rows(session) + + assert all(row.starts_at == date.today() for row in rows) From 984d20c234990b9876a09f5262ae84403db2cb5e Mon Sep 17 00:00:00 2001 From: jakeross Date: Wed, 26 Aug 2026 01:18:06 -0700 Subject: [PATCH 06/13] docs(claude): add working guidelines for coding changes Four rules on how to approach changes in this repo: state assumptions before implementing, keep the solution minimal, keep diffs surgical, and turn tasks into verifiable goals. Co-Authored-By: Claude Opus 5 --- CLAUDE.md | 67 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) diff --git a/CLAUDE.md b/CLAUDE.md index 976793b6..55cd62bc 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -392,3 +392,70 @@ transferers directly. - **OGC API**: `http://localhost:8000/ogcapi` for OGC API - Features endpoints - **CLI**: `oco --help` for Ocotillo CLI commands - **Sentry**: Error tracking and performance monitoring integrated + +## Working Guidelines + +Behavioral guidelines to reduce common LLM coding mistakes. These bias toward +caution over speed; for trivial tasks, use judgment. + +### 1. Think Before Coding + +**Don't assume. Don't hide confusion. Surface tradeoffs.** + +Before implementing: +- State your assumptions explicitly. If uncertain, ask. +- If multiple interpretations exist, present them - don't pick silently. +- If a simpler approach exists, say so. Push back when warranted. +- If something is unclear, stop. Name what's confusing. Ask. + +### 2. Simplicity First + +**Minimum code that solves the problem. Nothing speculative.** + +- No features beyond what was asked. +- No abstractions for single-use code. +- No "flexibility" or "configurability" that wasn't requested. +- No error handling for impossible scenarios. +- If you write 200 lines and it could be 50, rewrite it. + +Ask yourself: "Would a senior engineer say this is overcomplicated?" If yes, simplify. + +### 3. Surgical Changes + +**Touch only what you must. Clean up only your own mess.** + +When editing existing code: +- Don't "improve" adjacent code, comments, or formatting. +- Don't refactor things that aren't broken. +- Match existing style, even if you'd do it differently. +- If you notice unrelated dead code, mention it - don't delete it. + +When your changes create orphans: +- Remove imports/variables/functions that YOUR changes made unused. +- Don't remove pre-existing dead code unless asked. + +The test: Every changed line should trace directly to the user's request. + +### 4. Goal-Driven Execution + +**Define success criteria. Loop until verified.** + +Transform tasks into verifiable goals: +- "Add validation" -> "Write tests for invalid inputs, then make them pass" +- "Fix the bug" -> "Write a test that reproduces it, then make it pass" +- "Refactor X" -> "Ensure tests pass before and after" + +For multi-step tasks, state a brief plan: + +``` +1. [Step] -> verify: [check] +2. [Step] -> verify: [check] +3. [Step] -> verify: [check] +``` + +Strong success criteria let you loop independently. Weak criteria ("make it +work") require constant clarification. + +**These guidelines are working if:** fewer unnecessary changes in diffs, fewer +rewrites due to overcomplication, and clarifying questions come before +implementation rather than after mistakes. From a2d9ef699e5bfcfd2641d0467cec84fd4e3100ca Mon Sep 17 00:00:00 2001 From: jakeross Date: Wed, 26 Aug 2026 01:23:10 -0700 Subject: [PATCH 07/13] feat(access): put the public OGC collections under the field projection The projection chokepoint only covered payloads built by services/visibility.py; the OGC views selected their own columns in SQL. This closes that gap for all 27 public collections. DescribedPostgreSQLProvider now looks its table up in the new ogc: block of core/field-allowlists.yml and hands the result to pygeoapi as the provider's `properties`, which is what _select_properties builds the SELECT from. An unlisted column is never read out of Postgres, so it cannot surface in a feature, in /schema, or in /queryables, and a filter cannot probe for it. get_fields is narrowed to match. A collection with no entry publishes nothing, and tests/test_ogc_projection.py fails on a missing entry so it shows up in CI rather than in production. The internal mount (ogc_internal_*) is not projected. The lists were generated from what each view published on 2026-08-24 minus the never-public fields, so no consumer lost a field it was using. Two leaks did close: eleven collections were publishing nma_pk_welldata and ogc_temp_depth_measurements was publishing entered_by, a staff member's name. Also fixes PRINCIPAL_API_KEY, which said "api_key" where the lexicon says "api key". The route validated against the constant, so every API-key grant was rejected with a 422. A new test pins the domain vocabularies to the lexicon enums so they cannot drift again. Drops domain.access.published_thing_ids, which had no callers. Co-Authored-By: Claude Opus 5 --- ADR5.md | 14 +- CLAUDE.md | 7 +- core/feature_provider.py | 64 ++- core/field-allowlists.yml | 725 ++++++++++++++++++++++++++++++++ docs/access-field-projection.md | 44 +- domain/access.py | 20 +- services/field_projection.py | 67 ++- tests/test_domain_access.py | 22 + tests/test_ogc_projection.py | 179 ++++++++ 9 files changed, 1102 insertions(+), 40 deletions(-) create mode 100644 tests/test_ogc_projection.py diff --git a/ADR5.md b/ADR5.md index 454b9133..53639916 100644 --- a/ADR5.md +++ b/ADR5.md @@ -211,11 +211,15 @@ told. `authorization_audit` exist; `services/visibility.py` is the single evaluator; `api/access.py` is its one tenant. No existing endpoint routes through it yet. -4. **Done for the payloads the visibility layer builds.** - `services/field_projection.py` is the chokepoint, `core/field-allowlists.yml` - holds the per-audience allowlists and the never-public list, and coordinate - rounding is the first transform. The OGC views still select their own - columns in SQL and do not pass through it. +4. **Done.** `services/field_projection.py` is the chokepoint, + `core/field-allowlists.yml` holds the per-audience allowlists and the + never-public list, and coordinate rounding is the first transform. Both + publication paths go through it: destination payloads via + `services/visibility.py`, and all 27 public OGC collections via + `core/feature_provider.py`, which turns the allowlist into the provider's + `properties` so unlisted columns are never selected. Closing that gap + found eleven collections publishing `nma_pk_welldata` and one publishing + `entered_by`. 5. **Per environment, by hand:** `oco seed-access-grants` writes the day-one role baseline (ADR5, 5.2) -- one global grant per Authentik role, capability and data type, so nobody's access changes when the layer starts diff --git a/CLAUDE.md b/CLAUDE.md index 55cd62bc..7a65b1fd 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -238,8 +238,11 @@ The storage and the evaluator exist; the field projection does not. `services/field_projection.py` applies it below the routes, so a new route cannot skip it. An audience with no entry gets an empty record, and the `never_public` block overrides every allowlist. Protection includes - transformation -- public coordinates are rounded, not withheld. The OGC - views do **not** pass through this yet. Read + transformation -- public coordinates are rounded, not withheld. The public + OGC collections go through it too: `core/feature_provider.py` turns each + collection's allowlist into pygeoapi's `properties`, so an unlisted column + is never selected, and a collection with no entry publishes nothing. The + internal mount is outside it. Read **`docs/access-field-projection.md`** before touching the allowlists. - **The role baseline is seeded by hand, per environment.** `oco seed-access-grants` writes one global grant per (Authentik role, diff --git a/core/feature_provider.py b/core/feature_provider.py index e13751a3..9cfd08d2 100644 --- a/core/feature_provider.py +++ b/core/feature_provider.py @@ -13,15 +13,32 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== -"""Feature provider that publishes field-level prose alongside the columns. +"""Feature provider that projects the published columns and documents them. -pygeoapi's PostgreSQL provider reflects a table and reports each column's -JSON Schema type and format. It does not read column comments, and there is -no hook for documentation, so /collections/{id}/schema publishes bare column -names. This subclass annotates the reflected fields from -core/ogc-field-descriptions.yml on the way out. +Two jobs, both on the way out of pygeoapi's PostgreSQL provider. -Read docs/ogc-field-descriptions.md before changing this. +**Projection (ADR5, A.2).** Every public collection publishes the columns +named for it in ``core/field-allowlists.yml`` and no others. The allowlist is +handed to pygeoapi as the provider's ``properties``, which is what +``_select_properties`` builds the SELECT from, so an unlisted column is never +read out of Postgres -- it cannot appear in a feature, in ``/schema``, or in +``/queryables``, and a filter cannot probe it. ``get_fields`` is narrowed to +match, because the reflection sees the whole view. + +A public collection with no allowlist entry publishes no properties. That is +default deny, and ``tests/test_ogc_projection.py`` fails when an entry is +missing so it surfaces in CI rather than in production. The internal mount +(``ogc_internal_*``) is not projected: it serves authenticated Bureau staff, +and per-role internal field rules are the part of ADR5 3.5 that is not built. + +**Documentation.** pygeoapi reflects a table and reports each column's JSON +Schema type and format. It does not read column comments, and there is no hook +for documentation, so ``/collections/{id}/schema`` publishes bare column names. +This subclass annotates the reflected fields from +``core/ogc-field-descriptions.yml``. + +Read docs/ogc-field-descriptions.md and docs/access-field-projection.md before +changing this. """ import logging @@ -29,12 +46,39 @@ from pygeoapi.provider.sql import PostgreSQLProvider from core.ogc_field_metadata import describe_fields +from services.field_projection import ogc_allowlist LOGGER = logging.getLogger(__name__) class DescribedPostgreSQLProvider(PostgreSQLProvider): - """PostgreSQLProvider that annotates reflected columns with prose.""" + """PostgreSQLProvider that projects published columns and annotates them.""" + + def __init__(self, provider_def): + table = provider_def.get("table") + self._allowlist = ogc_allowlist(table) + + if self._allowlist is not None: + # pygeoapi reads `properties` in _select_properties and in get(), + # so setting it here is what actually keeps the column out of the + # query rather than out of the response only. + provider_def = dict(provider_def, properties=sorted(self._allowlist)) + if not self._allowlist: + LOGGER.warning( + "%s has no entry in core/field-allowlists.yml, so it " + "publishes no properties. Add one deliberately.", + table, + ) + + super().__init__(provider_def) + + def _published(self, fields: dict) -> dict: + """Drop reflected fields the allowlist does not name.""" + if self._allowlist is None: + return fields + return { + name: field for name, field in fields.items() if name in self._allowlist + } def get_fields(self): """Reflect the table, then annotate the result. @@ -49,7 +93,9 @@ def get_fields(self): """ fields = super().get_fields() if fields and not getattr(self, "_fields_described", False): - self._fields = describe_fields(self.table, fields) + # Narrow before describing: a column nobody publishes has no + # business in /schema or /queryables, described or not. + self._fields = describe_fields(self.table, self._published(fields)) # super().get_fields() short-circuits on a populated _fields, so # without this flag a later call would re-describe the annotated # dict. Harmless today (describe_fields is idempotent) but it diff --git a/core/field-allowlists.yml b/core/field-allowlists.yml index acd1505b..537e14f2 100644 --- a/core/field-allowlists.yml +++ b/core/field-allowlists.yml @@ -126,3 +126,728 @@ audiences: # for that destination's kind rather than adding to them, so what one partner # receives is readable in one place. by_slug: {} + +# --------------------------------------------------------------------------- +# The OGC collections. +# +# The public mount serves 27 collections: fifteen declared in +# core/pygeoapi-config.yml, ten built from THING_COLLECTIONS in +# core/pygeoapi.py, and two EDR collections wired there as well. All of them +# are served by +# core.feature_provider.DescribedPostgreSQLProvider, which reads the allowlist +# below and hands it to pygeoapi as the provider's `properties`. A column that +# is not listed is never selected from Postgres, so it cannot appear in a +# feature, in /schema, or in /queryables. +# +# These lists were generated from what each view published on 2026-08-24, minus +# the never-public fields, so nothing anyone consumes today disappeared -- with +# two exceptions, which is the point: eleven collections were publishing +# nma_pk_welldata and one was publishing entered_by. That +# makes this file the record of what is public, not an aspiration: narrowing a +# list is a deliberate, reviewed edit, and it should happen. +# +# A collection with no entry here publishes no properties at all. That is +# default deny working as intended, and `tests/test_ogc_projection.py` fails +# when a public collection is missing, so it surfaces in CI rather than in +# production. +# +# The internal mount (ogc_internal_*) is not projected. It serves authenticated +# Bureau staff, and per-role internal field rules are the part of ADR5 3.5 that +# is not built yet. + +ogc: + # Fields no public collection may publish, whatever its list says. Same + # policy status as `never_public` above: adding is safe, removing is not an + # engineering decision. + never_public: + # Legacy AMPAPI primary keys. ogc_well_water_column published + # nma_pk_welldata until this list existed. + - nma_pk_welldata + - nma_pk_location + # Who typed the record. ogc_temp_depth_measurements published entered_by, + # a staff member's name, inherited from the NM_Wells transfer. + - created_by_id + - created_by_name + - updated_by_id + - updated_by_name + - entered_by + # Free text written by staff for staff -- gate codes, lock combinations, + # candid landowner notes. Not public today; listed so it cannot become so. + - nma_location_notes + - nma_coordinate_notes + - search_vector + + collections: + ogc_actively_monitored_wells: + - id + - name + - thing_type + - well_depth + - elevation + - elevation_method + - formation_zone + - total_water_levels + - last_water_level + - last_water_level_datetime + - min_water_level + - max_water_level + - water_level_trend_ft_per_year + - group_ids + - group_names + - group_types + + ogc_bht_measurements: + - id + - api + - well_name + - well_num + - operator + - well_type + - well_tvd + - completion_date + - current_status + - total_depth + - cuttings + - core_exists + - county + - bht_depth + - bht + - hours_since_circulation + - date_measured + + ogc_depth_to_water_trend_wells: + - id + - name + - thing_type + - record_count + - first_observation_datetime + - last_observation_datetime + - span_years + - slope_ft_per_year + - trend_category + + ogc_diversions_surface_water: + - id + - name + - first_visit_date + - last_observation_date + - well_depth + - hole_depth + - well_casing_diameter + - well_casing_depth + - well_completion_date + - well_driller_name + - well_construction_method + - well_pump_type + - well_pump_depth + - formation_completion_code + - nma_formation_zone + - release_status + - elevation + + ogc_dst: + - id + - well_name + - well_num + - api + - dst_name + - dst_operator + - dst_number + - dst_date + - county + - state + - lat_dd83 + - long_dd83 + - from_depth + - to_depth + - target_fm + - packer_from + - packer_to + - srf_choke_sz + - bot_choke_sz + - depth_unit + - elev_gl + - elev_unspc + - prs_gage_dpt + - pipe_dia + - pipe_length + - flow_history + - init_flow + - flw_prs_in_min + - fin_flow + - flw_prs_fin_min + - prs_init_clsd_in + - in_sht_in_min + - fin_shut_in + - fn_sht_in_min + - hydrost_prs_in + - hyd_st_prs_fl + - press_units + - blanked_off + - fm_temp + + ogc_ephemeral_streams: + - id + - name + - first_visit_date + - last_observation_date + - well_depth + - hole_depth + - well_casing_diameter + - well_casing_depth + - well_completion_date + - well_driller_name + - well_construction_method + - well_pump_type + - well_pump_depth + - formation_completion_code + - nma_formation_zone + - release_status + - elevation + + ogc_geothermal_wells_bht: + - id + - well_data_id + - well_name + - api + - total_depth + - bht_count + - max_bht + - min_bht + - max_bht_c + - min_bht_c + - max_bht_depth + - temp_unit + - temp_unit_source + - temp_unit_mixed + - unconvertible_count + + ogc_geothermal_wells_temperature_profile: + - id + - well_data_id + - well_name + - api + - reading_count + - min_depth + - max_depth + - min_temp + - max_temp + - min_temp_c + - max_temp_c + - temp_unit + - temp_unit_source + - temp_unit_mixed + - unconvertible_count + - series + + ogc_heat_flow: + - id + - well_name + - well_num + - api + - county + - state + - lat_dd27 + - long_dd27 + - lat_dd83 + - long_dd83 + - source_id + - elev_gl + - elev_kb + - elev_unspc + - elevation_m + - depth_units + - total_depth + - total_depth_m + - from_depth + - to_depth + - therml_cond + - tcond_range + - tcond_error + - tcond_unit + - tc_si + - sample_type + - num_samples + - therml_grad + - tgrad_range + - tg_error + - grad_unit + - heat_flow + - ht_flow_unit + - heat_flow_si + - quality + - first_auth + - pub_year + - title + - journal + - volume + - page_no + - ht_flow_est + - entry_date + - ht_flow_est_si + + ogc_lakes_ponds_reservoirs: + - id + - name + - first_visit_date + - last_observation_date + - well_depth + - hole_depth + - well_casing_diameter + - well_casing_depth + - well_completion_date + - well_driller_name + - well_construction_method + - well_pump_type + - well_pump_depth + - formation_completion_code + - nma_formation_zone + - release_status + - elevation + + ogc_latest_tds_wells: + - id + - name + - thing_type + - major_chemistry_id + - latest_tds_observation_date + - latest_tds_value + - latest_tds_units + + ogc_major_chemistry_results: + - id + - location_id + - name + - thing_type + - analyte_count + - latest_chemistry_date + - tds + - calcium + - calcium_total + - magnesium + - magnesium_total + - sodium + - sodium_total + - potassium + - potassium_total + - sodium_plus_potassium + - bicarbonate + - carbonate + - sulfate + - chloride + - ion_balance + - total_anions + - total_cations + - alkalinity + - hardness + - specific_conductance + - ph + - nitrate + - fluoride + - silica + - tds_units + - calcium_units + - calcium_total_units + - magnesium_units + - magnesium_total_units + - sodium_units + - sodium_total_units + - potassium_units + - potassium_total_units + - sodium_plus_potassium_units + - bicarbonate_units + - carbonate_units + - sulfate_units + - chloride_units + - ion_balance_units + - total_anions_units + - total_cations_units + - alkalinity_units + - hardness_units + - specific_conductance_units + - ph_units + - nitrate_units + - fluoride_units + - silica_units + + ogc_meteorological_stations: + - id + - name + - first_visit_date + - last_observation_date + - well_depth + - hole_depth + - well_casing_diameter + - well_casing_depth + - well_completion_date + - well_driller_name + - well_construction_method + - well_pump_type + - well_pump_depth + - formation_completion_code + - nma_formation_zone + - release_status + - elevation + + ogc_minor_chemistry_wells: + - id + - location_id + - name + - thing_type + - analyte_count + - latest_chemistry_date + - h2r + - o18r + - c13r + - c14 + - c14_years + - fluoride + - barium + - barium_total + - copper + - copper_total + - zinc + - zinc_total + - molybdenum + - molybdenum_total + - silica + - silicon + - silicon_total + - manganese + - manganese_total + - iron + - iron_total + - strontium + - strontium_total + - chromium + - chromium_total + - boron + - boron_total + - uranium + - uranium_total + - lithium + - lithium_total + - silver + - silver_total + - antimony + - antimony_total + - beryllium + - beryllium_total + - lead + - lead_total + - thallium + - thallium_total + - bromide + - selenium + - selenium_total + - vanadium + - vanadium_total + - aluminum + - aluminum_total + - arsenic + - arsenic_total + - nickel + - nickel_total + - cadmium + - cadmium_total + - cobalt + - cobalt_total + - phosphate + - nitrite + - nitrate + - nitrate_as_n + - thorium + - thorium_total + - tin + - tin_total + - mercury + - mercury_total + - titanium + - titanium_total + - h2r_units + - o18r_units + - c13r_units + - c14_units + - c14_years_units + - fluoride_units + - barium_units + - barium_total_units + - copper_units + - copper_total_units + - zinc_units + - zinc_total_units + - molybdenum_units + - molybdenum_total_units + - silica_units + - silicon_units + - silicon_total_units + - manganese_units + - manganese_total_units + - iron_units + - iron_total_units + - strontium_units + - strontium_total_units + - chromium_units + - chromium_total_units + - boron_units + - boron_total_units + - uranium_units + - uranium_total_units + - lithium_units + - lithium_total_units + - silver_units + - silver_total_units + - antimony_units + - antimony_total_units + - beryllium_units + - beryllium_total_units + - lead_units + - lead_total_units + - thallium_units + - thallium_total_units + - bromide_units + - selenium_units + - selenium_total_units + - vanadium_units + - vanadium_total_units + - aluminum_units + - aluminum_total_units + - arsenic_units + - arsenic_total_units + - nickel_units + - nickel_total_units + - cadmium_units + - cadmium_total_units + - cobalt_units + - cobalt_total_units + - phosphate_units + - nitrite_units + - nitrate_units + - nitrate_as_n_units + - thorium_units + - thorium_total_units + - tin_units + - tin_total_units + - mercury_units + - mercury_total_units + - titanium_units + - titanium_total_units + + ogc_outfalls_wastewater_return_flow: + - id + - name + - first_visit_date + - last_observation_date + - well_depth + - hole_depth + - well_casing_diameter + - well_casing_depth + - well_completion_date + - well_driller_name + - well_construction_method + - well_pump_type + - well_pump_depth + - formation_completion_code + - nma_formation_zone + - release_status + - elevation + + ogc_perennial_streams: + - id + - name + - first_visit_date + - last_observation_date + - well_depth + - hole_depth + - well_casing_diameter + - well_casing_depth + - well_completion_date + - well_driller_name + - well_construction_method + - well_pump_type + - well_pump_depth + - formation_completion_code + - nma_formation_zone + - release_status + - elevation + + ogc_project_areas: + - id + - name + - description + - group_type + - release_status + + ogc_rock_sample_locations: + - id + - name + - first_visit_date + - last_observation_date + - well_depth + - hole_depth + - well_casing_diameter + - well_casing_depth + - well_completion_date + - well_driller_name + - well_construction_method + - well_pump_type + - well_pump_depth + - formation_completion_code + - nma_formation_zone + - release_status + - elevation + + ogc_soil_gas_sample_locations: + - id + - name + - first_visit_date + - last_observation_date + - well_depth + - hole_depth + - well_casing_diameter + - well_casing_depth + - well_completion_date + - well_driller_name + - well_construction_method + - well_pump_type + - well_pump_depth + - formation_completion_code + - nma_formation_zone + - release_status + - elevation + + ogc_springs: + - id + - name + - first_visit_date + - last_observation_date + - well_depth + - hole_depth + - well_casing_diameter + - well_casing_depth + - well_completion_date + - well_driller_name + - well_construction_method + - well_pump_type + - well_pump_depth + - formation_completion_code + - nma_formation_zone + - release_status + - elevation + + ogc_temp_depth_measurements: + - id + - well_name + - well_num + - api + - source_id + - sample_fm + - county + - state + - lat_dd27 + - long_dd27 + - lat_dd83 + - long_dd83 + - loc_acc_val + - entry_date + - depth + - depth_unit + - temp + - temp_unit + - elev_gl + - elev_unspc + - elev_kb + - sample_date + + ogc_water_chemistry: + - id + - thing_id + - station_name + - thing_type + - longitude + - latitude + - datetime + - value + - unit + - parameter_name + - sample_id + - release_status + + ogc_water_elevation_wells: + - id + - name + - thing_type + - observation_id + - observation_datetime + - elevation_m + - depth_to_water_below_ground_surface_ft + - water_elevation_ft + + ogc_water_well_summary: + - id + - name + - well_depth + - elevation + - elevation_method + - formation_zone + - total_water_levels + - last_water_level + - last_water_level_datetime + - min_water_level + - max_water_level + - water_level_trend_ft_per_year + + ogc_water_wells: + - id + - name + - first_visit_date + - last_observation_date + - well_depth + - hole_depth + - well_casing_diameter + - well_casing_depth + - well_completion_date + - well_driller_name + - well_construction_method + - well_pump_type + - well_pump_depth + - formation_completion_code + - nma_formation_zone + - release_status + - elevation + + ogc_waterlevels: + - id + - thing_id + - station_name + - longitude + - latitude + - datetime + - value + - unit + - parameter_name + - source + - deployment_id + - release_status + + ogc_well_water_column: + - id + - name + - first_visit_date + - well_depth + - hole_depth + - well_casing_diameter + - well_casing_depth + - well_completion_date + - well_driller_name + - well_construction_method + - well_pump_type + - well_pump_depth + - formation_completion_code + - nma_formation_zone + - release_status + - water_column_latest + - water_column_average + - water_column_maximum + - water_column_minimum + - elevation diff --git a/docs/access-field-projection.md b/docs/access-field-projection.md index f3c3fea0..944ec5e0 100644 --- a/docs/access-field-projection.md +++ b/docs/access-field-projection.md @@ -26,7 +26,8 @@ Fields are published by **allowlist, per audience, at one chokepoint**. | Rules | `domain/field_projection.py` | Omit, transform, validate. Plain dicts; no database, no config parsing. | | Configuration + chokepoint | `services/field_projection.py` | Parse and validate the YAML, build a record from a model row, project it. | | Configuration | `core/field-allowlists.yml` | The allowlists and the never-public list. | -| Only current consumer | `services/visibility.py` (`published_things`) | Builds every published payload through `project_entity`. | +| Destination payloads | `services/visibility.py` (`published_things`) | Builds them through `project_entity`. | +| Public OGC collections | `core/feature_provider.py` | Reads `ogc_allowlist(table)` at provider construction. | `api/access.py` never projects anything itself. That is the point: the projection sits *below* the routes, so a new route or a new output format @@ -78,15 +79,42 @@ the permissions interview was explicit that gate codes, lock combinations and candid landowner notes must never leave the Bureau, and that is the text those columns have collected. -## What this does *not* cover yet +## The OGC collections -The OGC collections do **not** go through the projection. `ogc_*` views select -their own column lists in SQL and gate on `release_status`, exactly the -distributed filtering ADR5 argues against. Bringing them under the chokepoint -is later work, and until it happens the guarantees on this page apply only to -payloads built by `services/visibility.py`. +The public `/ogcapi` mount serves 27 collections from three places: fifteen +declared in `core/pygeoapi-config.yml`, ten built from `THING_COLLECTIONS` in +`core/pygeoapi.py`, and two EDR collections wired there too. Missing that third +source is how eleven collections published `nma_pk_welldata` unnoticed. -Only `thing` and `location` are projectable entities. Adding another means +`DescribedPostgreSQLProvider` looks each table up in the `ogc:` block of +`core/field-allowlists.yml` and hands the result to pygeoapi as the provider's +`properties`. That is what `_select_properties` builds the SELECT from, so an +unlisted column is never read out of Postgres at all: it cannot appear in a +feature, in `/schema`, or in `/queryables`, and a filter cannot probe for it. +`get_fields` is narrowed to match, since the reflection sees the whole view. + +**Adding a collection** means adding its table to `ogc.collections`. Without an +entry it publishes *no* properties — default deny, logged as a warning — and +`tests/test_ogc_projection.py` fails, so a missing entry shows up in CI rather +than in production. + +**The internal mount is not projected.** `ogc_internal_*` tables return `None` +from `ogc_allowlist` and pass through whole. It serves authenticated Bureau +staff, and per-role internal field rules are the part of ADR5 3.5 that is not +built. + +The lists were generated from what each view published on 2026-08-24 minus the +never-public fields, so no consumer lost a field it was using. That makes the +file a record of what is public rather than an aspiration — narrowing it is +the next deliberate step, and it should happen. + +## What this does *not* cover + +The desktop-GIS layer files in `core/gis-curated-layers.yml` are not a separate +publication path — they point clients at the OGC collections above and so +inherit the projection. + +Only `thing` and `location` are projectable entities for destination payloads. Adding another means adding it to `ENTITY_MODELS` (and `DERIVED_FIELDS` if the payload carries values that are not columns, the way `latitude` and `longitude` stand in for the PostGIS `point`). diff --git a/domain/access.py b/domain/access.py index 68397b75..9bdd3bde 100644 --- a/domain/access.py +++ b/domain/access.py @@ -61,7 +61,12 @@ # consent, not as a grant, which is the two-table half of ADR5. PRINCIPAL_USER = "user" PRINCIPAL_ROLE = "role" -PRINCIPAL_API_KEY = "api_key" +# Spelled as the lexicon spells it. The lexicon is the source of truth for +# every controlled term, and a constant that disagreed with it made this +# principal type unwritable: the route validated "api key" against "api_key" +# and rejected every API-key grant with a 422. tests/test_domain_access.py +# pins the two together so it cannot drift again. +PRINCIPAL_API_KEY = "api key" PRINCIPAL_TYPES = frozenset({PRINCIPAL_USER, PRINCIPAL_ROLE, PRINCIPAL_API_KEY}) @@ -275,17 +280,4 @@ def any_consent_publishes( ) -def published_thing_ids(consents, destination_id: int, data_type: str, on_date: date): - """Thing ids a destination may read for one data type, in first-seen order.""" - seen = [] - for consent in consents: - if consent.thing_id in seen: - continue - if consent_covers( - consent, consent.thing_id, destination_id, data_type, on_date - ): - seen.append(consent.thing_id) - return seen - - # ============= EOF ============================================= diff --git a/services/field_projection.py b/services/field_projection.py index 2222a103..358ee36c 100644 --- a/services/field_projection.py +++ b/services/field_projection.py @@ -23,6 +23,14 @@ a new route or a new output format cannot skip the rule by omission -- it sits below them. The rules themselves are in ``domain/field_projection.py``. +Two consumers, because there are two publication paths: + +* ``services/visibility.py`` builds destination payloads from ORM rows and + calls :func:`project_entity`. +* ``core/feature_provider.py`` serves the public OGC collections and calls + :func:`ogc_allowlist` at provider construction, so the columns nobody + approved are never selected from Postgres at all. + The whole file is validated the first time it is read: an unknown field name, a transform on a field nobody publishes, or an allowlist naming a never-public field raises. Failing at load rather than at request time is deliberate; the @@ -37,7 +45,12 @@ from db.location import Location from db.thing import Thing -from domain.field_projection import EntityProjection, project, validate_projection +from domain.field_projection import ( + EntityProjection, + NeverPublicFieldAllowed, + project, + validate_projection, +) CONFIG_PATH = Path(__file__).parent.parent / "core" / "field-allowlists.yml" @@ -80,7 +93,27 @@ def _configuration(path: str = None) -> dict: projection = _build_projection(entity, rule, never_public) audiences[(keyed_by, audience, entity)] = projection - return {"never_public": never_public, "audiences": audiences} + ogc = raw.get("ogc") or {} + ogc_never = frozenset(ogc.get("never_public") or ()) + ogc_collections = {} + for table, columns in (ogc.get("collections") or {}).items(): + columns = frozenset(columns or ()) + forbidden = sorted(columns & ogc_never) + if forbidden: + # The same rule as the audience allowlists: a never-public field + # cannot be re-admitted by naming it somewhere else. + raise NeverPublicFieldAllowed( + f"{table}.{forbidden[0]} is on the OGC never-public list and " + "cannot be published by a collection." + ) + ogc_collections[table] = columns + + return { + "never_public": never_public, + "audiences": audiences, + "ogc_never_public": ogc_never, + "ogc_collections": ogc_collections, + } def _build_projection(entity: str, rule, never_public: dict) -> EntityProjection: @@ -130,6 +163,36 @@ def never_public_fields(entity: str) -> frozenset: return _configuration()["never_public"].get(entity, frozenset()) +# ============= OGC collections ============================================= + + +def ogc_never_public() -> frozenset: + """Fields no public OGC collection may publish.""" + return _configuration()["ogc_never_public"] + + +def ogc_allowlist(table: str) -> frozenset | None: + """Columns this public collection publishes, or None if it is not one. + + None means "not projected" and covers the internal mount, whose tables are + ``ogc_internal_*``. An empty frozenset means "listed nowhere", which is + default deny: the collection publishes no properties until someone says + what it may publish. + """ + if table is None or table.startswith("ogc_internal_"): + return None + + collections = _configuration()["ogc_collections"] + if table not in collections: + return frozenset() + return collections[table] + + +def ogc_collection_tables() -> frozenset: + """Every public table with an allowlist entry.""" + return frozenset(_configuration()["ogc_collections"]) + + def thing_record(thing) -> dict: """Every stored field of a thing, before projection.""" return { diff --git a/tests/test_domain_access.py b/tests/test_domain_access.py index 35362b2c..0362794e 100644 --- a/tests/test_domain_access.py +++ b/tests/test_domain_access.py @@ -21,6 +21,9 @@ from domain.access import ( AccessRequest, + CAPABILITIES, + PRINCIPAL_TYPES, + SCOPE_TYPES, BackwardsDateRange, Consent, Grant, @@ -212,6 +215,25 @@ def test_validation_errors_are_value_errors(): validate_grant("user", "read", "global", None, None, TODAY, None) +# ------ the domain constants and the lexicon must agree ---------- + + +def test_the_domain_vocabularies_match_the_lexicon(): + """domain/access.py cannot import the lexicon without taking a database + dependency, so its constants are hand-copied. They drifted once: the + lexicon says `api key` and the constant said `api_key`, which made every + API-key grant fail validation with a 422. This is the guard.""" + from core.enums import Capability, GrantScopeType, PrincipalType + + assert {member.value for member in PrincipalType} == set(PRINCIPAL_TYPES) + assert {member.value for member in Capability} == set(CAPABILITIES) + assert {member.value for member in GrantScopeType} == set(SCOPE_TYPES) + + +def test_an_api_key_grant_can_actually_be_written(): + validate_grant("api key", "read", "global", None, "water level", TODAY, None) + + # ------ publication consent ---------- diff --git a/tests/test_ogc_projection.py b/tests/test_ogc_projection.py new file mode 100644 index 00000000..169451c7 --- /dev/null +++ b/tests/test_ogc_projection.py @@ -0,0 +1,179 @@ +# =============================================================================== +# Copyright 2025 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""The public OGC collections, under the projection chokepoint (ADR5, A.2). + +Two of these pin leaks that were live before the allowlist existed: +ogc_well_water_column published nma_pk_welldata, and +ogc_temp_depth_measurements published entered_by, a staff member's name. +""" + +import re +from pathlib import Path + +import pytest +from sqlalchemy import text + +from core.pygeoapi import THING_COLLECTIONS +from db.engine import session_ctx +from tests import client +from services.field_projection import ( + ogc_allowlist, + ogc_collection_tables, + ogc_never_public, +) + +PUBLIC_CONFIG = Path(__file__).parent.parent / "core" / "pygeoapi-config.yml" +# The EDR collections are wired in core/pygeoapi.py rather than the YAML. +EDR_TABLES = ("ogc_waterlevels", "ogc_water_chemistry") + +COLUMNS_QUERY = text( + """ + select a.attname + from pg_attribute a + join pg_class c on c.oid = a.attrelid + join pg_namespace n on n.oid = c.relnamespace + where n.nspname = 'public' + and c.relname = :relation + and a.attnum > 0 + and not a.attisdropped + """ +) + + +def public_tables() -> list: + """Every table the public mount serves. + + Three sources, and missing the third is how eleven collections published + nma_pk_welldata unnoticed: the YAML config, the thing collections built in + core/pygeoapi.py, and the EDR collections wired there too. + """ + configured = set(re.findall(r"table: (ogc_[a-z_]+)", PUBLIC_CONFIG.read_text())) + things = { + "ogc_" + collection["id"] + for collection in THING_COLLECTIONS + if not collection.get("internal_only") + } + return sorted(configured | things | set(EDR_TABLES)) + + +def live_columns(relation: str) -> set: + with session_ctx() as session: + return set(session.execute(COLUMNS_QUERY, {"relation": relation}).scalars()) + + +# ------ every public collection is projected ---------- + + +@pytest.mark.parametrize("table", public_tables()) +def test_every_public_collection_has_an_allowlist(table): + """A collection with no entry publishes nothing. Fail here, not in prod.""" + assert table in ogc_collection_tables(), ( + f"{table} is served publicly but has no entry in " + "core/field-allowlists.yml, so it would publish no properties." + ) + + +@pytest.mark.parametrize("table", public_tables()) +def test_an_allowlist_never_names_a_never_public_column(table): + assert not ogc_allowlist(table) & ogc_never_public() + + +@pytest.mark.parametrize("table", public_tables()) +def test_an_allowlist_only_names_columns_the_view_has(table): + """A stale name would make pygeoapi select a column that is not there.""" + assert ogc_allowlist(table) <= live_columns(table) + + +@pytest.mark.parametrize("table", public_tables()) +def test_no_never_public_column_survives_into_a_published_collection(table): + """The guard that matters: whatever the view grew, it is not published.""" + published = ogc_allowlist(table) + assert not (live_columns(table) & ogc_never_public() & published) + + +# ------ the leaks this closed ---------- + + +@pytest.mark.parametrize( + "table", + [ + "ogc_well_water_column", + "ogc_water_wells", + "ogc_springs", + "ogc_meteorological_stations", + ], +) +def test_the_thing_layers_no_longer_publish_the_legacy_key(table): + """Eleven public collections carried nma_pk_welldata before this.""" + assert "nma_pk_welldata" in live_columns(table) + assert "nma_pk_welldata" not in ogc_allowlist(table) + + +def test_the_geothermal_layer_no_longer_publishes_who_typed_the_record(): + assert "entered_by" in live_columns("ogc_temp_depth_measurements") + assert "entered_by" not in ogc_allowlist("ogc_temp_depth_measurements") + + +# ------ what is and is not projected ---------- + + +def test_the_internal_mount_is_not_projected(): + """Authenticated staff see the whole view; per-role rules are not built.""" + assert ogc_allowlist("ogc_internal_locations") is None + + +def test_an_unlisted_public_collection_publishes_nothing(): + """Default deny, rather than falling back to everything.""" + assert ogc_allowlist("ogc_collection_nobody_configured") == frozenset() + + +# ------ through the mounted service ---------- + + +def _json(path): + response = client.get(path, headers={"Accept": "application/json"}) + assert response.status_code == 200, response.text + return response.text + + +@pytest.mark.parametrize( + "collection,hidden,published", + [ + ("well_water_column", "nma_pk_welldata", "well_depth"), + ("temp_depth_measurements", "entered_by", "depth"), + ], +) +def test_the_mount_publishes_the_allowlist_and_nothing_else( + collection, hidden, published +): + """Not just absent from features: absent from the schema and the + queryables too, so a filter cannot probe for it either.""" + for path in ( + f"/ogcapi/collections/{collection}/schema", + f"/ogcapi/collections/{collection}/queryables", + f"/ogcapi/collections/{collection}/items?limit=1", + ): + body = _json(path) + assert hidden not in body, f"{hidden} leaked through {path}" + + # The collection still works; this is projection, not breakage. + assert published in _json(f"/ogcapi/collections/{collection}/schema") + + +def test_release_status_is_still_published(): + """The filter column stays visible; consumers read it to know what they + have, and hiding it would be a behavior change nobody asked for.""" + assert "release_status" in ogc_allowlist("ogc_waterlevels") From acbcd47d27baa0e121805472484754a8718eaba5 Mon Sep 17 00:00:00 2001 From: jakeross Date: Wed, 26 Aug 2026 01:24:19 -0700 Subject: [PATCH 08/13] chore: ignore the .design canvas scratch directory Design-canvas artboards, deck sources and their generated PDFs. Working material, not part of the API, and the PDFs are binary. Co-Authored-By: Claude Opus 5 --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index eb6f7c34..218ff919 100644 --- a/.gitignore +++ b/.gitignore @@ -56,3 +56,6 @@ app.yaml # Claude Code .claude/settings.local.json .claude/worktrees/ + +# Design canvas scratch (artboards, decks, generated PDFs) +.design/ From 09e62c0a814899ce0f2f0eadf21358b64f8600b6 Mon Sep 17 00:00:00 2001 From: jirhiker <2035568+jirhiker@users.noreply.github.com> Date: Wed, 26 Aug 2026 08:25:06 +0000 Subject: [PATCH 09/13] Formatting changes --- tests/test_ogc_projection.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/test_ogc_projection.py b/tests/test_ogc_projection.py index 169451c7..42ed0428 100644 --- a/tests/test_ogc_projection.py +++ b/tests/test_ogc_projection.py @@ -39,8 +39,7 @@ # The EDR collections are wired in core/pygeoapi.py rather than the YAML. EDR_TABLES = ("ogc_waterlevels", "ogc_water_chemistry") -COLUMNS_QUERY = text( - """ +COLUMNS_QUERY = text(""" select a.attname from pg_attribute a join pg_class c on c.oid = a.attrelid @@ -49,8 +48,7 @@ and c.relname = :relation and a.attnum > 0 and not a.attisdropped - """ -) + """) def public_tables() -> list: From f916fecdc1e940d83022afaa4042b873d5e9737b Mon Sep 17 00:00:00 2001 From: jakeross Date: Thu, 27 Aug 2026 17:10:09 -0700 Subject: [PATCH 10/13] feat(access): list all grants, not just one principal's GET /access/grant required principal_id; there was no way to browse every grant for an admin audit view. principal_id is now an optional filter, alongside new capability/data_type/scope_type filters -- the bare route lists everything. Co-Authored-By: Claude Sonnet 5 --- api/access.py | 26 +++++++++++++++++++++----- tests/test_access.py | 17 +++++++++++++++++ 2 files changed, 38 insertions(+), 5 deletions(-) diff --git a/api/access.py b/api/access.py index 3ab6fb22..000d7abe 100644 --- a/api/access.py +++ b/api/access.py @@ -166,16 +166,32 @@ def revoke_permission_grant( return PermissionGrantResponse.model_validate(grant) -@router.get("/grant", summary="List grants held by a principal") +@router.get("/grant", summary="List grants") def get_permission_grants( session: session_dependency, user: admin_dependency, - principal_id: str = Query(description="Authentik subject, role, or key label"), + principal_id: str = Query( + default=None, description="Authentik subject, role, or key label" + ), + capability: str = Query(default=None), + data_type: str = Query(default=None), + scope_type: str = Query(default=None), include_revoked: bool = Query(default=False), ) -> list[PermissionGrantResponse]: - statement = select(PermissionGrant).where( - PermissionGrant.principal_id == principal_id - ) + """All grants, or a narrower slice of them. + + Every filter is optional, so the bare route is the admin-wide audit view; + passing ``principal_id`` narrows it to one principal, as before. + """ + statement = select(PermissionGrant) + if principal_id is not None: + statement = statement.where(PermissionGrant.principal_id == principal_id) + if capability is not None: + statement = statement.where(PermissionGrant.capability == capability) + if data_type is not None: + statement = statement.where(PermissionGrant.data_type == data_type) + if scope_type is not None: + statement = statement.where(PermissionGrant.scope_type == scope_type) if not include_revoked: statement = statement.where(PermissionGrant.revoked_at.is_(None)) diff --git a/tests/test_access.py b/tests/test_access.py index b499c50f..367b1df7 100644 --- a/tests/test_access.py +++ b/tests/test_access.py @@ -384,6 +384,23 @@ def test_a_grant_naming_no_data_type_cannot_be_written(grants): assert make_grant(grants, data_type=None).status_code == 422 +def test_listing_grants_with_no_filter_returns_everything(grants): + grant_id = make_grant(grants).json()["id"] + + everyone = client.get("/access/grant") + assert grant_id in [row["id"] for row in everyone.json()] + + +def test_listing_grants_filters_by_data_type(grants): + grant_id = make_grant(grants).json()["id"] + + match = client.get("/access/grant", params={"data_type": "water level"}) + assert grant_id in [row["id"] for row in match.json()] + + no_match = client.get("/access/grant", params={"data_type": "water chemistry"}) + assert grant_id not in [row["id"] for row in no_match.json()] + + def test_listing_grants_hides_revoked_ones_by_default(grants): grant_id = make_grant(grants).json()["id"] client.post(f"/access/grant/{grant_id}/revocation") From fc98227818560cbc540c0ee1fa0c7beea4bd3916 Mon Sep 17 00:00:00 2001 From: jakeross Date: Thu, 27 Aug 2026 17:10:12 -0700 Subject: [PATCH 11/13] fix(dev): seed lexicon terms on container start alembic/versions/79a3ab24627e_add_access_control_tables.py adds permission_grant etc. but deliberately does not seed the principal_type/capability/grant_scope_type/access_data_type lexicon terms those columns FK to -- that's core/lexicon.json via init_lexicon, kept out of the migration so a new capability is a data change, not a migration. entrypoint.sh only ran alembic upgrade head, so a fresh dev DB (e.g. after dropping the postgres_data_dev volume) has the tables but no terms, and any POST /access/grant 500s on a FK violation. Run `oco initialize-lexicon` right after migrations; it's idempotent (on_conflict_do_nothing), safe on every start. Dev-only -- staging/production deploy runs alembic directly and never calls this script. Co-Authored-By: Claude Sonnet 5 --- entrypoint.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/entrypoint.sh b/entrypoint.sh index c89c621c..8eb845d3 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -19,6 +19,13 @@ echo "PostgreSQL is ready!" if [ "$RUN_MIGRATIONS" = "true" ]; then echo "Applying migrations..." alembic upgrade head + + # Lexicon terms (principal_type, capability, ...) are seeded from + # core/lexicon.json separately from the migration that adds the tables + # referencing them -- see alembic/versions/79a3ab24627e_add_access_control_tables.py. + # Idempotent (on_conflict_do_nothing), safe to run on every start. + echo "Seeding lexicon..." + oco initialize-lexicon fi echo "Starting the application..." From 0833d8d3eb402ddc312f36ea5a9939d1d941a928 Mon Sep 17 00:00:00 2001 From: jakeross Date: Fri, 28 Aug 2026 12:57:04 -0700 Subject: [PATCH 12/13] fix(tests): seed A1 scenarios at head before downgrading The two A1 scenarios that need a pre-migration database downgraded to PRE_A1_REVISION and then seeded through the ORM. Every column added by a migration newer than that revision is mapped on the models but absent from the downgraded schema, so the seed insert failed: column "data_maturity" of relation "group" does not exist data_maturity (e7c1a9f4b2d8) is only the current example -- any future ReleaseMixin column would break these steps the same way. Seed at head first, then downgrade. The downgrade drops the newer columns but keeps the rows, which is the pre-migration state these scenarios are asserting against. This is the order the already-passing reversibility scenario uses. Co-Authored-By: Claude Opus 5 --- tests/features/steps/ogc-cleanup-sprint1.py | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/tests/features/steps/ogc-cleanup-sprint1.py b/tests/features/steps/ogc-cleanup-sprint1.py index 5ccf0b22..810c26dd 100644 --- a/tests/features/steps/ogc-cleanup-sprint1.py +++ b/tests/features/steps/ogc-cleanup-sprint1.py @@ -425,10 +425,17 @@ def _ensure_head(context): @given("a clean database state before the Sprint 1 migration") def step_given_clean_database_state(context): - command.downgrade(_alembic_config(), PRE_A1_REVISION) + # Seed at head, then downgrade -- not the other way around. _seed_all() + # writes through the ORM, whose models carry every column added by + # migrations newer than PRE_A1_REVISION (data_maturity, for one), so + # seeding against the downgraded schema fails on the missing columns. + # Downgrading afterwards drops those columns but keeps the rows, which + # is exactly the pre-migration state these scenarios need. + command.upgrade(_alembic_config(), "head") with session_ctx() as session: context.seed_ids = _seed_all(session) context.add_cleanup(_teardown_a1_seed_data) + command.downgrade(_alembic_config(), PRE_A1_REVISION) @when("the Sprint 1 Alembic migration is applied") @@ -650,12 +657,15 @@ def _seed_already_consistent_layers(session): @given("the following layers were already filtering correctly before the migration:") def step_given_already_consistent_layers(context): - command.downgrade(_alembic_config(), PRE_A1_REVISION) - reset_pygeoapi_reflection() + # Seed at head before downgrading, for the reason given in + # step_given_clean_database_state. + command.upgrade(_alembic_config(), "head") with session_ctx() as session: _seed_already_consistent_layers(session) session.commit() context.add_cleanup(_teardown_a1_seed_data) + command.downgrade(_alembic_config(), PRE_A1_REVISION) + reset_pygeoapi_reflection() context.already_consistent_counts = {} for row in context.table: From b9f62233b4fd2a82cdc1b7dd9e426b6ec1cbd03e Mon Sep 17 00:00:00 2001 From: jakeross Date: Tue, 1 Sep 2026 12:31:25 -0700 Subject: [PATCH 13/13] feat(access): embargo records until a scheduled release date An embargo is a record withheld from public release until a date decided in advance: `release_status = 'embargoed'` plus a `release_at` on every ReleaseMixin table. When the date arrives, `oco release-embargoed --apply` flips the level to `public` and the record appears in the public OGC collections on their next refresh. `release_at` is intent, not enforcement. Nothing on the read path consults it, and `domain/release.py` deliberately publishes no "is this visible" predicate: one that compared dates would disagree with the views for up to a day and would eventually be used as a read-path filter, which is the distributed filtering ADR5 exists to prevent (migration baba91fe5e83). Why a job rather than a date predicate in the views: seven of the public collections are materialized views refreshed by one pg_cron job at 09:00 UTC, and `current_date` inside a matview is frozen at refresh time. A predicate would buy nothing on more than half the public surface while costing a recreation of every relation carrying it. The refresh already sets the granularity, so an embargo lifts up to a day late and never early. The job must run before that refresh. Nothing schedules it yet -- the two options (an HTTP route reachable by App Engine cron, or a SQL reimplementation under pg_cron) each need a decision nobody has made. Failure is fail-closed either way: an embargo that does not lift leaves data private. Embargo only ever widens visibility. Withdrawing something already published is an immediate `release_status` change made by a person, for the reason `domain/access.py` never backdates a revocation. Enforcement, in the four public relations that read the observation chain (b4c5d6e7f8a9): `release_status IS DISTINCT FROM 'embargoed'` at every level -- observation, sample, activity, event -- so an embargoed reading cannot be reached through a public parent. `IS DISTINCT FROM` rather than `<>` because release_status is nullable and `NULL <> 'embargoed'` is NULL, which would drop the row. Deliberately not `= 'public'`, which is what the other observation-backed relations use. Matching them would also drop every observation at `draft`, `provisional` or NULL -- a release-policy change with its own row counts to check, not an embargo. As written the four relations return byte-identical results until something is actually embargoed; the migration's downgrade output is string-equal to f4a5b6c7d8e9's production SQL. Migration w1x2y3z4a5b6 is the record of what the tidier version costs when the row states are not what you assumed. The pair is guarded by a CHECK constraint per table rather than by pydantic alone: a PATCH body carrying only `release_status` is a fragment, not a row, and the CLI, the transfers and psql do not go through pydantic at all. Chemistry cannot be embargoed per record. Those four collections read the legacy NMA_* mirror tables, which carry no release columns; per-well works, per-result does not. Options are written up in docs/data-embargo.md. Co-Authored-By: Claude Opus 5 --- CLAUDE.md | 17 +- ...b4c5d6e7f8_add_release_at_for_embargoes.py | 138 ++++ ...exclude_embargoed_from_public_ogc_views.py | 592 ++++++++++++++++++ cli/cli.py | 39 ++ core/lexicon.json | 7 + db/authorization_audit.py | 6 +- db/base.py | 16 + db/notes.py | 2 + docs/data-embargo.md | 152 +++++ domain/release.py | 109 ++++ schemas/__init__.py | 33 + services/release_schedule.py | 170 +++++ tests/test_domain_release.py | 129 ++++ tests/test_ogc_embargo.py | 242 +++++++ tests/test_release_schedule.py | 230 +++++++ tests/test_thing.py | 1 + 16 files changed, 1881 insertions(+), 2 deletions(-) create mode 100644 alembic/versions/a3b4c5d6e7f8_add_release_at_for_embargoes.py create mode 100644 alembic/versions/b4c5d6e7f8a9_exclude_embargoed_from_public_ogc_views.py create mode 100644 docs/data-embargo.md create mode 100644 domain/release.py create mode 100644 services/release_schedule.py create mode 100644 tests/test_domain_release.py create mode 100644 tests/test_ogc_embargo.py create mode 100644 tests/test_release_schedule.py diff --git a/CLAUDE.md b/CLAUDE.md index 7a65b1fd..c8f6a36f 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -256,6 +256,20 @@ The storage and the evaluator exist; the field projection does not. about. `services/access_admin.py` writes an `authorization_audit` row in the same transaction as every change. +- **Embargo is a scheduled level change, not a read-path rule.** + `release_status = 'embargoed'` plus a `release_at` date withholds a record; + `oco release-embargoed --apply` flips it to `public` when the date arrives, + and must run before the 09:00 UTC materialized-view refresh. Nothing on the + read path reads `release_at` -- seven public collections are materialized + views where `current_date` is frozen at refresh time, so a date predicate + would buy nothing there. Embargo only ever *widens* visibility; withdrawing + something published is an immediate `release_status` change. The public + water-level views exclude embargoed rows with `IS DISTINCT FROM 'embargoed'` + at every level of the observation chain -- deliberately not `= 'public'`, + which would also drop draft and NULL rows. Chemistry cannot be embargoed per + record: those collections read the legacy `NMA_*` tables, which have no + release columns. Read **`docs/data-embargo.md`** before changing any of it. + Two vocabulary fixes from it have landed and matter when reading models: - **`db/field_access_consent.py`** (`FieldAccessConsent`, table @@ -267,7 +281,8 @@ Two vocabulary fixes from it have landed and matter when reading models: (who may see it) and `data_maturity` is the review state (`provisional`, `in review`, `approved`, NULL = not stated). The `release_status` lexicon still lists `provisional` and `final` for historical rows; new code should - put review state in `data_maturity`. + put review state in `data_maturity`. A third column, `release_at`, is + the embargo date and is NULL for every record that is not embargoed. ### Database Configuration diff --git a/alembic/versions/a3b4c5d6e7f8_add_release_at_for_embargoes.py b/alembic/versions/a3b4c5d6e7f8_add_release_at_for_embargoes.py new file mode 100644 index 00000000..91c452e1 --- /dev/null +++ b/alembic/versions/a3b4c5d6e7f8_add_release_at_for_embargoes.py @@ -0,0 +1,138 @@ +"""add release_at, the date an embargoed record becomes public + +An embargo is a record withheld from public release until a date decided in +advance: `release_status = 'embargoed'` plus a `release_at`. This adds the +second half of that pair to every ReleaseMixin table. + +`release_at` is intent, not enforcement. Nothing on the read path consults it: +`services/release_schedule.py` flips `release_status` to `public` when the date +arrives, and `release_status` stays the only column the OGC views filter on. +The reason is in that module -- seven of the public collections are +materialized views refreshed nightly, where a date predicate would be frozen +at refresh time and buy nothing. + +Nullable, defaulting to NULL, meaning "no embargo". That default is +load-bearing rather than incidental: migration w1x2y3z4a5b6 records three +NGWMN exports emptied outright by a release predicate whose column defaulted +to something other than "released". No existing row changes meaning, and no +row is embargoed until somebody says so. + +Unlike e7c1a9f4b2d8's data_maturity, `transducer_observation` is included -- +it has no head start on this column. + +Revision ID: a3b4c5d6e7f8 +Revises: 79a3ab24627e +Create Date: 2026-09-01 00:00:00.000000 + +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "a3b4c5d6e7f8" +down_revision: Union[str, Sequence[str], None] = "79a3ab24627e" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +# Every table backed by a model that mixes in ReleaseMixin. +RELEASE_TABLES = ( + "address", + "analysis_method", + "aquifer_system", + "aquifer_type", + "asset", + "contact", + "data_provenance", + "deployment", + "email", + "field_access_consent", + "field_activity", + "field_event", + "field_event_participant", + "geologic_formation", + "group", + "location", + "measuring_point_history", + "monitoring_frequency_history", + "notes", + "observation", + "parameter", + "phone", + "regulatory_limit", + "sample", + "sensor", + "status_history", + "thing", + "thing_aquifer_association", + "thing_geologic_formation_association", + "thing_id_link", + "transducer_observation", + "transducer_observation_block", + "well_casing_material", + "well_purpose", + "well_screen", +) + +# sqlalchemy-continuum mirrors every tracked column into the version table, so +# the models carrying __versioned__ need the new column there too. +VERSION_TABLES = ( + "aquifer_system_version", + "geologic_formation_version", + "location_version", + "observation_version", + "parameter_version", + "regulatory_limit_version", + "thing_version", +) + +COMMENT = ( + "Date an embargoed record becomes public. NULL means no embargo. " + "Read by services/release_schedule.py, never by the read path." +) + +# An embargo names its date, and a date means an embargo. Enforced in the +# table because the schema layer cannot: a PATCH body carrying only +# `release_status` is a fragment, not a row, and the CLI, the transfers and +# psql do not go through pydantic at all. Every existing row satisfies it -- +# release_at starts NULL everywhere and nothing is embargoed yet -- so the +# constraint validates without a scan finding anything. +# +# Not applied to the _version tables: those record history, and a constraint +# on what a past state may have been would be a claim nobody checked. +CHECK_NAME = "{table}_embargo_needs_date" +CHECK_SQL = ( + "(release_status = 'embargoed' AND release_at IS NOT NULL) " + "OR (release_status IS DISTINCT FROM 'embargoed' AND release_at IS NULL)" +) + + +def upgrade() -> None: + for table in RELEASE_TABLES: + op.add_column( + table, + sa.Column("release_at", sa.Date(), nullable=True, comment=COMMENT), + ) + + for table in RELEASE_TABLES: + op.create_check_constraint( + CHECK_NAME.format(table=table), table, sa.text(CHECK_SQL) + ) + + for table in VERSION_TABLES: + op.add_column( + table, + sa.Column("release_at", sa.Date(), autoincrement=False, nullable=True), + ) + + +def downgrade() -> None: + for table in VERSION_TABLES: + op.drop_column(table, "release_at") + + for table in RELEASE_TABLES: + op.drop_constraint(CHECK_NAME.format(table=table), table, type_="check") + op.drop_column(table, "release_at") diff --git a/alembic/versions/b4c5d6e7f8a9_exclude_embargoed_from_public_ogc_views.py b/alembic/versions/b4c5d6e7f8a9_exclude_embargoed_from_public_ogc_views.py new file mode 100644 index 00000000..7b815c1c --- /dev/null +++ b/alembic/versions/b4c5d6e7f8a9_exclude_embargoed_from_public_ogc_views.py @@ -0,0 +1,592 @@ +"""exclude embargoed records from the public water-level OGC views + +The four public relations that read the observation chain gate on the thing's +release_status and nothing below it, so an embargoed observation at a public +well would still be published. This adds the missing clause to each. + +**Why `IS DISTINCT FROM 'embargoed'` and not `= 'public'`.** The other +observation-backed public relations (ogc_well_water_column, and the Group A +thing views since b8c9d0e1f2a3) filter their observations on +`release_status = 'public'`. Matching them here would be tidier and is +deliberately not done: it would also drop every observation sitting at +`draft`, `provisional`, or NULL, which is a release-policy change with its own +row counts to check, not an embargo. This predicate removes exactly the +embargoed rows, of which there are none until somebody sets one, so the four +relations return byte-identical results the day this lands. Migration +w1x2y3z4a5b6 is the record of what the tidier version costs when the row +states are not what you assumed: three NGWMN exports emptied, 3005/3005 rows. + +`IS DISTINCT FROM` rather than `<>` because release_status is nullable, and +`NULL <> 'embargoed'` is NULL, which would filter the row out -- turning a +narrow embargo clause into exactly the silent emptying above. + +Whole-thing embargoes need no change: `t.release_status = 'public'` already +excludes 'embargoed', so a thing held back disappears from every public +relation the moment its level changes. + +The chemistry collections are untouched and cannot be fixed here. +ogc_major_chemistry_results, ogc_minor_chemistry_wells, ogc_avg_tds_wells and +ogc_latest_tds_wells read the legacy NMA_* mirror tables, which carry no +release columns at all -- their only gate is the joined thing. Per-record +chemistry embargo is a separate decision; see docs/data-embargo.md. + +Only public relations change. The ogc_internal_* mount serves Bureau staff and +has never filtered on release_status; seeing embargoed data before it is +published is the point of it. + +The view bodies below are character-for-character the templates from +f4a5b6c7d8e9 (and 986e0eb85ab3 for ogc_actively_monitored_wells) with the +embargo clause added; downgrade() restores them without it. + +Revision ID: b4c5d6e7f8a9 +Revises: a3b4c5d6e7f8 +Create Date: 2026-09-01 00:00:00.000000 + +""" + +from typing import Sequence, Union + +from alembic import op +from sqlalchemy import inspect, text + +# revision identifiers, used by Alembic. +revision: str = "b4c5d6e7f8a9" +down_revision: Union[str, Sequence[str], None] = "a3b4c5d6e7f8" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +REQUIRED_TABLES = { + "thing", + "location", + "location_thing_association", + "observation", + "sample", + "field_activity", + "field_event", +} + +# These relations only exist in their public-filtered form, so the thing-level +# clause is no longer a toggle -- it is always emitted, and only the embargo +# clause below is switched between upgrade and downgrade. +PUBLIC_FILTER = " AND t.release_status = 'public'" + +# One clause per level of the chain the CTEs join: observation, sample, +# activity, event. Indented to the 16 columns the surrounding WHERE clauses +# use, since it is interpolated into the middle of them. +EMBARGO_ALIASES = ("o", "s", "fa", "fe") +EMBARGO_FILTER = "".join( + f"\n{' ' * 16}AND {alias}.release_status IS DISTINCT FROM 'embargoed'" + for alias in EMBARGO_ALIASES +) + +METERS_TO_FEET = 3.28084 + +LATEST_LOCATION_CTE = """ +SELECT DISTINCT ON (lta.thing_id) + lta.thing_id, + lta.location_id, + lta.effective_start +FROM location_thing_association AS lta +WHERE lta.effective_end IS NULL +ORDER BY lta.thing_id, lta.effective_start DESC +""".strip() + + +def _drop_view_or_materialized_view(view_name: str) -> None: + # DROP VIEW IF EXISTS / DROP MATERIALIZED VIEW IF EXISTS only suppress + # "relation does not exist" -- Postgres still raises WrongObjectType if + # the relation exists as the other kind (e.g. DROP VIEW against an + # existing materialized view), so the relation's actual kind must be + # checked first rather than trying both blindly. + bind = op.get_bind() + relkind = bind.execute( + text("SELECT relkind FROM pg_class WHERE oid = to_regclass(:name)"), + {"name": view_name}, + ).scalar() + if relkind == "m": + op.execute(text(f"DROP MATERIALIZED VIEW IF EXISTS {view_name}")) + elif relkind == "v": + op.execute(text(f"DROP VIEW IF EXISTS {view_name}")) + + +def _check_required_tables() -> None: + bind = op.get_bind() + inspector = inspect(bind) + existing_tables = set(inspector.get_table_names(schema="public")) + missing = REQUIRED_TABLES - existing_tables + if missing: + raise RuntimeError( + "Cannot apply the embargo filter to the OGC views. " + f"Missing required tables: {', '.join(sorted(missing))}" + ) + + +def _create_latest_depth_view(with_embargo: bool) -> str: + release_filter = PUBLIC_FILTER + (EMBARGO_FILTER if with_embargo else "") + return f""" + CREATE MATERIALIZED VIEW ogc_latest_depth_to_water_wells AS + WITH latest_location AS ( +{LATEST_LOCATION_CTE} + ), + ranked_obs AS ( + SELECT + fe.thing_id, + o.id AS observation_id, + o.observation_datetime, + o.value, + o.measuring_point_height, + -- Treat NULL measuring_point_height as 0 when computing + -- depth_to_water_bgs. + ( + o.value - COALESCE(o.measuring_point_height, 0) + ) AS depth_to_water_bgs, + ROW_NUMBER() OVER ( + PARTITION BY fe.thing_id + ORDER BY o.observation_datetime DESC, o.id DESC + ) AS rn + FROM observation AS o + JOIN sample AS s ON s.id = o.sample_id + JOIN field_activity AS fa ON fa.id = s.field_activity_id + JOIN field_event AS fe ON fe.id = fa.field_event_id + JOIN thing AS t ON t.id = fe.thing_id + WHERE + t.thing_type = 'water well' + AND fa.activity_type = 'groundwater level' + AND o.value IS NOT NULL{release_filter} + ) + SELECT + t.id AS id, + t.name, + t.thing_type, + ro.observation_id, + ro.observation_datetime, + ro.value AS depth_to_water_reference, + ro.measuring_point_height, + ro.depth_to_water_bgs, + l.point + FROM ranked_obs AS ro + JOIN thing AS t ON t.id = ro.thing_id + JOIN latest_location AS ll ON ll.thing_id = t.id + JOIN location AS l ON l.id = ll.location_id + WHERE ro.rn = 1 + """ + + +def _create_depth_to_water_trend_view(with_embargo: bool) -> str: + release_filter = PUBLIC_FILTER + (EMBARGO_FILTER if with_embargo else "") + return f""" + CREATE MATERIALIZED VIEW ogc_depth_to_water_trend_wells AS + WITH latest_location AS ( +{LATEST_LOCATION_CTE} + ), + obs AS ( + SELECT + fe.thing_id, + o.observation_datetime, + (o.value - COALESCE(o.measuring_point_height, 0)) AS depth_to_water_bgs + FROM observation AS o + JOIN sample AS s ON s.id = o.sample_id + JOIN field_activity AS fa ON fa.id = s.field_activity_id + JOIN field_event AS fe ON fe.id = fa.field_event_id + JOIN thing AS t ON t.id = fe.thing_id + WHERE + t.thing_type = 'water well' + AND fa.activity_type = 'groundwater level' + AND o.value IS NOT NULL + AND o.observation_datetime IS NOT NULL{release_filter} + ), + agg AS ( + SELECT + ob.thing_id, + COUNT(*)::integer AS record_count, + MIN(ob.observation_datetime) AS first_observation_datetime, + MAX(ob.observation_datetime) AS last_observation_datetime, + EXTRACT(EPOCH FROM (MAX(ob.observation_datetime) - MIN(ob.observation_datetime))) + / 31557600.0 AS span_years, + REGR_SLOPE( + ob.depth_to_water_bgs, + EXTRACT(EPOCH FROM ob.observation_datetime) + ) * 31557600.0 AS slope_ft_per_year + FROM obs AS ob + GROUP BY ob.thing_id + ) + SELECT + t.id AS id, + t.name, + t.thing_type, + a.record_count, + a.first_observation_datetime, + a.last_observation_datetime, + a.span_years, + a.slope_ft_per_year, + CASE + WHEN a.record_count >= 10 OR (a.record_count >= 4 AND a.span_years >= 2.0) THEN + CASE + WHEN a.slope_ft_per_year IS NULL THEN 'not enough data' + WHEN a.slope_ft_per_year > 0.25 THEN 'increasing' + WHEN a.slope_ft_per_year < -0.25 THEN 'decreasing' + ELSE 'stable' + END + ELSE 'not enough data' + END AS trend_category, + l.point + FROM agg AS a + JOIN thing AS t ON t.id = a.thing_id + JOIN latest_location AS ll ON ll.thing_id = t.id + JOIN location AS l ON l.id = ll.location_id + """ + + +def _create_water_well_summary_view(with_embargo: bool) -> str: + release_filter = PUBLIC_FILTER + (EMBARGO_FILTER if with_embargo else "") + return f""" + CREATE MATERIALIZED VIEW ogc_water_well_summary AS + WITH latest_location AS ( +{LATEST_LOCATION_CTE} + ), + wl_obs AS ( + SELECT + fe.thing_id, + o.id AS observation_id, + o.observation_datetime, + (o.value - COALESCE(o.measuring_point_height, 0)) AS water_level + FROM observation AS o + JOIN sample AS s ON s.id = o.sample_id + JOIN field_activity AS fa ON fa.id = s.field_activity_id + JOIN field_event AS fe ON fe.id = fa.field_event_id + JOIN thing AS t ON t.id = fe.thing_id + WHERE + t.thing_type = 'water well' + AND fa.activity_type = 'groundwater level' + AND o.value IS NOT NULL + AND o.observation_datetime IS NOT NULL{release_filter} + ), + wl_agg AS ( + SELECT + w.thing_id, + COUNT(*)::integer AS total_water_levels, + MIN(w.water_level) AS min_water_level, + MAX(w.water_level) AS max_water_level, + REGR_SLOPE( + w.water_level, + EXTRACT(EPOCH FROM w.observation_datetime) + ) * 31557600.0 AS water_level_trend_ft_per_year + FROM wl_obs AS w + GROUP BY w.thing_id + ), + wl_last AS ( + SELECT + ranked.thing_id, + ranked.water_level AS last_water_level, + ranked.observation_datetime AS last_water_level_datetime + FROM ( + SELECT + w.thing_id, + w.water_level, + w.observation_datetime, + ROW_NUMBER() OVER ( + PARTITION BY w.thing_id + ORDER BY w.observation_datetime DESC, w.observation_id DESC + ) AS rn + FROM wl_obs AS w + ) AS ranked + WHERE ranked.rn = 1 + ) + SELECT + t.id AS id, + t.name, + t.well_depth, + l.elevation, + dpl.collection_method AS elevation_method, + t.nma_formation_zone AS formation_zone, + wa.total_water_levels, + wl.last_water_level, + wl.last_water_level_datetime, + wa.min_water_level, + wa.max_water_level, + wa.water_level_trend_ft_per_year, + l.point + FROM thing AS t + JOIN latest_location AS ll ON ll.thing_id = t.id + JOIN location AS l ON l.id = ll.location_id + JOIN wl_agg AS wa ON wa.thing_id = t.id + LEFT JOIN wl_last AS wl ON wl.thing_id = t.id + LEFT JOIN LATERAL ( + SELECT dp.collection_method + FROM data_provenance AS dp + WHERE + dp.target_table = 'location' + AND dp.target_id = l.id + AND dp.field_name = 'elevation' + ORDER BY dp.id DESC + LIMIT 1 + ) AS dpl ON true + WHERE t.thing_type = 'water well' + AND wa.total_water_levels > 0 + """ + + +def _create_water_elevation_view(with_embargo: bool) -> str: + release_filter = PUBLIC_FILTER + (EMBARGO_FILTER if with_embargo else "") + return f""" + CREATE MATERIALIZED VIEW ogc_water_elevation_wells AS + WITH latest_location AS ( +{LATEST_LOCATION_CTE} + ), + ranked_obs AS ( + SELECT + fe.thing_id, + o.id AS observation_id, + o.observation_datetime, + CASE + WHEN lower(trim(o.unit)) IN ('m', 'meter', 'meters', 'metre', 'metres') THEN + (o.value * {METERS_TO_FEET}) - COALESCE(o.measuring_point_height, 0) + WHEN lower(trim(o.unit)) IN ('ft', 'foot', 'feet') THEN + o.value - COALESCE(o.measuring_point_height, 0) + ELSE + NULL + END AS depth_to_water_below_ground_surface + FROM observation AS o + JOIN sample AS s ON s.id = o.sample_id + JOIN field_activity AS fa ON fa.id = s.field_activity_id + JOIN field_event AS fe ON fe.id = fa.field_event_id + JOIN thing AS t ON t.id = fe.thing_id + WHERE + t.thing_type = 'water well' + AND fa.activity_type = 'groundwater level' + AND o.value IS NOT NULL + AND o.observation_datetime IS NOT NULL + AND lower(trim(o.unit)) IN ( + 'm', + 'meter', + 'meters', + 'metre', + 'metres', + 'ft', + 'foot', + 'feet' + ){release_filter} + ), + latest_obs AS ( + SELECT + ro.*, + ROW_NUMBER() OVER ( + PARTITION BY ro.thing_id + ORDER BY ro.observation_datetime DESC, ro.observation_id DESC + ) AS rn + FROM ranked_obs AS ro + ) + SELECT + t.id AS id, + t.name, + t.thing_type, + lo.observation_id, + lo.observation_datetime, + l.elevation AS elevation_m, + lo.depth_to_water_below_ground_surface AS depth_to_water_below_ground_surface_ft, + ((l.elevation * {METERS_TO_FEET}) - lo.depth_to_water_below_ground_surface) + AS water_elevation_ft, + l.point + FROM latest_obs AS lo + JOIN thing AS t ON t.id = lo.thing_id + JOIN latest_location AS ll ON ll.thing_id = t.id + JOIN location AS l ON l.id = ll.location_id + WHERE lo.rn = 1 + """ + + +def _create_actively_monitored_wells_view(all_groups: bool) -> str: + if all_groups: + # Aggregated: one row per well, group_ids/group_names/group_types as + # arrays, so `id` stays unique even when a well belongs to several + # groups. release_status = 'public' is checked on the group row + # itself (mirrors _create_project_areas_view's public_only handling) + # since any group can appear here now, not just one hardcoded one. + # group_thing_association has no unique constraint on + # (group_id, thing_id), so distinct_memberships de-dupes before + # aggregating; all three arrays are ordered by the same group_id key + # so they stay index-aligned with each other (ordering each array by + # its own column, e.g. names alphabetically, would desync them). + return """ + CREATE VIEW ogc_actively_monitored_wells AS + WITH latest_monitoring_status AS ( + SELECT DISTINCT ON (sh.target_id) + sh.target_id AS thing_id, + sh.status_value + FROM status_history AS sh + WHERE + sh.target_table = 'thing' + AND sh.status_type = 'Monitoring Status' + ORDER BY sh.target_id, sh.start_date DESC, sh.id DESC + ), + distinct_memberships AS ( + SELECT DISTINCT + gta.thing_id, + g.id AS group_id, + g.name AS group_name, + g.group_type + FROM group_thing_association AS gta + JOIN "group" AS g ON g.id = gta.group_id + WHERE g.release_status = 'public' + ) + SELECT + wws.id, + wws.name, + 'water well'::text AS thing_type, + wws.well_depth, + wws.elevation, + wws.elevation_method, + wws.formation_zone, + wws.total_water_levels, + wws.last_water_level, + wws.last_water_level_datetime, + wws.min_water_level, + wws.max_water_level, + wws.water_level_trend_ft_per_year, + array_agg(dm.group_id ORDER BY dm.group_id) AS group_ids, + array_agg(dm.group_name ORDER BY dm.group_id) AS group_names, + array_agg(dm.group_type ORDER BY dm.group_id) AS group_types, + wws.point + FROM ogc_water_well_summary AS wws + JOIN latest_monitoring_status AS lms ON lms.thing_id = wws.id + JOIN distinct_memberships AS dm ON dm.thing_id = wws.id + WHERE lms.status_value = 'Currently monitored' + GROUP BY + wws.id, wws.name, wws.well_depth, wws.elevation, + wws.elevation_method, wws.formation_zone, + wws.total_water_levels, wws.last_water_level, + wws.last_water_level_datetime, wws.min_water_level, + wws.max_water_level, wws.water_level_trend_ft_per_year, + wws.point + """ + # Historical (downgrade target): byte-for-byte the pre-fix view, single + # group_id/group_name/group_type columns, scoped to one hardcoded group. + return """ + CREATE VIEW ogc_actively_monitored_wells AS + WITH latest_monitoring_status AS ( + SELECT DISTINCT ON (sh.target_id) + sh.target_id AS thing_id, + sh.status_value + FROM status_history AS sh + WHERE + sh.target_table = 'thing' + AND sh.status_type = 'Monitoring Status' + ORDER BY sh.target_id, sh.start_date DESC, sh.id DESC + ) + SELECT + wws.id, + wws.name, + 'water well'::text AS thing_type, + wws.well_depth, + wws.elevation, + wws.elevation_method, + wws.formation_zone, + wws.total_water_levels, + wws.last_water_level, + wws.last_water_level_datetime, + wws.min_water_level, + wws.max_water_level, + wws.water_level_trend_ft_per_year, + g.id AS group_id, + g.name AS group_name, + g.group_type, + wws.point + FROM "group" AS g + JOIN group_thing_association AS gta ON gta.group_id = g.id + JOIN ogc_water_well_summary AS wws ON wws.id = gta.thing_id + JOIN latest_monitoring_status AS lms ON lms.thing_id = wws.id + WHERE lower(trim(g.name)) = 'water level network' + AND lms.status_value = 'Currently monitored' + """ + + +def _recreate_water_level_views(with_embargo: bool) -> None: + """Rebuild the four observation-backed relations, embargo clause on or off. + + ogc_actively_monitored_wells depends on ogc_water_well_summary via a direct + JOIN; Postgres refuses to drop a materialized view while a dependent view + exists, so it goes first and comes back last. It is recreated from + 986e0eb85ab3's all-groups form -- the shape in production -- not + f4a5b6c7d8e9's, and its own SQL is unchanged: it inherits the embargo + filter through the summary it selects from. + """ + _drop_view_or_materialized_view("ogc_actively_monitored_wells") + + _drop_view_or_materialized_view("ogc_latest_depth_to_water_wells") + op.execute(text(_create_latest_depth_view(with_embargo))) + op.execute( + text( + "COMMENT ON MATERIALIZED VIEW ogc_latest_depth_to_water_wells IS " + "'Latest depth-to-water per well view for pygeoapi.'" + ) + ) + op.execute( + text( + "CREATE UNIQUE INDEX ux_ogc_latest_depth_to_water_wells_id " + "ON ogc_latest_depth_to_water_wells (id)" + ) + ) + + _drop_view_or_materialized_view("ogc_depth_to_water_trend_wells") + op.execute(text(_create_depth_to_water_trend_view(with_embargo))) + op.execute( + text( + "COMMENT ON MATERIALIZED VIEW ogc_depth_to_water_trend_wells IS " + "'Depth-to-water trend classification for water wells.'" + ) + ) + op.execute( + text( + "CREATE UNIQUE INDEX ux_ogc_depth_to_water_trend_wells_id " + "ON ogc_depth_to_water_trend_wells (id)" + ) + ) + + _drop_view_or_materialized_view("ogc_water_well_summary") + op.execute(text(_create_water_well_summary_view(with_embargo))) + op.execute( + text( + "COMMENT ON MATERIALIZED VIEW ogc_water_well_summary IS " + "'Summary statistics for water wells including water-level trend.'" + ) + ) + op.execute( + text( + "CREATE UNIQUE INDEX ux_ogc_water_well_summary_id " + "ON ogc_water_well_summary (id)" + ) + ) + + _drop_view_or_materialized_view("ogc_water_elevation_wells") + op.execute(text(_create_water_elevation_view(with_embargo))) + op.execute( + text( + "COMMENT ON MATERIALIZED VIEW ogc_water_elevation_wells IS " + "'Latest water elevation per well with explicit units: " + "elevation_m, depth_to_water_below_ground_surface_ft, water_elevation_ft.'" + ) + ) + op.execute( + text( + "CREATE UNIQUE INDEX ux_ogc_water_elevation_wells_id " + "ON ogc_water_elevation_wells (id)" + ) + ) + + # Recreate now that ogc_water_well_summary exists again. + op.execute(text(_create_actively_monitored_wells_view(all_groups=True))) + op.execute( + text( + "COMMENT ON VIEW ogc_actively_monitored_wells IS " + "'Actively (currently) monitored wells across all groups for pygeoapi.'" + ) + ) + + +def upgrade() -> None: + _check_required_tables() + _recreate_water_level_views(with_embargo=True) + + +def downgrade() -> None: + _recreate_water_level_views(with_embargo=False) diff --git a/cli/cli.py b/cli/cli.py index 80651d78..a3e39fc0 100644 --- a/cli/cli.py +++ b/cli/cli.py @@ -1469,6 +1469,45 @@ def seed_access_grants( typer.echo("Nothing was written. Re-run with --apply.") +@cli.command("release-embargoed") +def release_embargoed( + apply: bool = typer.Option( + False, + "--apply", + help="Write the changes. Without this the command only shows the plan.", + ), + verbose: bool = typer.Option( + False, "--verbose", help="List every record, not just the counts." + ), +): + """Publish records whose embargo date has arrived. + + Flips `release_status` from `embargoed` to `public` where `release_at` has + been reached, and writes an authorization_audit row for each. Meant to run + daily, before the 09:00 UTC materialized-view refresh -- a record released + after the refresh waits another day to appear in the OGC collections. + + Previews by default. Idempotent: a released record is no longer embargoed. + """ + from db.engine import session_ctx + from services.release_schedule import lift_due_embargoes + + with session_ctx() as session: + plan = lift_due_embargoes(session, apply=apply) + + verb = "Released" if apply else "Would release" + typer.echo(f"{verb} {len(plan.lifted)} record(s).") + for table, count in sorted(plan.by_table().items()): + typer.echo(f" {table}: {count}") + + if verbose: + for entry in plan.lifted: + typer.echo(f" + {plan.describe(entry)}") + + if plan.lifted and not apply: + typer.echo("Nothing was written. Re-run with --apply.") + + if __name__ == "__main__": cli() diff --git a/core/lexicon.json b/core/lexicon.json index 10da407b..5046b178 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -1828,6 +1828,13 @@ "term": "private", "definition": "private version" }, + { + "categories": [ + "release_status" + ], + "term": "embargoed", + "definition": "withheld from public release until release_at, then published by the scheduled release job" + }, { "categories": [ "relation" diff --git a/db/authorization_audit.py b/db/authorization_audit.py index 1eb22708..7faebec6 100644 --- a/db/authorization_audit.py +++ b/db/authorization_audit.py @@ -17,7 +17,7 @@ db/authorization_audit.py The append-only log of authorization events: grants, revocations, consent -captured or withdrawn, destinations registered or retired. +captured or withdrawn, destinations registered or retired, embargoes lifted. ADR5 asks for this from the first commit, for one reason: when something is exposed that should not have been, the first question is never "what was the @@ -46,6 +46,10 @@ CONSENT_RECORDED = "consent.recorded" CONSENT_REVOKED = "consent.revoked" DESTINATION_REGISTERED = "destination.registered" +# An embargo lifting is an authorization change like any other -- it widens +# who may see a record -- so it lands in the same log rather than a second one +# nobody thinks to read after an incident. +RELEASE_LIFTED = "release.lifted" class AuthorizationAudit(Base, AutoBaseMixin): diff --git a/db/base.py b/db/base.py index e909ce36..892868bf 100644 --- a/db/base.py +++ b/db/base.py @@ -41,6 +41,7 @@ from sqlalchemy import ( Column, + Date, DateTime, func, Integer, @@ -110,6 +111,17 @@ class ReleaseMixin: for the rows that predate it. The vocabulary and the column originated on ``TransducerObservation``; this mixin is where it now lives so that every released record can carry the second axis. + + ``release_at`` is the third, and it is *intent* rather than enforcement: + the date an ``embargoed`` record becomes ``public``. Nothing on the read + path consults it. ``services/release_schedule.py`` flips the level when + the date arrives, and ``release_status`` stays the only thing the OGC + views filter on. See ``docs/data-embargo.md``. + + NULL means no embargo, and that default is load-bearing rather than + incidental: migration ``w1x2y3z4a5b6`` records three NGWMN exports that + a release predicate emptied outright because the column it tested + defaulted to something other than "released". """ @declared_attr @@ -120,6 +132,10 @@ def release_status(self): def data_maturity(self): return lexicon_term(nullable=True) + @declared_attr + def release_at(self): + return mapped_column(Date, nullable=True) + class AuditMixin: """Mixin to add standard audit columns to a model.""" diff --git a/db/notes.py b/db/notes.py index 0a38b53f..390dc860 100644 --- a/db/notes.py +++ b/db/notes.py @@ -111,6 +111,7 @@ def add_note( note_type: str, release_status: str = "draft", data_maturity: str = None, + release_at=None, created_by: str = None, ) -> Notes: """ @@ -125,6 +126,7 @@ def add_note( target_table=self.__class__.__tablename__, release_status=release_status, data_maturity=data_maturity, + release_at=release_at, ) def _get_notes(self, note_type: str) -> list[Notes]: diff --git a/docs/data-embargo.md b/docs/data-embargo.md new file mode 100644 index 00000000..262f4c9b --- /dev/null +++ b/docs/data-embargo.md @@ -0,0 +1,152 @@ +# Embargoing data until a date + +An embargo is a record withheld from public release until a date decided in +advance. It is two columns on every `ReleaseMixin` table: + +| column | meaning | +| --- | --- | +| `release_status = 'embargoed'` | the record is being withheld | +| `release_at` | the date it becomes public | + +When the date arrives, `oco release-embargoed --apply` flips the level to +`public`. The record then appears in the public OGC collections on their next +refresh. + +## The two rules worth knowing before you touch this + +**`release_at` is intent. `release_status` is enforcement.** Nothing on the +read path consults `release_at` — not the OGC views, not the API, not the +visibility layer. It exists so a job can change `release_status` on the right +day. If you find yourself adding a date comparison to a read path, stop: that +is the distributed filtering ADR5 exists to prevent, and migration +`baba91fe5e83` is what it already cost this repository once. + +**An embargo only ever widens visibility.** The scheduled path turns +`embargoed` into `public`, and does nothing else. Withdrawing something already +published is an immediate change to `release_status`, made by a person — never +scheduled. This mirrors `domain/access.py`, where a revocation takes effect at +once and is never backdated: a promise to hide something *later* is not a +promise anyone should rely on. + +## Why a job and not a predicate in the views + +The obvious design is `WHERE release_at IS NULL OR release_at <= current_date` +in the OGC views, so that no job is needed and the embargo lifts itself at read +time. It was rejected. + +Seven of the public collections are **materialized** views, refreshed by one +pg_cron job at 09:00 UTC (`docs/pg_cron-nightly-refresh.md`). `current_date` +inside a materialized view is frozen at refresh time. A date predicate would +therefore buy nothing on more than half the public surface, while costing a +recreation of every relation that carries it. The refresh already sets the +granularity, so the cheaper mechanism with the same behaviour wins. + +Consequence to be honest about: **an embargo lifts up to a day late**, bounded +by when the release job and the refresh run. It never lifts early. + +## Running it + +```bash +oco release-embargoed +``` + +Previews by default, like `oco seed-access-grants`; add `--apply` to write. +Idempotent — a released record is no longer embargoed, so a second run finds +nothing. Every flip writes an `authorization_audit` row recording the date the +embargo was set for alongside the date it was actually lifted, so "was this +released early" stays answerable. + +**Ordering matters.** Run it *before* the 09:00 UTC materialized-view refresh. +A record released after the refresh waits another day to appear. + +### It is not scheduled yet + +The CLI exists; nothing runs it on a timer. Until that is wired, an embargo +lifts when someone runs the command. Two ways to wire it, neither free: + +- **App Engine cron / Cloud Scheduler → an HTTP route.** Fits the existing + deployment, but means adding a route whose effect is publishing data. That + route needs its own authorization decision — `X-Appengine-Cron` alone is a + header, not an authenticator, once anything else can reach the app. +- **pg_cron, next to the refresh job.** The scheduler is already there and the + ordering would be trivial to guarantee. But pg_cron cannot call Python, so + the flip would have to be reimplemented in SQL — a second copy of the rule, + which is the drift this repository has already paid for once. + +Failure in either direction is safe: if the job does not run, embargoed records +stay embargoed. + +## What is enforced, and where + +Migration `b4c5d6e7f8a9` adds the embargo clause to the four public relations +that read the observation chain: + +- `ogc_water_well_summary` +- `ogc_water_elevation_wells` +- `ogc_depth_to_water_trend_wells` +- `ogc_latest_depth_to_water_wells` + +The clause is `release_status IS DISTINCT FROM 'embargoed'`, applied at each +level of the chain — observation, sample, field activity, field event — so an +embargoed reading cannot be reached through a public parent. `IS DISTINCT +FROM` rather than `<>` because `release_status` is nullable and `NULL <> +'embargoed'` is NULL, which would drop the row. + +`ogc_well_water_column` and the Group A thing views need no change: they +already filter their observations on `release_status = 'public'`, which +excludes `embargoed` for free. Whole-thing embargoes need no change anywhere, +for the same reason. + +The `ogc_internal_*` mount is deliberately untouched. It serves Bureau staff +and has never filtered on release level; seeing embargoed data before it is +published is what it is for. + +### Why the clause is not `= 'public'` + +Matching the other relations would be tidier, and is deliberately not done. It +would also drop every observation sitting at `draft`, `provisional`, or NULL — +a release-policy change with its own row counts to check, not an embargo. As +written, the four relations returned byte-identical results the day the +migration landed, because nothing was embargoed yet. + +Migration `w1x2y3z4a5b6` is the record of what the tidier version costs when +the row states are not what you assumed: three NGWMN exports emptied outright, +3005/3005 rows on one of them. If someone later decides the public relations +should require `= 'public'` all the way down, that is its own change, with row +counts taken on a prod clone before and after. + +## Chemistry cannot be embargoed per record + +`ogc_major_chemistry_results`, `ogc_minor_chemistry_wells`, +`ogc_avg_tds_wells` and `ogc_latest_tds_wells` do not read `observation` and +`sample`. They read the legacy `NMA_Chemistry_SampleInfo`, +`NMA_MajorChemistry` and `NMA_MinorTraceChemistry` mirror tables, which are +plain `Base` models carrying no release columns at all — no `release_status`, +no `data_maturity`, no `release_at`. Their only gate is the joined thing. + +So chemistry can be embargoed **per well**, by embargoing the thing, and not +per result. Since chemistry is the usual reason anyone wants an embargo, this +is the gap to close first. Three ways, none cheap: + +1. **Release columns on the `NMA_*` mirrors.** They are deprecated and frozen + (`transfers/README.md`), and repopulated by a deprecated driver that would + have to learn to preserve a governance column it knows nothing about. Key + anything added there on `nma_global_id`, which is stable and UNIQUE — never + the autoincrement `id`. +2. **An embargo side-table** keyed by `nma_global_id`, joined into the four + views. Leaves the frozen tables alone, at the cost of governance living in + two places, which is what ADR5 exists to stop. +3. **Wait for chemistry to move onto the Ocotillo `observation` model.** Then + it joins the chain above and costs nothing. Not currently scheduled. + +## Files + +| file | what it holds | +| --- | --- | +| `domain/release.py` | the rules, over plain values, no database | +| `services/release_schedule.py` | loads due rows, flips them, writes the audit | +| `cli/cli.py` (`release-embargoed`) | the command | +| `db/base.py` (`ReleaseMixin`) | the `release_at` column | +| `alembic/versions/a3b4c5d6e7f8_*` | the column, on 35 tables and 7 version tables | +| `alembic/versions/b4c5d6e7f8a9_*` | the embargo clause in the four public views | +| `tests/test_ogc_embargo.py` | the claim, checked against the relations themselves | diff --git a/domain/release.py b/domain/release.py new file mode 100644 index 00000000..ab4e60ef --- /dev/null +++ b/domain/release.py @@ -0,0 +1,109 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Embargo rules, as plain functions over plain values. + +An embargo is a record held back from public release until a date that was +decided in advance: `release_status = 'embargoed'` plus a `release_at`. When +the date arrives, `services/release_schedule.py` flips the level to `public` +and the record appears in the OGC views on their next refresh. + +Three invariants this module exists to keep: + +* **Intent is not enforcement.** `release_status` is the only thing the views + filter on. `release_at` says when that level is due to change and is read by + the release job alone. This module therefore publishes no "is this visible" + predicate: one that compared dates would disagree with the views for up to a + day and would eventually be used as a read-path filter, which is the + distributed filtering ADR5 exists to prevent (see migration `baba91fe5e83`). + +* **An embargo only ever widens visibility.** The scheduled path turns + `embargoed` into `public` and does nothing else. Making a record *less* + visible is an immediate change to `release_status`, never a scheduled one, + for the same reason a revocation in `domain/access.py` is never backdated: + a promise to hide something later is not a promise anybody should rely on. + +* **An embargo names its date.** `embargoed` without a `release_at` would be + a hold nothing ever lifts, and a `release_at` on any other level would be a + date nothing ever reads. Both are rejected before the row is written. An + indefinite hold is `private`, which is honest about never lifting itself. +""" + +from datetime import date + +# The level a record sits at while it is being withheld, and the one the +# scheduled release moves it to. Spelled as the lexicon spells them; +# tests/test_domain_release.py pins these against core/lexicon.json so a +# rename there cannot silently strand the release job. +STATUS_EMBARGOED = "embargoed" +STATUS_PUBLIC = "public" +# The level for a hold that nothing lifts. Named here only so the error +# message can point at it. +STATUS_PRIVATE = "private" + + +class ReleaseRuleError(ValueError): + """Base for embargo rule violations. A ValueError, per ADR4.""" + + +class MissingReleaseDate(ReleaseRuleError): + pass + + +class UnscheduledReleaseDate(ReleaseRuleError): + pass + + +def validate_release(release_status: str | None, release_at: date | None) -> None: + """Reject an embargo that could not be lifted honestly. + + Raised before a row is written, so the invariant holds in the table + rather than in the reader. + """ + if release_status == STATUS_EMBARGOED and release_at is None: + raise MissingReleaseDate( + "An embargoed record names the date it is released. For a hold " + f"with no end, use '{STATUS_PRIVATE}' instead." + ) + if release_at is not None and release_status != STATUS_EMBARGOED: + raise UnscheduledReleaseDate( + f"release_at is only read for '{STATUS_EMBARGOED}' records, so " + f"setting it on a '{release_status}' record schedules nothing." + ) + + +def due_for_release( + release_status: str | None, + release_at: date | None, + on_date: date, +) -> bool: + """Whether this record's embargo has run out by ``on_date``. + + The only date comparison in the embargo feature, and it belongs to the + release job. A record is due on its ``release_at`` itself, not the day + after: the date is when the record becomes public, not the last day it is + held. + """ + if release_status != STATUS_EMBARGOED: + return False + if release_at is None: + # Unreachable through validate_release, but a row written before this + # module existed, or by hand, must not crash the nightly job. + return False + return on_date >= release_at + + +# ============= EOF ============================================= diff --git a/schemas/__init__.py b/schemas/__init__.py index de41b888..a556705d 100644 --- a/schemas/__init__.py +++ b/schemas/__init__.py @@ -17,11 +17,13 @@ from typing import Annotated from core.enums import DataMaturity, ReleaseStatus +from domain.release import validate_release from pydantic import ( BaseModel, ConfigDict, AwareDatetime, field_validator, + model_validator, ) from pydantic.functional_validators import AfterValidator from pydantic.json_schema import JsonSchemaValue @@ -39,6 +41,9 @@ class BaseCreateModel(BaseModel): # Orthogonal to release_status: data can be published and provisional at # the same time (ADR5). NULL means not stated. data_maturity: DataMaturity | None = None + # The date an embargoed record becomes public. NULL for everything that is + # not embargoed. See docs/data-embargo.md. + release_at: date | None = None @field_validator("release_status", mode="before") @classmethod @@ -50,10 +55,37 @@ def coerce_release_status(cls, v): raise ValueError(f"Invalid release_status: {v}") return v + @model_validator(mode="after") + def check_release_schedule(self): + """An embargo names its date, and a date means an embargo. + + Sound here because a create payload is the whole row. The same pair is + enforced by a CHECK constraint on every ReleaseMixin table, which is + what covers the paths this validator cannot see -- a PATCH that sets + one half, the CLI, a transfer, raw SQL. + """ + # release_status arrives as the lexicon-backed enum or as the plain + # string, depending on whether the subclass narrowed the annotation. + status = self.release_status + validate_release(getattr(status, "value", status), self.release_at) + return self + class BaseUpdateModel(BaseCreateModel): release_status: ReleaseStatus | None = None + @model_validator(mode="after") + def check_release_schedule(self): + """Deliberately not checked here. + + An update payload is a fragment: `{"release_status": "embargoed"}` on a + row that already carries a release_at is legitimate, and so is clearing + an embargo by setting the level back. The invariant is about the + resulting row, which this schema cannot see, so it is left to the CHECK + constraint that guards the table itself. + """ + return self + def past_or_today_validator( value: date | datetime | None, @@ -117,6 +149,7 @@ class BaseResponseModel(BaseModel): created_at: UTCAwareDatetime release_status: ReleaseStatus data_maturity: DataMaturity | None = None + release_at: date | None = None model_config = ConfigDict( from_attributes=True, diff --git a/services/release_schedule.py b/services/release_schedule.py new file mode 100644 index 00000000..308487f8 --- /dev/null +++ b/services/release_schedule.py @@ -0,0 +1,170 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +services/release_schedule.py + +Lifts embargoes whose date has arrived: `release_status = 'embargoed'` becomes +`'public'` once `release_at` is reached. + +**Why a job rather than a predicate in the views.** Seven of the public OGC +collections are materialized views refreshed by one pg_cron job at 09:00 UTC +(migration ``b6c7d8e9f0a1``). ``current_date`` inside a matview is frozen at +refresh time, so a date predicate in the view SQL would buy nothing on more +than half the public surface while costing a recreation of every relation. +The refresh already sets the granularity; this job runs before it and leaves +``release_status`` as the only thing the views test. + +**Fail closed.** If this job does not run, embargoed records stay embargoed. +The failure mode is data staying private a day longer than promised, which is +the direction that does not require an apology. + +**One direction.** Nothing here makes a record less visible. Withdrawing +something already published is an immediate `release_status` change made by a +person, for the reason ``domain/access.py`` never backdates a revocation. + +Every flip lands an ``authorization_audit`` row in the same transaction, and +records the date the embargo was set for, so the log answers "was this +released early" and not only "was this released". +""" + +from dataclasses import dataclass, field +from datetime import date + +from sqlalchemy import select + +from db.authorization_audit import RELEASE_LIFTED, AuthorizationAudit +from db.field import FieldActivity, FieldEvent +from db.location import Location +from db.observation import Observation +from db.sample import Sample +from db.thing import Thing +from domain.release import STATUS_EMBARGOED, STATUS_PUBLIC, due_for_release + +# Recorded as the actor. Not a person: nobody decided anything on the day the +# embargo lifted, which is the point of scheduling it. +RELEASE_ACTOR = "system:release-schedule" + +# The models a scheduled release may touch, named one by one. +# +# Every `ReleaseMixin` model carries `release_at`, so this could iterate the +# mapper registry instead. It does not, for the reason `domain/access.py` has +# no wildcard data type: a model added next year should not silently join the +# set of things a nightly job rewrites. Adding one here is a deliberate line +# in a diff. +# +# These six are the water-level chain the public OGC views actually filter on +# -- thing, location, and the field-data path down to the observation -- so an +# embargo on any of them changes what `/ogcapi` publishes. Chemistry is +# absent: those collections read the legacy `NMA_*` tables, which carry no +# release columns at all. See docs/data-embargo.md. +RELEASABLE_MODELS = ( + Thing, + Location, + FieldEvent, + FieldActivity, + Sample, + Observation, +) + + +@dataclass +class ReleasePlan: + """What lifting would do, or did.""" + + lifted: list = field(default_factory=list) + + def describe(self, entry) -> str: + table, row_id, release_at = entry + return f"{table}:{row_id} embargoed until {release_at.isoformat()}" + + def by_table(self) -> dict: + counts: dict = {} + for table, _, _ in self.lifted: + counts[table] = counts.get(table, 0) + 1 + return counts + + +def _due_rows(session, model, on_date: date) -> list: + """Embargoed rows of one model whose date has arrived. + + The date test is narrowed in SQL so the job does not load every embargoed + row in the database, and then applied again by ``due_for_release`` on each + candidate. The rule lives in the domain function; the WHERE clause is an + optimisation that is allowed to be looser than it, never tighter. + """ + rows = session.execute( + select(model).where( + model.release_status == STATUS_EMBARGOED, + model.release_at.is_not(None), + model.release_at <= on_date, + ) + ).scalars() + return [ + row + for row in rows + if due_for_release(row.release_status, row.release_at, on_date) + ] + + +def lift_due_embargoes( + session, on_date: date = None, apply: bool = True +) -> ReleasePlan: + """Publish every record whose embargo has run out. Safe to run repeatedly. + + With ``apply=False`` nothing is written and the plan describes what would + be. Release is security state, so the CLI previews by default. + + Idempotent by construction: a lifted row is no longer ``embargoed``, so the + next run does not see it. + """ + on_date = on_date or date.today() + plan = ReleasePlan() + + for model in RELEASABLE_MODELS: + table = model.__tablename__ + for row in _due_rows(session, model, on_date): + plan.lifted.append((table, row.id, row.release_at)) + if not apply: + continue + + embargoed_until = row.release_at + row.release_status = STATUS_PUBLIC + # Cleared so the row stops advertising a schedule it has already + # kept, and so `validate_release` still holds for it afterwards: + # a release_at on a public row schedules nothing. + row.release_at = None + session.add( + AuthorizationAudit( + event_type=RELEASE_LIFTED, + actor=RELEASE_ACTOR, + subject_table=table, + subject_id=row.id, + detail={ + "release_at": embargoed_until.isoformat(), + "released_on": on_date.isoformat(), + "from_status": STATUS_EMBARGOED, + "to_status": STATUS_PUBLIC, + }, + ) + ) + + if apply and plan.lifted: + session.commit() + + return plan + + +# ============= EOF ============================================= diff --git a/tests/test_domain_release.py b/tests/test_domain_release.py new file mode 100644 index 00000000..3165d6da --- /dev/null +++ b/tests/test_domain_release.py @@ -0,0 +1,129 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""Embargo rules, exercised without a database.""" + +from datetime import date, timedelta + +import pytest + +from domain.release import ( + STATUS_EMBARGOED, + STATUS_PRIVATE, + STATUS_PUBLIC, + MissingReleaseDate, + UnscheduledReleaseDate, + due_for_release, + validate_release, +) + +TODAY = date(2026, 9, 1) +YESTERDAY = TODAY - timedelta(days=1) +TOMORROW = TODAY + timedelta(days=1) + + +# ------ when an embargo runs out ---------- + + +def test_an_embargo_lifts_on_its_release_date(): + """release_at is the day the record becomes public, not the last day it + is held. Off by one here publishes a day late, every time.""" + assert due_for_release(STATUS_EMBARGOED, TODAY, TODAY) is True + + +def test_an_embargo_is_still_holding_the_day_before(): + assert due_for_release(STATUS_EMBARGOED, TOMORROW, TODAY) is False + + +def test_a_missed_run_still_releases_afterwards(): + """The job does not have to run on the day. A date that passed while + nothing ran is still due the next time it does.""" + assert due_for_release(STATUS_EMBARGOED, YESTERDAY, TODAY) is True + + +def test_expiry_is_checked_at_use_not_swept(): + """The same row answers differently on different days, with no job run.""" + assert due_for_release(STATUS_EMBARGOED, TODAY, YESTERDAY) is False + assert due_for_release(STATUS_EMBARGOED, TODAY, TODAY) is True + + +# ------ what the release job refuses to touch ---------- + + +@pytest.mark.parametrize( + "status", [STATUS_PUBLIC, STATUS_PRIVATE, "draft", "archived", None] +) +def test_only_embargoed_records_are_released(status): + """A past date on any other level schedules nothing. Publishing a draft + because it happens to carry a date would be the job inventing a decision + nobody made.""" + assert due_for_release(status, YESTERDAY, TODAY) is False + + +def test_an_embargo_without_a_date_is_never_due(): + """validate_release makes this unwritable, but a row predating this + feature, or written by hand, must not crash the nightly job.""" + assert due_for_release(STATUS_EMBARGOED, None, TODAY) is False + + +# ------ validation, before a row is written ---------- + + +def test_an_embargo_names_its_release_date(): + with pytest.raises(MissingReleaseDate): + validate_release(STATUS_EMBARGOED, None) + + +def test_a_release_date_on_another_level_is_rejected(): + """It would schedule nothing, and read as a promise that something was.""" + with pytest.raises(UnscheduledReleaseDate): + validate_release("draft", TOMORROW) + + +def test_a_public_record_carries_no_release_date(): + with pytest.raises(UnscheduledReleaseDate): + validate_release(STATUS_PUBLIC, TOMORROW) + + +def test_an_embargo_with_a_date_is_valid(): + assert validate_release(STATUS_EMBARGOED, TOMORROW) is None + + +def test_a_record_with_neither_is_valid(): + assert validate_release("draft", None) is None + + +def test_a_release_date_in_the_past_is_accepted(): + """Backfilling an embargo that already expired is a legitimate thing to + do; the next run publishes it. Rejecting it would push the caller into + setting a fake future date.""" + assert validate_release(STATUS_EMBARGOED, YESTERDAY) is None + + +# ------ the domain constants and the lexicon must agree ---------- + + +def test_the_release_levels_exist_in_the_lexicon(): + """domain/release.py cannot import the lexicon without taking a database + dependency, so its constants are hand-copied -- the same drift that cost + domain/access.py every API-key grant. A rename in core/lexicon.json that + stranded the release job would otherwise be silent: nothing would ever be + due, and nothing would raise.""" + from core.enums import ReleaseStatus + + levels = {member.value for member in ReleaseStatus} + assert STATUS_EMBARGOED in levels + assert STATUS_PUBLIC in levels + assert STATUS_PRIVATE in levels diff --git a/tests/test_ogc_embargo.py b/tests/test_ogc_embargo.py new file mode 100644 index 00000000..62d63c19 --- /dev/null +++ b/tests/test_ogc_embargo.py @@ -0,0 +1,242 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""An embargoed record must not reach the public OGC relations. + +The whole feature is one claim -- "this measurement is not published until +that date" -- and this is where it is checked against the relations a +consumer actually reads, rather than against the rule in isolation. + +The four relations here are the public ones that read the observation chain +(migration b4c5d6e7f8a9). The chemistry collections are absent because they +read the legacy NMA_* tables, which carry no release columns at all; see +docs/data-embargo.md. +""" + +from datetime import date, timedelta + +import pytest +from sqlalchemy import delete, text + +from db.engine import session_ctx +from db.field import FieldActivity, FieldEvent +from db.location import Location, LocationThingAssociation +from db.observation import Observation +from db.sample import Sample +from db.thing import Thing +from services.release_schedule import lift_due_embargoes +from tests import get_parameter_id + +POINT_ID = "EMBARGO-TEST-1" +TOMORROW = date.today() + timedelta(days=1) + +# Every public relation that reads the observation chain, with the matviews +# flagged: those answer from their last refresh, so a test that changes data +# has to refresh them the way the nightly pg_cron job does. +WATER_LEVEL_RELATIONS = ( + ("ogc_water_well_summary", True), + ("ogc_water_elevation_wells", True), + ("ogc_depth_to_water_trend_wells", True), + ("ogc_latest_depth_to_water_wells", True), +) + + +def _refresh(session): + for relation, materialized in WATER_LEVEL_RELATIONS: + if materialized: + session.execute(text(f"REFRESH MATERIALIZED VIEW {relation}")) + session.commit() + + +def _relations_naming(session, thing_id) -> set: + """Which of the public relations currently publish this well.""" + present = set() + for relation, _materialized in WATER_LEVEL_RELATIONS: + found = session.execute( + text(f"SELECT 1 FROM {relation} WHERE id = :id LIMIT 1"), + {"id": thing_id}, + ).scalar() + if found: + present.add(relation) + return present + + +@pytest.fixture() +def well_with_one_water_level(): + """A public well whose only measurement is public, and its ids.""" + with session_ctx() as session: + location = Location( + point="POINT(-106.5 34.5)", + elevation=1600.0, + release_status="public", + ) + session.add(location) + session.flush() + + thing = Thing( + name=POINT_ID, + thing_type="water well", + release_status="public", + well_depth=100.0, + ) + session.add(thing) + session.flush() + + session.add( + LocationThingAssociation( + location_id=location.id, + thing_id=thing.id, + effective_start="2025-02-01T00:00:00Z", + ) + ) + + event = FieldEvent( + thing_id=thing.id, + event_date="2024-03-15T19:00:00Z", + release_status="public", + ) + session.add(event) + session.flush() + + activity = FieldActivity( + field_event_id=event.id, + activity_type="groundwater level", + release_status="public", + ) + session.add(activity) + session.flush() + + sample = Sample( + field_activity_id=activity.id, + sample_date="2024-03-15T19:00:00Z", + sample_name=f"{POINT_ID}-wl-1", + sample_matrix="water", + sample_method="Steel-tape measurement", + qc_type="Normal", + release_status="public", + ) + session.add(sample) + session.flush() + + observation = Observation( + sample_id=sample.id, + parameter_id=get_parameter_id("groundwater level", "Field Parameter"), + observation_datetime="2024-03-15T19:00:00Z", + value=50.0, + unit="ft", + measuring_point_height=2.5, + release_status="public", + ) + session.add(observation) + session.commit() + + ids = { + "thing": thing.id, + "location": location.id, + "observation": observation.id, + "sample": sample.id, + "activity": activity.id, + "event": event.id, + } + _refresh(session) + + yield ids + + session.execute(delete(Thing).where(Thing.id == ids["thing"])) + session.execute(delete(Location).where(Location.id == ids["location"])) + session.commit() + _refresh(session) + + +def test_a_public_measurement_reaches_every_water_level_relation( + well_with_one_water_level, +): + """The control. Without this, a test asserting absence proves nothing -- + the well could be missing for any of a dozen unrelated reasons.""" + with session_ctx() as session: + present = _relations_naming(session, well_with_one_water_level["thing"]) + assert present == {relation for relation, _ in WATER_LEVEL_RELATIONS} + + +@pytest.mark.parametrize( + "level, model", + [ + ("observation", Observation), + ("sample", Sample), + ("activity", FieldActivity), + ("event", FieldEvent), + ], +) +def test_an_embargo_anywhere_in_the_chain_withholds_the_measurement( + well_with_one_water_level, level, model +): + """Embargoing any link hides the measurement. A consumer cannot reach an + embargoed reading by way of a public parent.""" + with session_ctx() as session: + row = session.get(model, well_with_one_water_level[level]) + row.release_status = "embargoed" + row.release_at = TOMORROW + session.commit() + _refresh(session) + + assert _relations_naming(session, well_with_one_water_level["thing"]) == set() + + +def test_an_embargoed_well_is_absent_from_every_relation(well_with_one_water_level): + """Whole-site embargo needs no clause of its own: `release_status = + 'public'` already excludes it.""" + with session_ctx() as session: + thing = session.get(Thing, well_with_one_water_level["thing"]) + thing.release_status = "embargoed" + thing.release_at = TOMORROW + session.commit() + _refresh(session) + + assert _relations_naming(session, well_with_one_water_level["thing"]) == set() + + +def test_a_record_returns_when_its_embargo_is_lifted(well_with_one_water_level): + """End to end: embargo, hidden; the date arrives, the job runs, the + refresh follows, and the measurement is published.""" + with session_ctx() as session: + observation = session.get(Observation, well_with_one_water_level["observation"]) + observation.release_status = "embargoed" + observation.release_at = date.today() + session.commit() + _refresh(session) + assert _relations_naming(session, well_with_one_water_level["thing"]) == set() + + lift_due_embargoes(session) + _refresh(session) + + assert _relations_naming(session, well_with_one_water_level["thing"]) == { + relation for relation, _ in WATER_LEVEL_RELATIONS + } + + +def test_a_draft_measurement_is_still_published(well_with_one_water_level): + """The migration adds an embargo clause, not a release-policy change. + Observations sitting at `draft` reached these four relations before it and + still do -- tightening that is a separate decision with its own row + counts (see migration b4c5d6e7f8a9).""" + with session_ctx() as session: + observation = session.get(Observation, well_with_one_water_level["observation"]) + observation.release_status = "draft" + session.commit() + _refresh(session) + + assert _relations_naming(session, well_with_one_water_level["thing"]) == { + relation for relation, _ in WATER_LEVEL_RELATIONS + } diff --git a/tests/test_release_schedule.py b/tests/test_release_schedule.py new file mode 100644 index 00000000..fdaed834 --- /dev/null +++ b/tests/test_release_schedule.py @@ -0,0 +1,230 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""The scheduled release of embargoed records.""" + +from datetime import date, timedelta + +import pytest +from sqlalchemy import delete, select +from sqlalchemy.exc import DBAPIError + +from db.authorization_audit import RELEASE_LIFTED, AuthorizationAudit +from db.engine import session_ctx +from db.location import Location +from services.release_schedule import RELEASE_ACTOR, lift_due_embargoes + +TODAY = date.today() +YESTERDAY = TODAY - timedelta(days=1) +TOMORROW = TODAY + timedelta(days=1) + + +@pytest.fixture() +def embargoed_locations(): + """Three locations: due yesterday, due today, still held.""" + created = {} + with session_ctx() as session: + for label, release_at in ( + ("overdue", YESTERDAY), + ("due", TODAY), + ("held", TOMORROW), + ): + location = Location( + point="POINT(-106.0 34.0)", + elevation=1500.0, + release_status="embargoed", + release_at=release_at, + ) + session.add(location) + session.commit() + session.refresh(location) + created[label] = location.id + + yield created + + session.execute( + delete(AuthorizationAudit).where( + AuthorizationAudit.subject_table == Location.__tablename__, + AuthorizationAudit.subject_id.in_(created.values()), + ) + ) + session.execute(delete(Location).where(Location.id.in_(created.values()))) + session.commit() + + +def _status(session, location_id): + location = session.get(Location, location_id) + return location.release_status, location.release_at + + +def test_a_record_is_released_on_its_date(embargoed_locations): + with session_ctx() as session: + lift_due_embargoes(session) + assert _status(session, embargoed_locations["due"]) == ("public", None) + + +def test_a_record_whose_date_passed_unnoticed_is_still_released(embargoed_locations): + """A day the job did not run does not strand the record.""" + with session_ctx() as session: + lift_due_embargoes(session) + assert _status(session, embargoed_locations["overdue"]) == ("public", None) + + +def test_a_record_still_under_embargo_is_left_alone(embargoed_locations): + with session_ctx() as session: + lift_due_embargoes(session) + assert _status(session, embargoed_locations["held"]) == ( + "embargoed", + TOMORROW, + ) + + +def test_the_release_date_is_cleared_on_release(embargoed_locations): + """A public row carrying a release_at would advertise a schedule it has + already kept, and would fail validate_release if it were re-checked.""" + with session_ctx() as session: + lift_due_embargoes(session) + _, release_at = _status(session, embargoed_locations["due"]) + assert release_at is None + + +def test_a_preview_writes_nothing(embargoed_locations): + with session_ctx() as session: + plan = lift_due_embargoes(session, apply=False) + assert len(plan.lifted) == 2 + assert _status(session, embargoed_locations["due"]) == ("embargoed", TODAY) + assert _status(session, embargoed_locations["overdue"]) == ( + "embargoed", + YESTERDAY, + ) + + +def test_running_twice_releases_nothing_the_second_time(embargoed_locations): + with session_ctx() as session: + first = lift_due_embargoes(session) + second = lift_due_embargoes(session) + assert len(first.lifted) == 2 + assert second.lifted == [] + + +def test_every_release_lands_in_the_authorization_log(embargoed_locations): + """The question after an incident is who published this, and when.""" + with session_ctx() as session: + lift_due_embargoes(session) + rows = ( + session.execute( + select(AuthorizationAudit).where( + AuthorizationAudit.subject_table == Location.__tablename__, + AuthorizationAudit.subject_id.in_(embargoed_locations.values()), + ) + ) + .scalars() + .all() + ) + + assert len(rows) == 2 + assert {row.event_type for row in rows} == {RELEASE_LIFTED} + assert {row.actor for row in rows} == {RELEASE_ACTOR} + + by_subject = {row.subject_id: row.detail for row in rows} + # The date the embargo was set for is recorded next to the date it was + # actually lifted, so "was this released early" stays answerable. + assert by_subject[embargoed_locations["due"]]["release_at"] == TODAY.isoformat() + assert ( + by_subject[embargoed_locations["overdue"]]["release_at"] + == YESTERDAY.isoformat() + ) + assert by_subject[embargoed_locations["due"]]["released_on"] == TODAY.isoformat() + + +def test_an_earlier_run_date_releases_less(embargoed_locations): + """on_date is a parameter so the job can be reasoned about, and so a + backfill can be replayed as of a past date.""" + with session_ctx() as session: + plan = lift_due_embargoes(session, on_date=YESTERDAY, apply=False) + assert len(plan.lifted) == 1 + assert plan.lifted[0][1] == embargoed_locations["overdue"] + + +def test_a_record_at_another_level_is_left_where_it_is(): + """Only 'embargoed' is scheduled. The job walks past everything else -- + a private record is not published because the job happened to look at it. + + It carries no release_at: the CHECK constraint below makes that pairing + unwritable, which is a stronger guarantee than this test could assert. + """ + with session_ctx() as session: + location = Location( + point="POINT(-106.1 34.1)", + elevation=1500.0, + release_status="private", + ) + session.add(location) + session.commit() + session.refresh(location) + location_id = location.id + + try: + lift_due_embargoes(session) + assert _status(session, location_id) == ("private", None) + finally: + session.execute(delete(Location).where(Location.id == location_id)) + session.commit() + + +# ------ the table refuses what the schema layer cannot see ---------- + + +def test_the_table_rejects_an_embargo_with_no_date(): + """A PATCH body carrying only release_status is a fragment, so the pair is + guarded by a CHECK constraint rather than by pydantic. This is that + constraint, reached the way the CLI and the transfers reach it.""" + with session_ctx() as session: + session.add( + Location( + point="POINT(-106.2 34.2)", + elevation=1500.0, + release_status="embargoed", + release_at=None, + ) + ) + # pg8000 maps a CHECK violation (SQLSTATE 23514) to ProgrammingError + # rather than IntegrityError, so match on the parent and the name. + with pytest.raises(DBAPIError, match="location_embargo_needs_date"): + session.commit() + session.rollback() + + +def test_the_table_rejects_a_release_date_without_an_embargo(): + with session_ctx() as session: + session.add( + Location( + point="POINT(-106.3 34.3)", + elevation=1500.0, + release_status="draft", + release_at=TOMORROW, + ) + ) + with pytest.raises(DBAPIError, match="location_embargo_needs_date"): + session.commit() + session.rollback() + + +def test_the_released_row_satisfies_the_constraint(embargoed_locations): + """The job clears release_at as it publishes; if it did not, its own + UPDATE would violate the constraint it just satisfied.""" + with session_ctx() as session: + lift_due_embargoes(session) + assert _status(session, embargoed_locations["due"]) == ("public", None) diff --git a/tests/test_thing.py b/tests/test_thing.py index 3e1ebb3d..025cc0b0 100644 --- a/tests/test_thing.py +++ b/tests/test_thing.py @@ -928,6 +928,7 @@ def test_get_water_wells_includes_contact_summary( "created_at": contact.created_at.astimezone(timezone.utc).strftime(DT_FMT), "release_status": contact.release_status, "data_maturity": contact.data_maturity, + "release_at": contact.release_at, "name": contact.name, "organization": contact.organization, "contact_type": contact.contact_type,