diff --git a/.gitignore b/.gitignore index eb6f7c340..218ff919b 100644 --- a/.gitignore +++ b/.gitignore @@ -56,3 +56,6 @@ app.yaml # Claude Code .claude/settings.local.json .claude/worktrees/ + +# Design canvas scratch (artboards, decks, generated PDFs) +.design/ diff --git a/ADR5.md b/ADR5.md new file mode 100644 index 000000000..536399163 --- /dev/null +++ b/ADR5.md @@ -0,0 +1,246 @@ +# ADR5: One Enforcement Engine, Two Grant Tables + +## Status + +Proposed. Decides PUB-D14 / PERM-D3, the engine-count fork named in the +whitepaper *Who May See What, and Who May Do What* (Confluence, Ocotillo space). +Nothing is built yet. This record exists so the grant table and the visibility +layer are not designed twice. + +## Summary + +Ocotillo needs to answer one question everywhere: *may this principal exercise +this capability within this scope?* The whitepaper argues that publication +consent (a landowner's agreement about their well) and internal permission (an +institution's trust in a person) reduce to the same grammar, and then leaves +open whether they should therefore share one policy engine. + +**Decision: two tables, one engine.** Landowner publication consent and internal +permission grants are stored separately, with separate governance, separate +admin surfaces, and separate audit semantics. Both are evaluated by a single +visibility layer and projected through a single field-allowlist chokepoint. No +service answers "is this published" on its own. + +The split is about *who decides and on what authority*. The join is about *where +the answer is computed*. Those are different questions, and the current codebase +already shows what happens when the second one is answered "in each view." + +## Context + +### Enforcement is currently distributed, and it has already drifted + +Publication today is a `release_status` column plus a filter re-written into +every consuming view. There are twelve of them — six public collections +(`ogc_locations`, `ogc_waterlevels`, `ogc_water_chemistry`, +`ogc_actively_monitored_wells`, `ogc_latest_tds_wells`, `ogc_project_areas`) and +six internal mirrors that deliberately carry non-public rows for authenticated +staff GIS clients. Each carries its own copy of the rule in SQL. + +Migration `baba91fe5e83` is the drift, in the repository, with a date on it: +`ogc_waterlevels` filtered on the reading's own `release_status` and not the +parent well's, so a well marked `draft` or `private` still published its public +readings through OGC API - EDR — with the well's name and coordinates attached. +`ogc_water_chemistry` had required the parent thing to be public since +`d9e0f1a2b3c4`. Two views, two answers to the same question, and the difference +was invisible until someone read both. + +That is not a bug in a view. It is what distributed enforcement does over time, +and every new destination multiplies it. The engine-count decision is really a +decision about whether that class of defect stays possible. + +### Publication is currently a bulk act, not a consent record + +`data_migrations/migrations/20260714_0001_publish_project_areas.py` publishes by +setting `release_status='public'` across a selection. That is the coarse +mechanism the whitepaper describes: association publishes everything about a +record, and an owner who would share water levels but not chemistry cannot be +represented at all. + +### The grant shape already exists twice + +`db/permission.py`'s `Permission` was landowner field-access consent — +`contact_id`, `allow_sampling`, `allow_installation`, an optional date range, a +polymorphic target (`permissible_id` / `permissible_type`). It was also dead: +unregistered in `db/__init__.py`, with no table in any migration and no caller. +It has been deleted, along with `PermissionMixin`. `PermissionHistory` +generalizes the same idea into a typed, lexicon-controlled `permission_type` +with `permission_allowed`, a required `start_date`, an optional `end_date`, and +`target_id` / `target_table`; it is now `FieldAccessConsent`, table +`field_access_consent`. Neither decides what an API caller may see. + +Both are consent about physical site access. Neither is authorization. The shape +recurring independently is evidence the shape is right; it is not an argument +for merging those tables into the access-control model. + +### Principals are already more than users + +`/ogcapi-internal` is gated at the ASGI layer by `core/internal_ogc_auth.py`, +which accepts an Authentik JWT carrying `OGCInternal` **or** a static API key, +because ArcGIS Pro cannot present a bearer token. That key is a principal with +no user behind it, and today its scope is "everything the internal mount +serves." Any model that assumes principal == person is already wrong here. + +### Two axes in one column + +San Acacia data is public and provisional at once; `automated_ingestion/ +ocotillo/loader.py` says so in a comment and works around it. `ReleaseMixin` +gave one lexicon column defaulting to `draft`, and that lexicon lists `draft`, +`public`, `private`, `published`, and `archived` — visibility — as siblings of +`provisional` and `final` — review state. Two axes, one column, so a record +could hold only one of the two answers. + +`ReleaseMixin` now carries `data_maturity` alongside `release_status`. This is +not a new vocabulary: the lexicon category (`provisional`, `in review`, +`approved`, described in `core/lexicon.json` as "orthogonal to release_status") +and the column already existed on `transducer_observation`, and the split +generalizes them rather than inventing a parallel flag. Grants will need the +second axis, because some destinations want approved data only. + +## Decision + +### 1. Two tables + +**`permission_grant`** — internal authorization. Principal (user subject, role, +API key), capability (read, enter, correct, administer), scope (project, thing, +data type, field group), time bounds, `granted_by`, `granted_at`, `reason`. +Governed by data services staff. Answers "is this person trusted with this." + +**`publication_consent`** — landowner-facing publication. One row per +(thing, destination, data type), against a destination registry that holds the +anonymous public, NGWMN, and partner agencies. Carries the consenting contact, +the date it was recorded, and who recorded it. Governed by the data owner and by +whoever took the phone call. Answers "did the owner agree to this." + +Neither table gets a NULL-as-wildcard data type. A grant names its types, so a +data type added next year is never published by an existing row. + +### 2. One engine + +A single visibility layer resolves both tables into one answer per request, and +a single serialization chokepoint applies the per-audience field allowlist and +its transformation hooks (coordinate rounding is the known case). The +never-public field list is enforced there and nowhere else. + +Grants are read from the database at request time. They are never encoded into +token claims, and expiry is checked at use, because immediate revocation is a +promise made to landowners and a claim baked into a token outlives it. + +### 3. Both tables feed one append-only authorization audit log + +Grant, revocation, consent capture, publication-configuration change, membership +change. The log is shared even though the tables are not, because the question +after an incident — "who granted that, and when" — does not care which table the +row came from. `AuditMixin` and sqlalchemy-continuum cover data attribution and +data history; this is a separate structure, written from the application, with a +database-level backstop for writes that bypass it. + +## Why not the alternatives + +**One table.** Cheapest to enforce, and the enforcement argument is already won +by the shared engine, so the merge buys little. What it costs is governance: a +landowner's consent and a staff member's clearance would live in one table, one +admin screen, and one review path, decided by different people for different +reasons and with different consequences for being wrong. Revoking consent is a +phone call honored immediately; revoking clearance is an HR-shaped event. The +whitepaper's caution — that consent authoring and permission granting need +visible separation even if they share a surface — is easier to keep with a +schema that separates them than with a column that distinguishes them. + +**Two engines.** Cleanest governance separation, and it reintroduces exactly the +`baba91fe5e83` failure: two independent implementations of "is this published," +diverging quietly, with an outside party finding the difference first. Ocotillo +has one read path per record; it should have one answer. + +## What this does not decide + +- **Data type granularity** (PUB-U11). Water levels versus chemistry is + required. Per-analyte grants may be over-engineering. The admin screen decides + this, not the schema. +- **Whether field sensitivity is global or per destination**, and whether + protection is removal only or also transformation (PUB-U7 / PUB-U8). The + chokepoint supports both; which is configured is open. +- **What happens to rows already marked `release_status='provisional'`.** The + two axes now exist, but no data moves between them. Rewriting those rows as + a level plus `data_maturity='provisional'` needs someone to say what level + each of them meant, and data migrations have no CD path in this repo — they + are run by hand. +- **Whether `provisional` and `final` should leave the `release_status` + lexicon.** Removing them is what makes the split enforceable rather than + conventional, and it cannot happen before the row migration above. +- **Where coarse group membership lives**, Authentik or Ocotillo (PERM-D9). + Authentik roles become role principals with broad grants either way, and + nobody's access changes on the day the tables land. +- **What each audit event records** (PERM-U11 / PERM-U13). Before-and-after + capture forces an event-based write path; whether that rigidity is required + depends on a compliance driver nobody has yet identified. +- **Who owns the never-public field list.** A policy decision with a named + owner, and a prerequisite to anything shipping externally. Engineering + guarantees the list is enforced; it cannot decide what is on it. +- **Healy migration** (PUB-D13). Grandfathering wells as full consent preserves + today's behavior; re-consenting per data type honors the model. The data owner + chooses, explicitly, and it happens last. + +## Consequences + +**Good.** New destinations become a registry row plus grants, not a new pipeline +with its own copy of the rules. A field nobody approved is invisible outside the +Bureau by default, including fields added later. The correctness burden +concentrates in one layer that can be reviewed, tested, and monitored, instead +of in the next twelve views. A staff member can tell a well owner what is shared, +per kind of data, and revoke it from a screen. + +**Cost.** Two tables to keep coherent where one would do, and a shared engine +that must not leak one table's governance into the other's admin surface. The +visibility layer sits on the read path of every service, so its query shape +matters: grants are few and cacheable per principal, but an unbounded cache +silently defeats the immediate-revocation promise. Cache with short TTLs and +explicit invalidation on revoke. + +**Honesty requirement.** For a harvesting destination, revoking consent means +the data stops being offered. Copies already harvested live in someone else's +system. "Unpublish" means "stop offering," and that is what owners should be +told. + +## Sequencing implied by this decision + +1. This record (here). +2. **Done.** `permission_history` is `field_access_consent`, dead `Permission` + and `PermissionMixin` are gone, and `ReleaseMixin` carries `release_status` + (level) plus `data_maturity` (review state). +3. **Done.** `permission_grant`, `publication_consent`, `destination` and + `authorization_audit` exist; `services/visibility.py` is the single + evaluator; `api/access.py` is its one tenant. No existing endpoint routes + through it yet. +4. **Done.** `services/field_projection.py` is the chokepoint, + `core/field-allowlists.yml` holds the per-audience allowlists and the + never-public list, and coordinate rounding is the first transform. Both + publication paths go through it: destination payloads via + `services/visibility.py`, and all 27 public OGC collections via + `core/feature_provider.py`, which turns the allowlist into the provider's + `properties` so unlisted columns are never selected. Closing that gap + found eleven collections publishing `nma_pk_welldata` and one publishing + `entered_by`. +5. **Per environment, by hand:** `oco seed-access-grants` writes the day-one + role baseline (ADR5, 5.2) -- one global grant per Authentik role, + capability and data type, so nobody's access changes when the layer starts + being consulted. Until it runs, that environment denies everyone. + Idempotent, and it does not resurrect a revoked seeded grant. +6. Console administration. +7. Healy migration, after the data owner decides grandfathering. + +## References + +- Confluence, Ocotillo space: *Who May See What, and Who May Do What: A + scope-based, attribute-level access control system for Ocotillo* (whitepaper + this record answers), *Ocotillo Needs an Operator Console*, *Authentik + Access-Control Matrix Summary and Role Definitions*, *User Research Outcomes: + Ocotillo Permissions Interview with Ethan Mamer* +- [ADR3](ADR3.md): OGC API - EDR collections backed by publication-filtered views +- [ADR4](ADR4.md): the `domain/` layer the grant-evaluation rules belong in +- [db/field_access_consent.py](db/field_access_consent.py): landowner + field-access consent, and the in-house precedent for the grant shape +- [db/base.py](db/base.py): `ReleaseMixin`, `AuditMixin`, versioning wiring +- [alembic/versions/baba91fe5e83_gate_ogc_waterlevels_on_thing_release.py](alembic/versions/baba91fe5e83_gate_ogc_waterlevels_on_thing_release.py): + the drift this decision is meant to make impossible +- [core/internal_ogc_auth.py](core/internal_ogc_auth.py) and + [docs/internal-ogc-desktop-gis.md](docs/internal-ogc-desktop-gis.md): the API-key principal diff --git a/CLAUDE.md b/CLAUDE.md index a5f9358c7..c8f6a36f5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -215,6 +215,75 @@ the pinned pygeoapi version — most sharply, `BaseProvider.fields` returns **`docs/ogc-field-descriptions.md`** before changing field metadata or upgrading pygeoapi. +### Access control and release state + +**`ADR5.md`** decides the shape of the access-control work: two grant tables +(internal permission vs landowner publication consent), one visibility layer, +one field-projection chokepoint. + +The storage and the evaluator exist; the field projection does not. + +- **`services/visibility.py` is the only evaluator.** `may()` answers internal + authorization, `published_things()` answers "what does this destination + get". It loads rows and calls `domain/access.py`, which holds the rules and + touches no database. Do not filter by grant or consent anywhere else -- + migration `baba91fe5e83` is what distributed filtering already cost. +- **`api/access.py` (`/access`) is its only tenant.** Grants, destinations, + consent, and a `/access/decision` introspection route. No pre-existing + endpoint consults the layer yet, so release_status still governs what the + OGC views publish. The prefix is `/access`, not `/publication`, because + `api/publication.py` is the bibliography. +- **Fields are published by allowlist, per audience.** + `core/field-allowlists.yml` says what each audience receives; + `services/field_projection.py` applies it below the routes, so a new route + cannot skip it. An audience with no entry gets an empty record, and the + `never_public` block overrides every allowlist. Protection includes + transformation -- public coordinates are rounded, not withheld. The public + OGC collections go through it too: `core/feature_provider.py` turns each + collection's allowlist into pygeoapi's `properties`, so an unlisted column + is never selected, and a collection with no entry publishes nothing. The + internal mount is outside it. Read + **`docs/access-field-projection.md`** before touching the allowlists. +- **The role baseline is seeded by hand, per environment.** + `oco seed-access-grants` writes one global grant per (Authentik role, + capability, data type) so today's roles keep today's access; it previews by + default and needs `--apply` to write. Idempotent, and it will not resurrect + a seeded grant somebody revoked, because narrowing the baseline is the point. + Until it is run in an environment, `/access/decision` denies everyone there. +- **Default deny, no wildcards, expiry at use.** A grant with no matching row + is a no; a grant names its `data_type` (there is no term meaning "all"); and + nothing sweeps expired rows, so every check compares against the date asked + about. `services/access_admin.py` writes an `authorization_audit` row in the + same transaction as every change. + +- **Embargo is a scheduled level change, not a read-path rule.** + `release_status = 'embargoed'` plus a `release_at` date withholds a record; + `oco release-embargoed --apply` flips it to `public` when the date arrives, + and must run before the 09:00 UTC materialized-view refresh. Nothing on the + read path reads `release_at` -- seven public collections are materialized + views where `current_date` is frozen at refresh time, so a date predicate + would buy nothing there. Embargo only ever *widens* visibility; withdrawing + something published is an immediate `release_status` change. The public + water-level views exclude embargoed rows with `IS DISTINCT FROM 'embargoed'` + at every level of the observation chain -- deliberately not `= 'public'`, + which would also drop draft and NULL rows. Chemistry cannot be embargoed per + record: those collections read the legacy `NMA_*` tables, which have no + release columns. Read **`docs/data-embargo.md`** before changing any of it. + +Two vocabulary fixes from it have landed and matter when reading models: + +- **`db/field_access_consent.py`** (`FieldAccessConsent`, table + `field_access_consent`, formerly `PermissionHistory` / `permission_history`) + is a landowner's consent to *physical site access*. It is not authorization. + Its `permission_type` / `permission_allowed` columns keep their names, and + Thing responses still publish it under the `permissions` key. +- **`ReleaseMixin` has two axes**: `release_status` is the release *level* + (who may see it) and `data_maturity` is the review state (`provisional`, + `in review`, `approved`, NULL = not stated). The `release_status` lexicon + still lists `provisional` and `final` for historical rows; new code should + put review state in `data_maturity`. A third column, `release_at`, is + the embargo date and is NULL for every record that is not embargoed. + ### Database Configuration The application supports two database modes (configured via `DB_DRIVER` in `.env`): @@ -341,3 +410,70 @@ transferers directly. - **OGC API**: `http://localhost:8000/ogcapi` for OGC API - Features endpoints - **CLI**: `oco --help` for Ocotillo CLI commands - **Sentry**: Error tracking and performance monitoring integrated + +## Working Guidelines + +Behavioral guidelines to reduce common LLM coding mistakes. These bias toward +caution over speed; for trivial tasks, use judgment. + +### 1. Think Before Coding + +**Don't assume. Don't hide confusion. Surface tradeoffs.** + +Before implementing: +- State your assumptions explicitly. If uncertain, ask. +- If multiple interpretations exist, present them - don't pick silently. +- If a simpler approach exists, say so. Push back when warranted. +- If something is unclear, stop. Name what's confusing. Ask. + +### 2. Simplicity First + +**Minimum code that solves the problem. Nothing speculative.** + +- No features beyond what was asked. +- No abstractions for single-use code. +- No "flexibility" or "configurability" that wasn't requested. +- No error handling for impossible scenarios. +- If you write 200 lines and it could be 50, rewrite it. + +Ask yourself: "Would a senior engineer say this is overcomplicated?" If yes, simplify. + +### 3. Surgical Changes + +**Touch only what you must. Clean up only your own mess.** + +When editing existing code: +- Don't "improve" adjacent code, comments, or formatting. +- Don't refactor things that aren't broken. +- Match existing style, even if you'd do it differently. +- If you notice unrelated dead code, mention it - don't delete it. + +When your changes create orphans: +- Remove imports/variables/functions that YOUR changes made unused. +- Don't remove pre-existing dead code unless asked. + +The test: Every changed line should trace directly to the user's request. + +### 4. Goal-Driven Execution + +**Define success criteria. Loop until verified.** + +Transform tasks into verifiable goals: +- "Add validation" -> "Write tests for invalid inputs, then make them pass" +- "Fix the bug" -> "Write a test that reproduces it, then make it pass" +- "Refactor X" -> "Ensure tests pass before and after" + +For multi-step tasks, state a brief plan: + +``` +1. [Step] -> verify: [check] +2. [Step] -> verify: [check] +3. [Step] -> verify: [check] +``` + +Strong success criteria let you loop independently. Weak criteria ("make it +work") require constant clarification. + +**These guidelines are working if:** fewer unnecessary changes in diffs, fewer +rewrites due to overcomplication, and clarifying questions come before +implementation rather than after mistakes. diff --git a/alembic/versions/79a3ab24627e_add_access_control_tables.py b/alembic/versions/79a3ab24627e_add_access_control_tables.py new file mode 100644 index 000000000..1787b1e4b --- /dev/null +++ b/alembic/versions/79a3ab24627e_add_access_control_tables.py @@ -0,0 +1,183 @@ +"""add the access-control tables + +The storage half of ADR5: two grant tables, one destination registry, one +append-only authorization audit log. Nothing reads them yet except +services/visibility.py and the /publication routes; no existing endpoint +changes behavior. + +* ``destination`` -- where published data is offered: the public web, a + harvester, a partner agency. +* ``permission_grant`` -- internal authorization, principal x capability x + scope, one row per data type. No NULL-as-wildcard: a grant names its type. +* ``publication_consent`` -- what a landowner agreed to publish about their + well, per destination and data type. One live row per combination; revoked + rows stay, which is why the unique index is partial. +* ``authorization_audit`` -- every grant, revocation and consent event, append + only, so "who granted that, and when" has an answer. + +The vocabularies (principal_type, capability, grant_scope_type, +access_data_type, destination_kind) are lexicon categories seeded from +core/lexicon.json by init_lexicon, not enum types, so adding a destination +kind is not a migration. + +Revision ID: 79a3ab24627e +Revises: e7c1a9f4b2d8 +Create Date: 2026-08-24 22:09:54.479541 + +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "79a3ab24627e" +down_revision: Union[str, Sequence[str], None] = "e7c1a9f4b2d8" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "authorization_audit", + sa.Column("event_type", sa.String(length=50), nullable=False), + sa.Column("actor", sa.String(length=255), nullable=False), + sa.Column("subject_table", sa.String(length=50), nullable=False), + sa.Column("subject_id", sa.Integer(), nullable=True), + sa.Column("detail", sa.JSON(), nullable=True), + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("timezone('UTC', now())"), + nullable=False, + ), + sa.Column("created_by_name", sa.String(length=255), nullable=True), + sa.Column("created_by_id", sa.String(length=255), nullable=True), + sa.Column("updated_by_name", sa.String(length=255), nullable=True), + sa.Column("updated_by_id", sa.String(length=255), nullable=True), + sa.PrimaryKeyConstraint("id"), + ) + op.create_table( + "destination", + sa.Column("slug", sa.String(length=50), nullable=False), + sa.Column("name", sa.String(length=255), nullable=False), + sa.Column("destination_kind", sa.String(length=100), nullable=False), + sa.Column("description", sa.Text(), nullable=True), + sa.Column("active", sa.Boolean(), nullable=False), + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("timezone('UTC', now())"), + nullable=False, + ), + sa.Column("created_by_name", sa.String(length=255), nullable=True), + sa.Column("created_by_id", sa.String(length=255), nullable=True), + sa.Column("updated_by_name", sa.String(length=255), nullable=True), + sa.Column("updated_by_id", sa.String(length=255), nullable=True), + sa.ForeignKeyConstraint( + ["destination_kind"], ["lexicon_term.term"], onupdate="CASCADE" + ), + sa.PrimaryKeyConstraint("id"), + sa.UniqueConstraint("slug"), + ) + op.create_table( + "permission_grant", + sa.Column("principal_type", sa.String(length=100), nullable=False), + sa.Column("principal_id", sa.String(length=255), nullable=False), + sa.Column("capability", sa.String(length=100), nullable=False), + sa.Column("scope_type", sa.String(length=100), nullable=False), + sa.Column("scope_id", sa.Integer(), nullable=True), + sa.Column("data_type", sa.String(length=100), nullable=False), + sa.Column("starts_at", sa.Date(), nullable=False), + sa.Column("ends_at", sa.Date(), nullable=True), + sa.Column("granted_by", sa.String(length=255), nullable=False), + sa.Column("reason", sa.Text(), nullable=True), + sa.Column("revoked_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("revoked_by", sa.String(length=255), nullable=True), + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("timezone('UTC', now())"), + nullable=False, + ), + sa.Column("created_by_name", sa.String(length=255), nullable=True), + sa.Column("created_by_id", sa.String(length=255), nullable=True), + sa.Column("updated_by_name", sa.String(length=255), nullable=True), + sa.Column("updated_by_id", sa.String(length=255), nullable=True), + sa.ForeignKeyConstraint( + ["capability"], ["lexicon_term.term"], onupdate="CASCADE" + ), + sa.ForeignKeyConstraint( + ["data_type"], ["lexicon_term.term"], onupdate="CASCADE" + ), + sa.ForeignKeyConstraint( + ["principal_type"], ["lexicon_term.term"], onupdate="CASCADE" + ), + sa.ForeignKeyConstraint( + ["scope_type"], ["lexicon_term.term"], onupdate="CASCADE" + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + "ix_permission_grant_principal", + "permission_grant", + ["principal_type", "principal_id", "capability"], + unique=False, + ) + op.create_table( + "publication_consent", + sa.Column("thing_id", sa.Integer(), nullable=False), + sa.Column("destination_id", sa.Integer(), nullable=False), + sa.Column("data_type", sa.String(length=100), nullable=False), + sa.Column("contact_id", sa.Integer(), nullable=True), + sa.Column("recorded_by", sa.String(length=255), nullable=False), + sa.Column("notes", sa.Text(), nullable=True), + sa.Column("starts_at", sa.Date(), nullable=False), + sa.Column("ends_at", sa.Date(), nullable=True), + sa.Column("revoked_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("revoked_by", sa.String(length=255), nullable=True), + sa.Column("id", sa.Integer(), autoincrement=True, nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("timezone('UTC', now())"), + nullable=False, + ), + sa.Column("created_by_name", sa.String(length=255), nullable=True), + sa.Column("created_by_id", sa.String(length=255), nullable=True), + sa.Column("updated_by_name", sa.String(length=255), nullable=True), + sa.Column("updated_by_id", sa.String(length=255), nullable=True), + sa.ForeignKeyConstraint( + ["contact_id"], + ["contact.id"], + ), + sa.ForeignKeyConstraint( + ["data_type"], ["lexicon_term.term"], onupdate="CASCADE" + ), + sa.ForeignKeyConstraint( + ["destination_id"], + ["destination.id"], + ), + sa.ForeignKeyConstraint(["thing_id"], ["thing.id"], ondelete="CASCADE"), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + "uq_publication_consent_live", + "publication_consent", + ["thing_id", "destination_id", "data_type"], + unique=True, + postgresql_where=sa.text("revoked_at IS NULL"), + ) + + +def downgrade() -> None: + op.drop_index("uq_publication_consent_live", table_name="publication_consent") + op.drop_table("publication_consent") + op.drop_index("ix_permission_grant_principal", table_name="permission_grant") + op.drop_table("permission_grant") + op.drop_table("destination") + op.drop_table("authorization_audit") diff --git a/alembic/versions/a3b4c5d6e7f8_add_release_at_for_embargoes.py b/alembic/versions/a3b4c5d6e7f8_add_release_at_for_embargoes.py new file mode 100644 index 000000000..91c452e10 --- /dev/null +++ b/alembic/versions/a3b4c5d6e7f8_add_release_at_for_embargoes.py @@ -0,0 +1,138 @@ +"""add release_at, the date an embargoed record becomes public + +An embargo is a record withheld from public release until a date decided in +advance: `release_status = 'embargoed'` plus a `release_at`. This adds the +second half of that pair to every ReleaseMixin table. + +`release_at` is intent, not enforcement. Nothing on the read path consults it: +`services/release_schedule.py` flips `release_status` to `public` when the date +arrives, and `release_status` stays the only column the OGC views filter on. +The reason is in that module -- seven of the public collections are +materialized views refreshed nightly, where a date predicate would be frozen +at refresh time and buy nothing. + +Nullable, defaulting to NULL, meaning "no embargo". That default is +load-bearing rather than incidental: migration w1x2y3z4a5b6 records three +NGWMN exports emptied outright by a release predicate whose column defaulted +to something other than "released". No existing row changes meaning, and no +row is embargoed until somebody says so. + +Unlike e7c1a9f4b2d8's data_maturity, `transducer_observation` is included -- +it has no head start on this column. + +Revision ID: a3b4c5d6e7f8 +Revises: 79a3ab24627e +Create Date: 2026-09-01 00:00:00.000000 + +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "a3b4c5d6e7f8" +down_revision: Union[str, Sequence[str], None] = "79a3ab24627e" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +# Every table backed by a model that mixes in ReleaseMixin. +RELEASE_TABLES = ( + "address", + "analysis_method", + "aquifer_system", + "aquifer_type", + "asset", + "contact", + "data_provenance", + "deployment", + "email", + "field_access_consent", + "field_activity", + "field_event", + "field_event_participant", + "geologic_formation", + "group", + "location", + "measuring_point_history", + "monitoring_frequency_history", + "notes", + "observation", + "parameter", + "phone", + "regulatory_limit", + "sample", + "sensor", + "status_history", + "thing", + "thing_aquifer_association", + "thing_geologic_formation_association", + "thing_id_link", + "transducer_observation", + "transducer_observation_block", + "well_casing_material", + "well_purpose", + "well_screen", +) + +# sqlalchemy-continuum mirrors every tracked column into the version table, so +# the models carrying __versioned__ need the new column there too. +VERSION_TABLES = ( + "aquifer_system_version", + "geologic_formation_version", + "location_version", + "observation_version", + "parameter_version", + "regulatory_limit_version", + "thing_version", +) + +COMMENT = ( + "Date an embargoed record becomes public. NULL means no embargo. " + "Read by services/release_schedule.py, never by the read path." +) + +# An embargo names its date, and a date means an embargo. Enforced in the +# table because the schema layer cannot: a PATCH body carrying only +# `release_status` is a fragment, not a row, and the CLI, the transfers and +# psql do not go through pydantic at all. Every existing row satisfies it -- +# release_at starts NULL everywhere and nothing is embargoed yet -- so the +# constraint validates without a scan finding anything. +# +# Not applied to the _version tables: those record history, and a constraint +# on what a past state may have been would be a claim nobody checked. +CHECK_NAME = "{table}_embargo_needs_date" +CHECK_SQL = ( + "(release_status = 'embargoed' AND release_at IS NOT NULL) " + "OR (release_status IS DISTINCT FROM 'embargoed' AND release_at IS NULL)" +) + + +def upgrade() -> None: + for table in RELEASE_TABLES: + op.add_column( + table, + sa.Column("release_at", sa.Date(), nullable=True, comment=COMMENT), + ) + + for table in RELEASE_TABLES: + op.create_check_constraint( + CHECK_NAME.format(table=table), table, sa.text(CHECK_SQL) + ) + + for table in VERSION_TABLES: + op.add_column( + table, + sa.Column("release_at", sa.Date(), autoincrement=False, nullable=True), + ) + + +def downgrade() -> None: + for table in VERSION_TABLES: + op.drop_column(table, "release_at") + + for table in RELEASE_TABLES: + op.drop_constraint(CHECK_NAME.format(table=table), table, type_="check") + op.drop_column(table, "release_at") diff --git a/alembic/versions/b4c5d6e7f8a9_exclude_embargoed_from_public_ogc_views.py b/alembic/versions/b4c5d6e7f8a9_exclude_embargoed_from_public_ogc_views.py new file mode 100644 index 000000000..7b815c1cb --- /dev/null +++ b/alembic/versions/b4c5d6e7f8a9_exclude_embargoed_from_public_ogc_views.py @@ -0,0 +1,592 @@ +"""exclude embargoed records from the public water-level OGC views + +The four public relations that read the observation chain gate on the thing's +release_status and nothing below it, so an embargoed observation at a public +well would still be published. This adds the missing clause to each. + +**Why `IS DISTINCT FROM 'embargoed'` and not `= 'public'`.** The other +observation-backed public relations (ogc_well_water_column, and the Group A +thing views since b8c9d0e1f2a3) filter their observations on +`release_status = 'public'`. Matching them here would be tidier and is +deliberately not done: it would also drop every observation sitting at +`draft`, `provisional`, or NULL, which is a release-policy change with its own +row counts to check, not an embargo. This predicate removes exactly the +embargoed rows, of which there are none until somebody sets one, so the four +relations return byte-identical results the day this lands. Migration +w1x2y3z4a5b6 is the record of what the tidier version costs when the row +states are not what you assumed: three NGWMN exports emptied, 3005/3005 rows. + +`IS DISTINCT FROM` rather than `<>` because release_status is nullable, and +`NULL <> 'embargoed'` is NULL, which would filter the row out -- turning a +narrow embargo clause into exactly the silent emptying above. + +Whole-thing embargoes need no change: `t.release_status = 'public'` already +excludes 'embargoed', so a thing held back disappears from every public +relation the moment its level changes. + +The chemistry collections are untouched and cannot be fixed here. +ogc_major_chemistry_results, ogc_minor_chemistry_wells, ogc_avg_tds_wells and +ogc_latest_tds_wells read the legacy NMA_* mirror tables, which carry no +release columns at all -- their only gate is the joined thing. Per-record +chemistry embargo is a separate decision; see docs/data-embargo.md. + +Only public relations change. The ogc_internal_* mount serves Bureau staff and +has never filtered on release_status; seeing embargoed data before it is +published is the point of it. + +The view bodies below are character-for-character the templates from +f4a5b6c7d8e9 (and 986e0eb85ab3 for ogc_actively_monitored_wells) with the +embargo clause added; downgrade() restores them without it. + +Revision ID: b4c5d6e7f8a9 +Revises: a3b4c5d6e7f8 +Create Date: 2026-09-01 00:00:00.000000 + +""" + +from typing import Sequence, Union + +from alembic import op +from sqlalchemy import inspect, text + +# revision identifiers, used by Alembic. +revision: str = "b4c5d6e7f8a9" +down_revision: Union[str, Sequence[str], None] = "a3b4c5d6e7f8" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + +REQUIRED_TABLES = { + "thing", + "location", + "location_thing_association", + "observation", + "sample", + "field_activity", + "field_event", +} + +# These relations only exist in their public-filtered form, so the thing-level +# clause is no longer a toggle -- it is always emitted, and only the embargo +# clause below is switched between upgrade and downgrade. +PUBLIC_FILTER = " AND t.release_status = 'public'" + +# One clause per level of the chain the CTEs join: observation, sample, +# activity, event. Indented to the 16 columns the surrounding WHERE clauses +# use, since it is interpolated into the middle of them. +EMBARGO_ALIASES = ("o", "s", "fa", "fe") +EMBARGO_FILTER = "".join( + f"\n{' ' * 16}AND {alias}.release_status IS DISTINCT FROM 'embargoed'" + for alias in EMBARGO_ALIASES +) + +METERS_TO_FEET = 3.28084 + +LATEST_LOCATION_CTE = """ +SELECT DISTINCT ON (lta.thing_id) + lta.thing_id, + lta.location_id, + lta.effective_start +FROM location_thing_association AS lta +WHERE lta.effective_end IS NULL +ORDER BY lta.thing_id, lta.effective_start DESC +""".strip() + + +def _drop_view_or_materialized_view(view_name: str) -> None: + # DROP VIEW IF EXISTS / DROP MATERIALIZED VIEW IF EXISTS only suppress + # "relation does not exist" -- Postgres still raises WrongObjectType if + # the relation exists as the other kind (e.g. DROP VIEW against an + # existing materialized view), so the relation's actual kind must be + # checked first rather than trying both blindly. + bind = op.get_bind() + relkind = bind.execute( + text("SELECT relkind FROM pg_class WHERE oid = to_regclass(:name)"), + {"name": view_name}, + ).scalar() + if relkind == "m": + op.execute(text(f"DROP MATERIALIZED VIEW IF EXISTS {view_name}")) + elif relkind == "v": + op.execute(text(f"DROP VIEW IF EXISTS {view_name}")) + + +def _check_required_tables() -> None: + bind = op.get_bind() + inspector = inspect(bind) + existing_tables = set(inspector.get_table_names(schema="public")) + missing = REQUIRED_TABLES - existing_tables + if missing: + raise RuntimeError( + "Cannot apply the embargo filter to the OGC views. " + f"Missing required tables: {', '.join(sorted(missing))}" + ) + + +def _create_latest_depth_view(with_embargo: bool) -> str: + release_filter = PUBLIC_FILTER + (EMBARGO_FILTER if with_embargo else "") + return f""" + CREATE MATERIALIZED VIEW ogc_latest_depth_to_water_wells AS + WITH latest_location AS ( +{LATEST_LOCATION_CTE} + ), + ranked_obs AS ( + SELECT + fe.thing_id, + o.id AS observation_id, + o.observation_datetime, + o.value, + o.measuring_point_height, + -- Treat NULL measuring_point_height as 0 when computing + -- depth_to_water_bgs. + ( + o.value - COALESCE(o.measuring_point_height, 0) + ) AS depth_to_water_bgs, + ROW_NUMBER() OVER ( + PARTITION BY fe.thing_id + ORDER BY o.observation_datetime DESC, o.id DESC + ) AS rn + FROM observation AS o + JOIN sample AS s ON s.id = o.sample_id + JOIN field_activity AS fa ON fa.id = s.field_activity_id + JOIN field_event AS fe ON fe.id = fa.field_event_id + JOIN thing AS t ON t.id = fe.thing_id + WHERE + t.thing_type = 'water well' + AND fa.activity_type = 'groundwater level' + AND o.value IS NOT NULL{release_filter} + ) + SELECT + t.id AS id, + t.name, + t.thing_type, + ro.observation_id, + ro.observation_datetime, + ro.value AS depth_to_water_reference, + ro.measuring_point_height, + ro.depth_to_water_bgs, + l.point + FROM ranked_obs AS ro + JOIN thing AS t ON t.id = ro.thing_id + JOIN latest_location AS ll ON ll.thing_id = t.id + JOIN location AS l ON l.id = ll.location_id + WHERE ro.rn = 1 + """ + + +def _create_depth_to_water_trend_view(with_embargo: bool) -> str: + release_filter = PUBLIC_FILTER + (EMBARGO_FILTER if with_embargo else "") + return f""" + CREATE MATERIALIZED VIEW ogc_depth_to_water_trend_wells AS + WITH latest_location AS ( +{LATEST_LOCATION_CTE} + ), + obs AS ( + SELECT + fe.thing_id, + o.observation_datetime, + (o.value - COALESCE(o.measuring_point_height, 0)) AS depth_to_water_bgs + FROM observation AS o + JOIN sample AS s ON s.id = o.sample_id + JOIN field_activity AS fa ON fa.id = s.field_activity_id + JOIN field_event AS fe ON fe.id = fa.field_event_id + JOIN thing AS t ON t.id = fe.thing_id + WHERE + t.thing_type = 'water well' + AND fa.activity_type = 'groundwater level' + AND o.value IS NOT NULL + AND o.observation_datetime IS NOT NULL{release_filter} + ), + agg AS ( + SELECT + ob.thing_id, + COUNT(*)::integer AS record_count, + MIN(ob.observation_datetime) AS first_observation_datetime, + MAX(ob.observation_datetime) AS last_observation_datetime, + EXTRACT(EPOCH FROM (MAX(ob.observation_datetime) - MIN(ob.observation_datetime))) + / 31557600.0 AS span_years, + REGR_SLOPE( + ob.depth_to_water_bgs, + EXTRACT(EPOCH FROM ob.observation_datetime) + ) * 31557600.0 AS slope_ft_per_year + FROM obs AS ob + GROUP BY ob.thing_id + ) + SELECT + t.id AS id, + t.name, + t.thing_type, + a.record_count, + a.first_observation_datetime, + a.last_observation_datetime, + a.span_years, + a.slope_ft_per_year, + CASE + WHEN a.record_count >= 10 OR (a.record_count >= 4 AND a.span_years >= 2.0) THEN + CASE + WHEN a.slope_ft_per_year IS NULL THEN 'not enough data' + WHEN a.slope_ft_per_year > 0.25 THEN 'increasing' + WHEN a.slope_ft_per_year < -0.25 THEN 'decreasing' + ELSE 'stable' + END + ELSE 'not enough data' + END AS trend_category, + l.point + FROM agg AS a + JOIN thing AS t ON t.id = a.thing_id + JOIN latest_location AS ll ON ll.thing_id = t.id + JOIN location AS l ON l.id = ll.location_id + """ + + +def _create_water_well_summary_view(with_embargo: bool) -> str: + release_filter = PUBLIC_FILTER + (EMBARGO_FILTER if with_embargo else "") + return f""" + CREATE MATERIALIZED VIEW ogc_water_well_summary AS + WITH latest_location AS ( +{LATEST_LOCATION_CTE} + ), + wl_obs AS ( + SELECT + fe.thing_id, + o.id AS observation_id, + o.observation_datetime, + (o.value - COALESCE(o.measuring_point_height, 0)) AS water_level + FROM observation AS o + JOIN sample AS s ON s.id = o.sample_id + JOIN field_activity AS fa ON fa.id = s.field_activity_id + JOIN field_event AS fe ON fe.id = fa.field_event_id + JOIN thing AS t ON t.id = fe.thing_id + WHERE + t.thing_type = 'water well' + AND fa.activity_type = 'groundwater level' + AND o.value IS NOT NULL + AND o.observation_datetime IS NOT NULL{release_filter} + ), + wl_agg AS ( + SELECT + w.thing_id, + COUNT(*)::integer AS total_water_levels, + MIN(w.water_level) AS min_water_level, + MAX(w.water_level) AS max_water_level, + REGR_SLOPE( + w.water_level, + EXTRACT(EPOCH FROM w.observation_datetime) + ) * 31557600.0 AS water_level_trend_ft_per_year + FROM wl_obs AS w + GROUP BY w.thing_id + ), + wl_last AS ( + SELECT + ranked.thing_id, + ranked.water_level AS last_water_level, + ranked.observation_datetime AS last_water_level_datetime + FROM ( + SELECT + w.thing_id, + w.water_level, + w.observation_datetime, + ROW_NUMBER() OVER ( + PARTITION BY w.thing_id + ORDER BY w.observation_datetime DESC, w.observation_id DESC + ) AS rn + FROM wl_obs AS w + ) AS ranked + WHERE ranked.rn = 1 + ) + SELECT + t.id AS id, + t.name, + t.well_depth, + l.elevation, + dpl.collection_method AS elevation_method, + t.nma_formation_zone AS formation_zone, + wa.total_water_levels, + wl.last_water_level, + wl.last_water_level_datetime, + wa.min_water_level, + wa.max_water_level, + wa.water_level_trend_ft_per_year, + l.point + FROM thing AS t + JOIN latest_location AS ll ON ll.thing_id = t.id + JOIN location AS l ON l.id = ll.location_id + JOIN wl_agg AS wa ON wa.thing_id = t.id + LEFT JOIN wl_last AS wl ON wl.thing_id = t.id + LEFT JOIN LATERAL ( + SELECT dp.collection_method + FROM data_provenance AS dp + WHERE + dp.target_table = 'location' + AND dp.target_id = l.id + AND dp.field_name = 'elevation' + ORDER BY dp.id DESC + LIMIT 1 + ) AS dpl ON true + WHERE t.thing_type = 'water well' + AND wa.total_water_levels > 0 + """ + + +def _create_water_elevation_view(with_embargo: bool) -> str: + release_filter = PUBLIC_FILTER + (EMBARGO_FILTER if with_embargo else "") + return f""" + CREATE MATERIALIZED VIEW ogc_water_elevation_wells AS + WITH latest_location AS ( +{LATEST_LOCATION_CTE} + ), + ranked_obs AS ( + SELECT + fe.thing_id, + o.id AS observation_id, + o.observation_datetime, + CASE + WHEN lower(trim(o.unit)) IN ('m', 'meter', 'meters', 'metre', 'metres') THEN + (o.value * {METERS_TO_FEET}) - COALESCE(o.measuring_point_height, 0) + WHEN lower(trim(o.unit)) IN ('ft', 'foot', 'feet') THEN + o.value - COALESCE(o.measuring_point_height, 0) + ELSE + NULL + END AS depth_to_water_below_ground_surface + FROM observation AS o + JOIN sample AS s ON s.id = o.sample_id + JOIN field_activity AS fa ON fa.id = s.field_activity_id + JOIN field_event AS fe ON fe.id = fa.field_event_id + JOIN thing AS t ON t.id = fe.thing_id + WHERE + t.thing_type = 'water well' + AND fa.activity_type = 'groundwater level' + AND o.value IS NOT NULL + AND o.observation_datetime IS NOT NULL + AND lower(trim(o.unit)) IN ( + 'm', + 'meter', + 'meters', + 'metre', + 'metres', + 'ft', + 'foot', + 'feet' + ){release_filter} + ), + latest_obs AS ( + SELECT + ro.*, + ROW_NUMBER() OVER ( + PARTITION BY ro.thing_id + ORDER BY ro.observation_datetime DESC, ro.observation_id DESC + ) AS rn + FROM ranked_obs AS ro + ) + SELECT + t.id AS id, + t.name, + t.thing_type, + lo.observation_id, + lo.observation_datetime, + l.elevation AS elevation_m, + lo.depth_to_water_below_ground_surface AS depth_to_water_below_ground_surface_ft, + ((l.elevation * {METERS_TO_FEET}) - lo.depth_to_water_below_ground_surface) + AS water_elevation_ft, + l.point + FROM latest_obs AS lo + JOIN thing AS t ON t.id = lo.thing_id + JOIN latest_location AS ll ON ll.thing_id = t.id + JOIN location AS l ON l.id = ll.location_id + WHERE lo.rn = 1 + """ + + +def _create_actively_monitored_wells_view(all_groups: bool) -> str: + if all_groups: + # Aggregated: one row per well, group_ids/group_names/group_types as + # arrays, so `id` stays unique even when a well belongs to several + # groups. release_status = 'public' is checked on the group row + # itself (mirrors _create_project_areas_view's public_only handling) + # since any group can appear here now, not just one hardcoded one. + # group_thing_association has no unique constraint on + # (group_id, thing_id), so distinct_memberships de-dupes before + # aggregating; all three arrays are ordered by the same group_id key + # so they stay index-aligned with each other (ordering each array by + # its own column, e.g. names alphabetically, would desync them). + return """ + CREATE VIEW ogc_actively_monitored_wells AS + WITH latest_monitoring_status AS ( + SELECT DISTINCT ON (sh.target_id) + sh.target_id AS thing_id, + sh.status_value + FROM status_history AS sh + WHERE + sh.target_table = 'thing' + AND sh.status_type = 'Monitoring Status' + ORDER BY sh.target_id, sh.start_date DESC, sh.id DESC + ), + distinct_memberships AS ( + SELECT DISTINCT + gta.thing_id, + g.id AS group_id, + g.name AS group_name, + g.group_type + FROM group_thing_association AS gta + JOIN "group" AS g ON g.id = gta.group_id + WHERE g.release_status = 'public' + ) + SELECT + wws.id, + wws.name, + 'water well'::text AS thing_type, + wws.well_depth, + wws.elevation, + wws.elevation_method, + wws.formation_zone, + wws.total_water_levels, + wws.last_water_level, + wws.last_water_level_datetime, + wws.min_water_level, + wws.max_water_level, + wws.water_level_trend_ft_per_year, + array_agg(dm.group_id ORDER BY dm.group_id) AS group_ids, + array_agg(dm.group_name ORDER BY dm.group_id) AS group_names, + array_agg(dm.group_type ORDER BY dm.group_id) AS group_types, + wws.point + FROM ogc_water_well_summary AS wws + JOIN latest_monitoring_status AS lms ON lms.thing_id = wws.id + JOIN distinct_memberships AS dm ON dm.thing_id = wws.id + WHERE lms.status_value = 'Currently monitored' + GROUP BY + wws.id, wws.name, wws.well_depth, wws.elevation, + wws.elevation_method, wws.formation_zone, + wws.total_water_levels, wws.last_water_level, + wws.last_water_level_datetime, wws.min_water_level, + wws.max_water_level, wws.water_level_trend_ft_per_year, + wws.point + """ + # Historical (downgrade target): byte-for-byte the pre-fix view, single + # group_id/group_name/group_type columns, scoped to one hardcoded group. + return """ + CREATE VIEW ogc_actively_monitored_wells AS + WITH latest_monitoring_status AS ( + SELECT DISTINCT ON (sh.target_id) + sh.target_id AS thing_id, + sh.status_value + FROM status_history AS sh + WHERE + sh.target_table = 'thing' + AND sh.status_type = 'Monitoring Status' + ORDER BY sh.target_id, sh.start_date DESC, sh.id DESC + ) + SELECT + wws.id, + wws.name, + 'water well'::text AS thing_type, + wws.well_depth, + wws.elevation, + wws.elevation_method, + wws.formation_zone, + wws.total_water_levels, + wws.last_water_level, + wws.last_water_level_datetime, + wws.min_water_level, + wws.max_water_level, + wws.water_level_trend_ft_per_year, + g.id AS group_id, + g.name AS group_name, + g.group_type, + wws.point + FROM "group" AS g + JOIN group_thing_association AS gta ON gta.group_id = g.id + JOIN ogc_water_well_summary AS wws ON wws.id = gta.thing_id + JOIN latest_monitoring_status AS lms ON lms.thing_id = wws.id + WHERE lower(trim(g.name)) = 'water level network' + AND lms.status_value = 'Currently monitored' + """ + + +def _recreate_water_level_views(with_embargo: bool) -> None: + """Rebuild the four observation-backed relations, embargo clause on or off. + + ogc_actively_monitored_wells depends on ogc_water_well_summary via a direct + JOIN; Postgres refuses to drop a materialized view while a dependent view + exists, so it goes first and comes back last. It is recreated from + 986e0eb85ab3's all-groups form -- the shape in production -- not + f4a5b6c7d8e9's, and its own SQL is unchanged: it inherits the embargo + filter through the summary it selects from. + """ + _drop_view_or_materialized_view("ogc_actively_monitored_wells") + + _drop_view_or_materialized_view("ogc_latest_depth_to_water_wells") + op.execute(text(_create_latest_depth_view(with_embargo))) + op.execute( + text( + "COMMENT ON MATERIALIZED VIEW ogc_latest_depth_to_water_wells IS " + "'Latest depth-to-water per well view for pygeoapi.'" + ) + ) + op.execute( + text( + "CREATE UNIQUE INDEX ux_ogc_latest_depth_to_water_wells_id " + "ON ogc_latest_depth_to_water_wells (id)" + ) + ) + + _drop_view_or_materialized_view("ogc_depth_to_water_trend_wells") + op.execute(text(_create_depth_to_water_trend_view(with_embargo))) + op.execute( + text( + "COMMENT ON MATERIALIZED VIEW ogc_depth_to_water_trend_wells IS " + "'Depth-to-water trend classification for water wells.'" + ) + ) + op.execute( + text( + "CREATE UNIQUE INDEX ux_ogc_depth_to_water_trend_wells_id " + "ON ogc_depth_to_water_trend_wells (id)" + ) + ) + + _drop_view_or_materialized_view("ogc_water_well_summary") + op.execute(text(_create_water_well_summary_view(with_embargo))) + op.execute( + text( + "COMMENT ON MATERIALIZED VIEW ogc_water_well_summary IS " + "'Summary statistics for water wells including water-level trend.'" + ) + ) + op.execute( + text( + "CREATE UNIQUE INDEX ux_ogc_water_well_summary_id " + "ON ogc_water_well_summary (id)" + ) + ) + + _drop_view_or_materialized_view("ogc_water_elevation_wells") + op.execute(text(_create_water_elevation_view(with_embargo))) + op.execute( + text( + "COMMENT ON MATERIALIZED VIEW ogc_water_elevation_wells IS " + "'Latest water elevation per well with explicit units: " + "elevation_m, depth_to_water_below_ground_surface_ft, water_elevation_ft.'" + ) + ) + op.execute( + text( + "CREATE UNIQUE INDEX ux_ogc_water_elevation_wells_id " + "ON ogc_water_elevation_wells (id)" + ) + ) + + # Recreate now that ogc_water_well_summary exists again. + op.execute(text(_create_actively_monitored_wells_view(all_groups=True))) + op.execute( + text( + "COMMENT ON VIEW ogc_actively_monitored_wells IS " + "'Actively (currently) monitored wells across all groups for pygeoapi.'" + ) + ) + + +def upgrade() -> None: + _check_required_tables() + _recreate_water_level_views(with_embargo=True) + + +def downgrade() -> None: + _recreate_water_level_views(with_embargo=False) diff --git a/alembic/versions/e7c1a9f4b2d8_field_access_consent_and_data_maturity.py b/alembic/versions/e7c1a9f4b2d8_field_access_consent_and_data_maturity.py new file mode 100644 index 000000000..63f75e24a --- /dev/null +++ b/alembic/versions/e7c1a9f4b2d8_field_access_consent_and_data_maturity.py @@ -0,0 +1,170 @@ +"""rename permission_history to field_access_consent; add data_maturity + +Two vocabulary fixes from ADR5, neither of which changes any API response +shape: + +1. ``permission_history`` records a landowner's consent to physical site + access. It is a domain fact, not authorization, and the name implied + otherwise. It becomes ``field_access_consent``. The ``permission_type`` + and ``permission_allowed`` columns keep their names -- ``permission_type`` + is lexicon-backed, and both are published in Thing responses. + +2. ``release_status`` was one column carrying two axes. Its lexicon lists + `draft`, `public`, `private`, `published`, `archived` (visibility) next to + `provisional` and `final` (review state), so a record could not be public + and provisional at once -- which San Acacia data is (see + automated_ingestion/ocotillo/loader.py). Release *level* stays in + ``release_status``; review state moves to ``data_maturity``. + + ``data_maturity`` is not a new vocabulary. The lexicon category + (`provisional`, `in review`, `approved`) and the column already existed on + ``transducer_observation``; this generalizes them onto every ReleaseMixin + table, so ``transducer_observation`` is deliberately absent from the list + below -- it already has the column, unchanged. + +``data_maturity`` is nullable and starts NULL, meaning "not stated". No +existing row changes meaning. Rows that carry `release_status='provisional'` +are left alone; splitting them onto the two axes is a data migration with its +own decision to make. + +Revision ID: e7c1a9f4b2d8 +Revises: c9d0e1f2a3b4 +Create Date: 2026-08-24 00:00:00.000000 + +""" + +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "e7c1a9f4b2d8" +down_revision: Union[str, Sequence[str], None] = "c9d0e1f2a3b4" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +# Every table backed by a model that mixes in ReleaseMixin, with +# permission_history already under its new name and transducer_observation +# omitted because it already carries data_maturity. +RELEASE_TABLES = ( + "address", + "analysis_method", + "aquifer_system", + "aquifer_type", + "asset", + "contact", + "data_provenance", + "deployment", + "email", + "field_access_consent", + "field_activity", + "field_event", + "field_event_participant", + "geologic_formation", + "group", + "location", + "measuring_point_history", + "monitoring_frequency_history", + "notes", + "observation", + "parameter", + "phone", + "regulatory_limit", + "sample", + "sensor", + "status_history", + "thing", + "thing_aquifer_association", + "thing_geologic_formation_association", + "thing_id_link", + "transducer_observation_block", + "well_casing_material", + "well_purpose", + "well_screen", +) + +# sqlalchemy-continuum mirrors every tracked column into the version table, so +# the models carrying __versioned__ need the new column there too. Version rows +# are written explicitly and historical rows have no answer, so it is nullable. +VERSION_TABLES = ( + "aquifer_system_version", + "geologic_formation_version", + "location_version", + "observation_version", + "parameter_version", + "regulatory_limit_version", + "thing_version", +) + + +def upgrade() -> None: + op.rename_table("permission_history", "field_access_consent") + # rename_table leaves the constraint and sequence names behind, which is + # cosmetic but makes the old name resurface in every error message. + op.execute( + sa.text( + "ALTER TABLE field_access_consent " + "RENAME CONSTRAINT permission_history_pkey TO field_access_consent_pkey" + ) + ) + op.execute( + sa.text( + "ALTER SEQUENCE IF EXISTS permission_history_id_seq " + "RENAME TO field_access_consent_id_seq" + ) + ) + + for table in RELEASE_TABLES: + op.add_column( + table, + sa.Column( + "data_maturity", + sa.String(length=100), + nullable=True, + comment="Review state; orthogonal to release_status", + ), + ) + op.create_foreign_key( + f"{table}_data_maturity_fkey", + table, + "lexicon_term", + ["data_maturity"], + ["term"], + onupdate="CASCADE", + ) + + for table in VERSION_TABLES: + op.add_column( + table, + sa.Column( + "data_maturity", + sa.String(length=100), + autoincrement=False, + nullable=True, + ), + ) + + +def downgrade() -> None: + for table in VERSION_TABLES: + op.drop_column(table, "data_maturity") + + for table in RELEASE_TABLES: + op.drop_constraint(f"{table}_data_maturity_fkey", table, type_="foreignkey") + op.drop_column(table, "data_maturity") + + op.execute( + sa.text( + "ALTER SEQUENCE IF EXISTS field_access_consent_id_seq " + "RENAME TO permission_history_id_seq" + ) + ) + op.execute( + sa.text( + "ALTER TABLE field_access_consent " + "RENAME CONSTRAINT field_access_consent_pkey TO permission_history_pkey" + ) + ) + op.rename_table("field_access_consent", "permission_history") diff --git a/api/access.py b/api/access.py new file mode 100644 index 000000000..000d7abeb --- /dev/null +++ b/api/access.py @@ -0,0 +1,387 @@ +# =============================================================================== +# Copyright 2025 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +api/access.py + +The first tenant of the ADR5 visibility layer. + +One router, deliberately small: + +* ``/access/grant`` administers internal grants, and ``/access/decision`` + answers "may I, right now?" from ``services/visibility.py`` rather than from + anything local. +* ``/access/destination`` and ``/access/consent`` administer destinations and + landowner consent, and ``/access/destination/{slug}/thing`` answers "what + does this destination get". + +The prefix is ``/access`` rather than ``/publication`` because +``api/publication.py`` is already the bibliography -- citations, not consent. + +Nothing here changes what an existing endpoint returns. The layer is proved +end to end behind one service before it is put behind all of them (ADR5, A.6). +Every route is authorized: administration is Admin, reading is Viewer. +""" + +from datetime import date + +from fastapi import APIRouter, Query +from sqlalchemy import select +from starlette.status import HTTP_201_CREATED + +from core.dependencies import ( + admin_dependency, + session_dependency, + viewer_dependency, +) +from db.destination import Destination +from db.permission_grant import PermissionGrant +from db.publication_consent import PublicationConsent +from schemas.access import ( + AccessDecision, + CreateDestination, + CreatePermissionGrant, + CreatePublicationConsent, + DestinationResponse, + PermissionGrantResponse, + PublicationConsentResponse, + PublishedThing, +) +from services.access_admin import ( + AlreadyRevoked, + actor_from_payload, + create_grant, + record_consent, + register_destination, + revoke_consent, + revoke_grant, +) +from services.exceptions_helper import PydanticStyleException +from services.visibility import ( + destination_by_slug, + may, + principals_from_payload, + published_things, +) + +router = APIRouter(prefix="/access", tags=["access control"]) + + +def _not_found(what: str, value): + return PydanticStyleException( + status_code=404, + detail=[ + { + "loc": ["path", what], + "msg": f"No {what} {value!r}.", + "type": "value_error", + "input": value, + } + ], + ) + + +def _invalid(field: str, message: str, value): + return PydanticStyleException( + status_code=422, + detail=[ + { + "loc": ["body", field], + "msg": message, + "type": "value_error", + "input": value, + } + ], + ) + + +# ============= Permission grants ============================================= + + +@router.post( + "/grant", summary="Create a permission grant", status_code=HTTP_201_CREATED +) +def create_permission_grant( + payload: CreatePermissionGrant, + session: session_dependency, + user: admin_dependency, +) -> PermissionGrantResponse: + """Grant a principal one capability over one data type within one scope. + + The grant names its data type; there is no wildcard, so a data type added + later is never covered by this row. + """ + try: + grant = create_grant( + session, + actor_from_payload(user), + principal_type=payload.principal_type.value, + principal_id=payload.principal_id, + capability=payload.capability.value, + scope_type=payload.scope_type.value, + scope_id=payload.scope_id, + data_type=payload.data_type.value, + starts_at=payload.starts_at, + ends_at=payload.ends_at, + reason=payload.reason, + ) + except ValueError as exception: + raise _invalid("scope_id", str(exception), payload.scope_id) + + return PermissionGrantResponse.model_validate(grant) + + +@router.post( + "/grant/{grant_id}/revocation", + summary="Revoke a permission grant", + status_code=HTTP_201_CREATED, +) +def revoke_permission_grant( + grant_id: int, + session: session_dependency, + user: admin_dependency, +) -> PermissionGrantResponse: + """Revoke now. Effective at the next read, not at the next token refresh.""" + grant = session.get(PermissionGrant, grant_id) + if grant is None: + raise _not_found("grant_id", grant_id) + + try: + grant = revoke_grant(session, actor_from_payload(user), grant) + except AlreadyRevoked as exception: + raise _invalid("grant_id", str(exception), grant_id) + + return PermissionGrantResponse.model_validate(grant) + + +@router.get("/grant", summary="List grants") +def get_permission_grants( + session: session_dependency, + user: admin_dependency, + principal_id: str = Query( + default=None, description="Authentik subject, role, or key label" + ), + capability: str = Query(default=None), + data_type: str = Query(default=None), + scope_type: str = Query(default=None), + include_revoked: bool = Query(default=False), +) -> list[PermissionGrantResponse]: + """All grants, or a narrower slice of them. + + Every filter is optional, so the bare route is the admin-wide audit view; + passing ``principal_id`` narrows it to one principal, as before. + """ + statement = select(PermissionGrant) + if principal_id is not None: + statement = statement.where(PermissionGrant.principal_id == principal_id) + if capability is not None: + statement = statement.where(PermissionGrant.capability == capability) + if data_type is not None: + statement = statement.where(PermissionGrant.data_type == data_type) + if scope_type is not None: + statement = statement.where(PermissionGrant.scope_type == scope_type) + if not include_revoked: + statement = statement.where(PermissionGrant.revoked_at.is_(None)) + + return [ + PermissionGrantResponse.model_validate(row) + for row in session.execute(statement).scalars() + ] + + +@router.get("/decision", summary="Ask the visibility layer about yourself") +def get_access_decision( + session: session_dependency, + user: viewer_dependency, + capability: str = Query(), + data_type: str = Query(), + thing_id: int = Query(default=None), + on_date: date = Query(default=None), +) -> AccessDecision: + """May the caller do this? Answered by the one visibility layer. + + Default deny: an unrecognized capability or a caller the token says + nothing about gets False rather than an error, because a question this + layer cannot answer is not a yes. + """ + principals = principals_from_payload(user) + return AccessDecision( + allowed=may( + session, + principals, + capability=capability, + data_type=data_type, + thing_id=thing_id, + on_date=on_date, + ), + capability=capability, + data_type=data_type, + thing_id=thing_id, + principals=[f"{kind}:{identifier}" for kind, identifier in principals], + ) + + +# ============= Destinations ============================================= + + +@router.post( + "/destination", summary="Register a destination", status_code=HTTP_201_CREATED +) +def create_destination( + payload: CreateDestination, + session: session_dependency, + user: admin_dependency, +) -> DestinationResponse: + existing = destination_by_slug(session, payload.slug) + if existing is not None: + raise PydanticStyleException( + status_code=409, + detail=[ + { + "loc": ["body", "slug"], + "msg": f"Destination {payload.slug!r} already exists.", + "type": "value_error", + "input": payload.slug, + } + ], + ) + + destination = register_destination( + session, + actor_from_payload(user), + slug=payload.slug, + name=payload.name, + destination_kind=payload.destination_kind.value, + description=payload.description, + ) + return DestinationResponse.model_validate(destination) + + +@router.get("/destination", summary="List destinations") +def get_destinations( + session: session_dependency, + user: viewer_dependency, +) -> list[DestinationResponse]: + return [ + DestinationResponse.model_validate(row) + for row in session.execute(select(Destination).order_by(Destination.slug)) + .scalars() + .all() + ] + + +@router.get("/destination/{slug}/thing", summary="What this destination may read") +def get_published_things( + slug: str, + session: session_dependency, + user: viewer_dependency, + data_type: str = Query(default=None), + on_date: date = Query(default=None), +) -> list[PublishedThing]: + """The destination's view, computed from consent rows at request time. + + A retired destination gets an empty list, and so does one nobody has + consented to yet: default deny, with no separate "unpublished" state to + keep in sync. + """ + destination = destination_by_slug(session, slug) + if destination is None: + raise _not_found("slug", slug) + + return [ + PublishedThing(**entry) + for entry in published_things( + session, destination, data_type=data_type, on_date=on_date + ) + ] + + +# ============= Publication consent ============================================= + + +@router.post( + "/consent", summary="Record publication consent", status_code=HTTP_201_CREATED +) +def create_publication_consent( + payload: CreatePublicationConsent, + session: session_dependency, + user: admin_dependency, +) -> PublicationConsentResponse: + """Record that an owner agreed to publish one data type to one destination.""" + destination = destination_by_slug(session, payload.destination_slug) + if destination is None: + raise _not_found("destination_slug", payload.destination_slug) + + try: + consent = record_consent( + session, + actor_from_payload(user), + thing_id=payload.thing_id, + destination_id=destination.id, + data_type=payload.data_type.value, + starts_at=payload.starts_at, + ends_at=payload.ends_at, + contact_id=payload.contact_id, + notes=payload.notes, + ) + except ValueError as exception: + raise _invalid("ends_at", str(exception), payload.ends_at) + + return PublicationConsentResponse.model_validate(consent) + + +@router.post( + "/consent/{consent_id}/revocation", + summary="Withdraw publication consent", + status_code=HTTP_201_CREATED, +) +def revoke_publication_consent( + consent_id: int, + session: session_dependency, + user: admin_dependency, +) -> PublicationConsentResponse: + """Stop offering it. Copies already harvested are not recalled.""" + consent = session.get(PublicationConsent, consent_id) + if consent is None: + raise _not_found("consent_id", consent_id) + + try: + consent = revoke_consent(session, actor_from_payload(user), consent) + except AlreadyRevoked as exception: + raise _invalid("consent_id", str(exception), consent_id) + + return PublicationConsentResponse.model_validate(consent) + + +@router.get("/consent", summary="List consent for a thing") +def get_publication_consent( + session: session_dependency, + user: viewer_dependency, + thing_id: int = Query(), + include_revoked: bool = Query(default=False), +) -> list[PublicationConsentResponse]: + statement = select(PublicationConsent).where( + PublicationConsent.thing_id == thing_id + ) + if not include_revoked: + statement = statement.where(PublicationConsent.revoked_at.is_(None)) + + return [ + PublicationConsentResponse.model_validate(row) + for row in session.execute(statement).scalars() + ] + + +# ============= EOF ============================================= diff --git a/api/well_inventory.py b/api/well_inventory.py index 46138a8fa..64b37af4c 100644 --- a/api/well_inventory.py +++ b/api/well_inventory.py @@ -45,7 +45,7 @@ # FieldEventParticipant, # FieldActivity, # Contact, -# PermissionHistory, +# FieldAccessConsent, # Thing, # ) # from schemas.thing import CreateWell diff --git a/cli/cli.py b/cli/cli.py index 7a7332e9f..a3e39fc01 100644 --- a/cli/cli.py +++ b/cli/cli.py @@ -1431,6 +1431,83 @@ def import_project_area_boundaries_command( ) +@cli.command("seed-access-grants") +def seed_access_grants( + apply: bool = typer.Option( + False, + "--apply", + help="Write the grants. Without this the command only shows the plan.", + ), + verbose: bool = typer.Option( + False, "--verbose", help="List every grant, not just the counts." + ), +): + """Give the Authentik roles the access they already had (ADR5, 5.2). + + An empty grant table means default deny, which is correct and useless: the + visibility layer says no to everyone until the roles that exist today are + written down as role principals. + + Previews by default. Idempotent, and it will not resurrect a seeded grant + somebody revoked. + """ + from db.engine import session_ctx + from services.access_seed import seed_role_grants + + with session_ctx() as session: + plan = seed_role_grants(session, apply=apply) + + verb = "Created" if apply else "Would create" + typer.echo(f"{verb} {len(plan.created)} grant(s).") + typer.echo(f"Left {len(plan.skipped)} existing grant(s) alone.") + + if verbose: + for entry in plan.created: + typer.echo(f" + {plan.describe(entry)}") + + if plan.created and not apply: + typer.echo("Nothing was written. Re-run with --apply.") + + +@cli.command("release-embargoed") +def release_embargoed( + apply: bool = typer.Option( + False, + "--apply", + help="Write the changes. Without this the command only shows the plan.", + ), + verbose: bool = typer.Option( + False, "--verbose", help="List every record, not just the counts." + ), +): + """Publish records whose embargo date has arrived. + + Flips `release_status` from `embargoed` to `public` where `release_at` has + been reached, and writes an authorization_audit row for each. Meant to run + daily, before the 09:00 UTC materialized-view refresh -- a record released + after the refresh waits another day to appear in the OGC collections. + + Previews by default. Idempotent: a released record is no longer embargoed. + """ + from db.engine import session_ctx + from services.release_schedule import lift_due_embargoes + + with session_ctx() as session: + plan = lift_due_embargoes(session, apply=apply) + + verb = "Released" if apply else "Would release" + typer.echo(f"{verb} {len(plan.lifted)} record(s).") + for table, count in sorted(plan.by_table().items()): + typer.echo(f" {table}: {count}") + + if verbose: + for entry in plan.lifted: + typer.echo(f" + {plan.describe(entry)}") + + if plan.lifted and not apply: + typer.echo("Nothing was written. Re-run with --apply.") + + if __name__ == "__main__": cli() diff --git a/core/enums.py b/core/enums.py index 790272125..675d7fe1a 100644 --- a/core/enums.py +++ b/core/enums.py @@ -83,4 +83,13 @@ Lithology: type[Enum] = build_enum_from_lexicon_category("lithology") FormationCode: type[Enum] = build_enum_from_lexicon_category("formation_code") NoteType: type[Enum] = build_enum_from_lexicon_category("note_type") + +# Access control (ADR5). Vocabularies rather than hardcoded literals so a new +# destination kind is a lexicon entry, not a deploy. There is deliberately no +# access_data_type meaning "all". +PrincipalType: type[Enum] = build_enum_from_lexicon_category("principal_type") +Capability: type[Enum] = build_enum_from_lexicon_category("capability") +GrantScopeType: type[Enum] = build_enum_from_lexicon_category("grant_scope_type") +AccessDataType: type[Enum] = build_enum_from_lexicon_category("access_data_type") +DestinationKind: type[Enum] = build_enum_from_lexicon_category("destination_kind") # ============= EOF ============================================= diff --git a/core/feature_provider.py b/core/feature_provider.py index e13751a38..9cfd08d2e 100644 --- a/core/feature_provider.py +++ b/core/feature_provider.py @@ -13,15 +13,32 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== -"""Feature provider that publishes field-level prose alongside the columns. +"""Feature provider that projects the published columns and documents them. -pygeoapi's PostgreSQL provider reflects a table and reports each column's -JSON Schema type and format. It does not read column comments, and there is -no hook for documentation, so /collections/{id}/schema publishes bare column -names. This subclass annotates the reflected fields from -core/ogc-field-descriptions.yml on the way out. +Two jobs, both on the way out of pygeoapi's PostgreSQL provider. -Read docs/ogc-field-descriptions.md before changing this. +**Projection (ADR5, A.2).** Every public collection publishes the columns +named for it in ``core/field-allowlists.yml`` and no others. The allowlist is +handed to pygeoapi as the provider's ``properties``, which is what +``_select_properties`` builds the SELECT from, so an unlisted column is never +read out of Postgres -- it cannot appear in a feature, in ``/schema``, or in +``/queryables``, and a filter cannot probe it. ``get_fields`` is narrowed to +match, because the reflection sees the whole view. + +A public collection with no allowlist entry publishes no properties. That is +default deny, and ``tests/test_ogc_projection.py`` fails when an entry is +missing so it surfaces in CI rather than in production. The internal mount +(``ogc_internal_*``) is not projected: it serves authenticated Bureau staff, +and per-role internal field rules are the part of ADR5 3.5 that is not built. + +**Documentation.** pygeoapi reflects a table and reports each column's JSON +Schema type and format. It does not read column comments, and there is no hook +for documentation, so ``/collections/{id}/schema`` publishes bare column names. +This subclass annotates the reflected fields from +``core/ogc-field-descriptions.yml``. + +Read docs/ogc-field-descriptions.md and docs/access-field-projection.md before +changing this. """ import logging @@ -29,12 +46,39 @@ from pygeoapi.provider.sql import PostgreSQLProvider from core.ogc_field_metadata import describe_fields +from services.field_projection import ogc_allowlist LOGGER = logging.getLogger(__name__) class DescribedPostgreSQLProvider(PostgreSQLProvider): - """PostgreSQLProvider that annotates reflected columns with prose.""" + """PostgreSQLProvider that projects published columns and annotates them.""" + + def __init__(self, provider_def): + table = provider_def.get("table") + self._allowlist = ogc_allowlist(table) + + if self._allowlist is not None: + # pygeoapi reads `properties` in _select_properties and in get(), + # so setting it here is what actually keeps the column out of the + # query rather than out of the response only. + provider_def = dict(provider_def, properties=sorted(self._allowlist)) + if not self._allowlist: + LOGGER.warning( + "%s has no entry in core/field-allowlists.yml, so it " + "publishes no properties. Add one deliberately.", + table, + ) + + super().__init__(provider_def) + + def _published(self, fields: dict) -> dict: + """Drop reflected fields the allowlist does not name.""" + if self._allowlist is None: + return fields + return { + name: field for name, field in fields.items() if name in self._allowlist + } def get_fields(self): """Reflect the table, then annotate the result. @@ -49,7 +93,9 @@ def get_fields(self): """ fields = super().get_fields() if fields and not getattr(self, "_fields_described", False): - self._fields = describe_fields(self.table, fields) + # Narrow before describing: a column nobody publishes has no + # business in /schema or /queryables, described or not. + self._fields = describe_fields(self.table, self._published(fields)) # super().get_fields() short-circuits on a populated _fields, so # without this flag a later call would re-describe the annotated # dict. Harmless today (describe_fields is idempotent) but it diff --git a/core/field-allowlists.yml b/core/field-allowlists.yml new file mode 100644 index 000000000..537e14f24 --- /dev/null +++ b/core/field-allowlists.yml @@ -0,0 +1,853 @@ +# Per-audience field allowlists for published records (ADR5, 3.5 and A.2). +# +# Two rules this file exists to keep: +# +# 1. Omission produces silence. Only fields listed here reach an audience, so +# a column added next year is invisible until someone adds it deliberately. +# An audience with no entry receives nothing -- default deny, not "all". +# +# 2. `never_public` wins over every allowlist. Engineering guarantees these +# fields are enforced everywhere; what belongs on the list is a policy +# decision with a named data owner, and that owner has not been named yet +# (ADR5, "What this does not decide"). Adding a field here is safe at any +# time. Removing one is not an engineering call. +# +# Enforced by services/field_projection.py, which validates the whole file at +# load: an unknown field, a transform on a field nobody publishes, or an +# allowlist naming a never-public field raises rather than degrading quietly. +# +# Audiences are matched by destination slug first, then by destination kind. +# `round` reduces coordinate precision: 2 decimal places is roughly a +# kilometre, 4 is roughly ten metres. + +never_public: + thing: + # Provenance columns. Who typed a record is internal, always. + - created_by_id + - created_by_name + - updated_by_id + - updated_by_name + # Legacy AMPAPI primary keys. Publishing them invites outside systems to + # key on identifiers the migration is meant to retire. + - nma_pk_welldata + - nma_pk_location + - search_vector + location: + - created_by_id + - created_by_name + - updated_by_id + - updated_by_name + - nma_pk_location + # Free text written by staff for staff. The permissions interview was + # explicit that gate codes, lock combinations and candid landowner notes + # must never leave the Bureau, and this is where such text has landed. + - nma_location_notes + - nma_coordinate_notes + +audiences: + # Per-destination-kind defaults. A newly registered destination inherits the + # rules for its kind; nothing inherits "everything". + by_kind: + public web: + thing: + fields: + - id + - name + - thing_type + - well_depth + - hole_depth + - well_completion_date + location: + fields: + - id + - latitude + - longitude + - elevation + - county + - state + # Protect the landowner without dropping the well off the map. + transforms: + latitude: + round: 2 + longitude: + round: 2 + + harvester: + thing: + fields: + - id + - name + - thing_type + - well_depth + - hole_depth + - well_casing_depth + - well_completion_date + location: + fields: + - id + - latitude + - longitude + - elevation + - county + - state + - quad_name + # A harvester plotting hydrographs needs the well in the right place; + # ten metres is close enough for that and not close enough to walk to. + transforms: + latitude: + round: 4 + longitude: + round: 4 + + partner agency: + thing: + fields: + - id + - name + - thing_type + - well_depth + - hole_depth + - well_casing_diameter + - well_casing_depth + - well_completion_date + - well_construction_method + location: + fields: + - id + - latitude + - longitude + - elevation + - county + - state + - quad_name + - description + + # Per-destination overrides, keyed by slug. An entry here replaces the rules + # for that destination's kind rather than adding to them, so what one partner + # receives is readable in one place. + by_slug: {} + +# --------------------------------------------------------------------------- +# The OGC collections. +# +# The public mount serves 27 collections: fifteen declared in +# core/pygeoapi-config.yml, ten built from THING_COLLECTIONS in +# core/pygeoapi.py, and two EDR collections wired there as well. All of them +# are served by +# core.feature_provider.DescribedPostgreSQLProvider, which reads the allowlist +# below and hands it to pygeoapi as the provider's `properties`. A column that +# is not listed is never selected from Postgres, so it cannot appear in a +# feature, in /schema, or in /queryables. +# +# These lists were generated from what each view published on 2026-08-24, minus +# the never-public fields, so nothing anyone consumes today disappeared -- with +# two exceptions, which is the point: eleven collections were publishing +# nma_pk_welldata and one was publishing entered_by. That +# makes this file the record of what is public, not an aspiration: narrowing a +# list is a deliberate, reviewed edit, and it should happen. +# +# A collection with no entry here publishes no properties at all. That is +# default deny working as intended, and `tests/test_ogc_projection.py` fails +# when a public collection is missing, so it surfaces in CI rather than in +# production. +# +# The internal mount (ogc_internal_*) is not projected. It serves authenticated +# Bureau staff, and per-role internal field rules are the part of ADR5 3.5 that +# is not built yet. + +ogc: + # Fields no public collection may publish, whatever its list says. Same + # policy status as `never_public` above: adding is safe, removing is not an + # engineering decision. + never_public: + # Legacy AMPAPI primary keys. ogc_well_water_column published + # nma_pk_welldata until this list existed. + - nma_pk_welldata + - nma_pk_location + # Who typed the record. ogc_temp_depth_measurements published entered_by, + # a staff member's name, inherited from the NM_Wells transfer. + - created_by_id + - created_by_name + - updated_by_id + - updated_by_name + - entered_by + # Free text written by staff for staff -- gate codes, lock combinations, + # candid landowner notes. Not public today; listed so it cannot become so. + - nma_location_notes + - nma_coordinate_notes + - search_vector + + collections: + ogc_actively_monitored_wells: + - id + - name + - thing_type + - well_depth + - elevation + - elevation_method + - formation_zone + - total_water_levels + - last_water_level + - last_water_level_datetime + - min_water_level + - max_water_level + - water_level_trend_ft_per_year + - group_ids + - group_names + - group_types + + ogc_bht_measurements: + - id + - api + - well_name + - well_num + - operator + - well_type + - well_tvd + - completion_date + - current_status + - total_depth + - cuttings + - core_exists + - county + - bht_depth + - bht + - hours_since_circulation + - date_measured + + ogc_depth_to_water_trend_wells: + - id + - name + - thing_type + - record_count + - first_observation_datetime + - last_observation_datetime + - span_years + - slope_ft_per_year + - trend_category + + ogc_diversions_surface_water: + - id + - name + - first_visit_date + - last_observation_date + - well_depth + - hole_depth + - well_casing_diameter + - well_casing_depth + - well_completion_date + - well_driller_name + - well_construction_method + - well_pump_type + - well_pump_depth + - formation_completion_code + - nma_formation_zone + - release_status + - elevation + + ogc_dst: + - id + - well_name + - well_num + - api + - dst_name + - dst_operator + - dst_number + - dst_date + - county + - state + - lat_dd83 + - long_dd83 + - from_depth + - to_depth + - target_fm + - packer_from + - packer_to + - srf_choke_sz + - bot_choke_sz + - depth_unit + - elev_gl + - elev_unspc + - prs_gage_dpt + - pipe_dia + - pipe_length + - flow_history + - init_flow + - flw_prs_in_min + - fin_flow + - flw_prs_fin_min + - prs_init_clsd_in + - in_sht_in_min + - fin_shut_in + - fn_sht_in_min + - hydrost_prs_in + - hyd_st_prs_fl + - press_units + - blanked_off + - fm_temp + + ogc_ephemeral_streams: + - id + - name + - first_visit_date + - last_observation_date + - well_depth + - hole_depth + - well_casing_diameter + - well_casing_depth + - well_completion_date + - well_driller_name + - well_construction_method + - well_pump_type + - well_pump_depth + - formation_completion_code + - nma_formation_zone + - release_status + - elevation + + ogc_geothermal_wells_bht: + - id + - well_data_id + - well_name + - api + - total_depth + - bht_count + - max_bht + - min_bht + - max_bht_c + - min_bht_c + - max_bht_depth + - temp_unit + - temp_unit_source + - temp_unit_mixed + - unconvertible_count + + ogc_geothermal_wells_temperature_profile: + - id + - well_data_id + - well_name + - api + - reading_count + - min_depth + - max_depth + - min_temp + - max_temp + - min_temp_c + - max_temp_c + - temp_unit + - temp_unit_source + - temp_unit_mixed + - unconvertible_count + - series + + ogc_heat_flow: + - id + - well_name + - well_num + - api + - county + - state + - lat_dd27 + - long_dd27 + - lat_dd83 + - long_dd83 + - source_id + - elev_gl + - elev_kb + - elev_unspc + - elevation_m + - depth_units + - total_depth + - total_depth_m + - from_depth + - to_depth + - therml_cond + - tcond_range + - tcond_error + - tcond_unit + - tc_si + - sample_type + - num_samples + - therml_grad + - tgrad_range + - tg_error + - grad_unit + - heat_flow + - ht_flow_unit + - heat_flow_si + - quality + - first_auth + - pub_year + - title + - journal + - volume + - page_no + - ht_flow_est + - entry_date + - ht_flow_est_si + + ogc_lakes_ponds_reservoirs: + - id + - name + - first_visit_date + - last_observation_date + - well_depth + - hole_depth + - well_casing_diameter + - well_casing_depth + - well_completion_date + - well_driller_name + - well_construction_method + - well_pump_type + - well_pump_depth + - formation_completion_code + - nma_formation_zone + - release_status + - elevation + + ogc_latest_tds_wells: + - id + - name + - thing_type + - major_chemistry_id + - latest_tds_observation_date + - latest_tds_value + - latest_tds_units + + ogc_major_chemistry_results: + - id + - location_id + - name + - thing_type + - analyte_count + - latest_chemistry_date + - tds + - calcium + - calcium_total + - magnesium + - magnesium_total + - sodium + - sodium_total + - potassium + - potassium_total + - sodium_plus_potassium + - bicarbonate + - carbonate + - sulfate + - chloride + - ion_balance + - total_anions + - total_cations + - alkalinity + - hardness + - specific_conductance + - ph + - nitrate + - fluoride + - silica + - tds_units + - calcium_units + - calcium_total_units + - magnesium_units + - magnesium_total_units + - sodium_units + - sodium_total_units + - potassium_units + - potassium_total_units + - sodium_plus_potassium_units + - bicarbonate_units + - carbonate_units + - sulfate_units + - chloride_units + - ion_balance_units + - total_anions_units + - total_cations_units + - alkalinity_units + - hardness_units + - specific_conductance_units + - ph_units + - nitrate_units + - fluoride_units + - silica_units + + ogc_meteorological_stations: + - id + - name + - first_visit_date + - last_observation_date + - well_depth + - hole_depth + - well_casing_diameter + - well_casing_depth + - well_completion_date + - well_driller_name + - well_construction_method + - well_pump_type + - well_pump_depth + - formation_completion_code + - nma_formation_zone + - release_status + - elevation + + ogc_minor_chemistry_wells: + - id + - location_id + - name + - thing_type + - analyte_count + - latest_chemistry_date + - h2r + - o18r + - c13r + - c14 + - c14_years + - fluoride + - barium + - barium_total + - copper + - copper_total + - zinc + - zinc_total + - molybdenum + - molybdenum_total + - silica + - silicon + - silicon_total + - manganese + - manganese_total + - iron + - iron_total + - strontium + - strontium_total + - chromium + - chromium_total + - boron + - boron_total + - uranium + - uranium_total + - lithium + - lithium_total + - silver + - silver_total + - antimony + - antimony_total + - beryllium + - beryllium_total + - lead + - lead_total + - thallium + - thallium_total + - bromide + - selenium + - selenium_total + - vanadium + - vanadium_total + - aluminum + - aluminum_total + - arsenic + - arsenic_total + - nickel + - nickel_total + - cadmium + - cadmium_total + - cobalt + - cobalt_total + - phosphate + - nitrite + - nitrate + - nitrate_as_n + - thorium + - thorium_total + - tin + - tin_total + - mercury + - mercury_total + - titanium + - titanium_total + - h2r_units + - o18r_units + - c13r_units + - c14_units + - c14_years_units + - fluoride_units + - barium_units + - barium_total_units + - copper_units + - copper_total_units + - zinc_units + - zinc_total_units + - molybdenum_units + - molybdenum_total_units + - silica_units + - silicon_units + - silicon_total_units + - manganese_units + - manganese_total_units + - iron_units + - iron_total_units + - strontium_units + - strontium_total_units + - chromium_units + - chromium_total_units + - boron_units + - boron_total_units + - uranium_units + - uranium_total_units + - lithium_units + - lithium_total_units + - silver_units + - silver_total_units + - antimony_units + - antimony_total_units + - beryllium_units + - beryllium_total_units + - lead_units + - lead_total_units + - thallium_units + - thallium_total_units + - bromide_units + - selenium_units + - selenium_total_units + - vanadium_units + - vanadium_total_units + - aluminum_units + - aluminum_total_units + - arsenic_units + - arsenic_total_units + - nickel_units + - nickel_total_units + - cadmium_units + - cadmium_total_units + - cobalt_units + - cobalt_total_units + - phosphate_units + - nitrite_units + - nitrate_units + - nitrate_as_n_units + - thorium_units + - thorium_total_units + - tin_units + - tin_total_units + - mercury_units + - mercury_total_units + - titanium_units + - titanium_total_units + + ogc_outfalls_wastewater_return_flow: + - id + - name + - first_visit_date + - last_observation_date + - well_depth + - hole_depth + - well_casing_diameter + - well_casing_depth + - well_completion_date + - well_driller_name + - well_construction_method + - well_pump_type + - well_pump_depth + - formation_completion_code + - nma_formation_zone + - release_status + - elevation + + ogc_perennial_streams: + - id + - name + - first_visit_date + - last_observation_date + - well_depth + - hole_depth + - well_casing_diameter + - well_casing_depth + - well_completion_date + - well_driller_name + - well_construction_method + - well_pump_type + - well_pump_depth + - formation_completion_code + - nma_formation_zone + - release_status + - elevation + + ogc_project_areas: + - id + - name + - description + - group_type + - release_status + + ogc_rock_sample_locations: + - id + - name + - first_visit_date + - last_observation_date + - well_depth + - hole_depth + - well_casing_diameter + - well_casing_depth + - well_completion_date + - well_driller_name + - well_construction_method + - well_pump_type + - well_pump_depth + - formation_completion_code + - nma_formation_zone + - release_status + - elevation + + ogc_soil_gas_sample_locations: + - id + - name + - first_visit_date + - last_observation_date + - well_depth + - hole_depth + - well_casing_diameter + - well_casing_depth + - well_completion_date + - well_driller_name + - well_construction_method + - well_pump_type + - well_pump_depth + - formation_completion_code + - nma_formation_zone + - release_status + - elevation + + ogc_springs: + - id + - name + - first_visit_date + - last_observation_date + - well_depth + - hole_depth + - well_casing_diameter + - well_casing_depth + - well_completion_date + - well_driller_name + - well_construction_method + - well_pump_type + - well_pump_depth + - formation_completion_code + - nma_formation_zone + - release_status + - elevation + + ogc_temp_depth_measurements: + - id + - well_name + - well_num + - api + - source_id + - sample_fm + - county + - state + - lat_dd27 + - long_dd27 + - lat_dd83 + - long_dd83 + - loc_acc_val + - entry_date + - depth + - depth_unit + - temp + - temp_unit + - elev_gl + - elev_unspc + - elev_kb + - sample_date + + ogc_water_chemistry: + - id + - thing_id + - station_name + - thing_type + - longitude + - latitude + - datetime + - value + - unit + - parameter_name + - sample_id + - release_status + + ogc_water_elevation_wells: + - id + - name + - thing_type + - observation_id + - observation_datetime + - elevation_m + - depth_to_water_below_ground_surface_ft + - water_elevation_ft + + ogc_water_well_summary: + - id + - name + - well_depth + - elevation + - elevation_method + - formation_zone + - total_water_levels + - last_water_level + - last_water_level_datetime + - min_water_level + - max_water_level + - water_level_trend_ft_per_year + + ogc_water_wells: + - id + - name + - first_visit_date + - last_observation_date + - well_depth + - hole_depth + - well_casing_diameter + - well_casing_depth + - well_completion_date + - well_driller_name + - well_construction_method + - well_pump_type + - well_pump_depth + - formation_completion_code + - nma_formation_zone + - release_status + - elevation + + ogc_waterlevels: + - id + - thing_id + - station_name + - longitude + - latitude + - datetime + - value + - unit + - parameter_name + - source + - deployment_id + - release_status + + ogc_well_water_column: + - id + - name + - first_visit_date + - well_depth + - hole_depth + - well_casing_diameter + - well_casing_depth + - well_completion_date + - well_driller_name + - well_construction_method + - well_pump_type + - well_pump_depth + - formation_completion_code + - nma_formation_zone + - release_status + - water_column_latest + - water_column_average + - water_column_maximum + - water_column_minimum + - elevation diff --git a/core/initializers.py b/core/initializers.py index 9f419caa2..a35506f40 100644 --- a/core/initializers.py +++ b/core/initializers.py @@ -227,7 +227,9 @@ def register_api_routes(app): from api.geothermal import router as geothermal_router from api.chemisty import router as chemistry_router from api.gis_artifacts import router as gis_artifacts_router + from api.access import router as access_router + app.include_router(access_router) app.include_router(asset_router) app.include_router(chemistry_router) app.include_router(author_router) diff --git a/core/lexicon.json b/core/lexicon.json index 40291d696..5046b178d 100644 --- a/core/lexicon.json +++ b/core/lexicon.json @@ -247,6 +247,26 @@ { "name": "data_maturity", "description": "How far through review a measurement is, on USGS terms. Orthogonal to release_status, which controls visibility rather than trust." + }, + { + "name": "principal_type", + "description": "What kind of thing an access grant is made to: a person, a role, or an API key. Destinations are not principals -- publishing is recorded as consent (ADR5)." + }, + { + "name": "capability", + "description": "What a principal may do: read, enter measurements, correct records, administer grants (ADR5)." + }, + { + "name": "grant_scope_type", + "description": "How far a grant reaches: the whole institution, one group/project, or one thing (ADR5)." + }, + { + "name": "access_data_type", + "description": "The kind of data a grant or a publication consent names. Deliberately coarse, and deliberately without a term meaning 'all' (ADR5, PUB-U11)." + }, + { + "name": "destination_kind", + "description": "What sort of place published data is offered to: the public web, a harvesting agency, a partner with a standing connection (ADR5)." } ], "terms": [ @@ -1808,6 +1828,13 @@ "term": "private", "definition": "private version" }, + { + "categories": [ + "release_status" + ], + "term": "embargoed", + "definition": "withheld from public release until release_at, then published by the scheduled release job" + }, { "categories": [ "relation" @@ -1859,7 +1886,8 @@ }, { "categories": [ - "activity_type" + "activity_type", + "access_data_type" ], "term": "water chemistry", "definition": "water chemistry" @@ -8508,6 +8536,118 @@ ], "term": "in review", "definition": "Under review and not yet approved. Intermediate state from the USGS Aquarius approval levels used for continuous records." + }, + { + "categories": [ + "principal_type" + ], + "term": "user", + "definition": "A person, referenced by their Authentik subject identifier." + }, + { + "categories": [ + "principal_type" + ], + "term": "role", + "definition": "An Authentik group, carrying the stable institutional part of access." + }, + { + "categories": [ + "principal_type" + ], + "term": "api key", + "definition": "A key issued to a client that has no person behind it." + }, + { + "categories": [ + "capability" + ], + "term": "read", + "definition": "May see the data." + }, + { + "categories": [ + "capability" + ], + "term": "enter", + "definition": "May record new measurements." + }, + { + "categories": [ + "capability" + ], + "term": "correct", + "definition": "May revise existing records." + }, + { + "categories": [ + "capability" + ], + "term": "administer", + "definition": "May grant and revoke access." + }, + { + "categories": [ + "grant_scope_type" + ], + "term": "global", + "definition": "Reaches every thing. Used for the broad role grants that preserve today's access." + }, + { + "categories": [ + "grant_scope_type" + ], + "term": "group", + "definition": "Reaches the things in one group -- a project, a network." + }, + { + "categories": [ + "grant_scope_type" + ], + "term": "thing", + "definition": "Reaches exactly one thing." + }, + { + "categories": [ + "access_data_type" + ], + "term": "water level", + "definition": "Depth-to-water readings, manual and continuous." + }, + { + "categories": [ + "access_data_type" + ], + "term": "well construction", + "definition": "Casing, screen, borehole and completion detail." + }, + { + "categories": [ + "access_data_type" + ], + "term": "site metadata", + "definition": "Where the thing is and what it is: name, location, type, status." + }, + { + "categories": [ + "destination_kind" + ], + "term": "public web", + "definition": "Offered to anyone, unauthenticated." + }, + { + "categories": [ + "destination_kind" + ], + "term": "harvester", + "definition": "A machine consumer that pulls on a schedule, such as NGWMN." + }, + { + "categories": [ + "destination_kind" + ], + "term": "partner agency", + "definition": "A named organization reading through a standing connection." } ] -} \ No newline at end of file +} diff --git a/db/__init__.py b/db/__init__.py index 4e2e7fb3a..c4475b450 100644 --- a/db/__init__.py +++ b/db/__init__.py @@ -31,13 +31,16 @@ from db.aquifer_system import * from db.aquifer_type import * from db.asset import * +from db.authorization_audit import * from db.base import * from db.base import Base from db.collabnet import * from db.contact import * from db.data_provenance import * from db.deployment import * +from db.destination import * from db.field import * +from db.field_access_consent import * from db.geochronology import * from db.geologic_formation import * from db.geothermal import * @@ -48,8 +51,9 @@ from db.notes import * from db.observation import * from db.parameter import * -from db.permission_history import * +from db.permission_grant import * from db.publication import * +from db.publication_consent import * from db.regulatory_limit import * from db.sample import * from db.sensor import * diff --git a/db/authorization_audit.py b/db/authorization_audit.py new file mode 100644 index 000000000..7faebec61 --- /dev/null +++ b/db/authorization_audit.py @@ -0,0 +1,79 @@ +# =============================================================================== +# Copyright 2025 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +db/authorization_audit.py + +The append-only log of authorization events: grants, revocations, consent +captured or withdrawn, destinations registered or retired, embargoes lifted. + +ADR5 asks for this from the first commit, for one reason: when something is +exposed that should not have been, the first question is never "what was the +value". It is "who granted that, and when". + +Append only. Nothing in the application updates or deletes a row here, and a +database-level backstop for writes that bypass the application is still owed +(ADR5, 4.3). ``AuditMixin.created_at`` is the event time; there is no second +timestamp column to disagree with it. + +Separate from sqlalchemy-continuum's versioning, which records data history. +Authorization changes are a different, higher-value target. +""" + +from typing import Optional + +from sqlalchemy import JSON, Integer, String +from sqlalchemy.orm import Mapped, mapped_column + +from db.base import Base, AutoBaseMixin + +# Event types. Strings rather than lexicon terms: the log must be able to +# record an event whose vocabulary row was itself just deleted. +GRANT_CREATED = "grant.created" +GRANT_REVOKED = "grant.revoked" +CONSENT_RECORDED = "consent.recorded" +CONSENT_REVOKED = "consent.revoked" +DESTINATION_REGISTERED = "destination.registered" +# An embargo lifting is an authorization change like any other -- it widens +# who may see a record -- so it lands in the same log rather than a second one +# nobody thinks to read after an incident. +RELEASE_LIFTED = "release.lifted" + + +class AuthorizationAudit(Base, AutoBaseMixin): + """One authorization event, as it happened.""" + + event_type: Mapped[str] = mapped_column(String(50), nullable=False) + + # Who did it, as the identifier the application had at the time. + actor: Mapped[str] = mapped_column(String(255), nullable=False) + + # What it happened to: table name and row id, kept as loose values so a + # deleted row does not take its own audit trail with it. + subject_table: Mapped[str] = mapped_column(String(50), nullable=False) + subject_id: Mapped[Optional[int]] = mapped_column(Integer, nullable=True) + + # The event's own fields, whatever they were. What each event should + # record is open (PERM-U11); recording the payload keeps the question + # answerable later rather than losing it now. + detail: Mapped[Optional[dict]] = mapped_column(JSON, nullable=True) + + def __str__(self): + return ( + f"{self.event_type} {self.subject_table}:{self.subject_id} by {self.actor}" + ) + + +# ============= EOF ============================================= diff --git a/db/base.py b/db/base.py index e9b0d7f2b..892868bff 100644 --- a/db/base.py +++ b/db/base.py @@ -29,7 +29,7 @@ - `ReleaseMixin`: Adds a release status column referencing the `lexicon_term` table. - `AuditMixin`: Adds standard audit columns (created_at, created_by, updated_at, updated_by). 5. A simple `User` model for tracking user information in audit columns. -6. Polymorphic helper mixins (`StatusHistoryMixin`, `NotesMixin`, `DataProvenanceMixin`, `PermissionMixin`.) +6. Polymorphic helper mixins (`StatusHistoryMixin`, `NotesMixin`, `DataProvenanceMixin`.) which provide a clean, reusable way to add relationships to the polymorphic metadata tables. Any model that can have a status history (like Thing or Location) can simply inherit from the `StatusHistoryMixin` mixin. @@ -41,6 +41,7 @@ from sqlalchemy import ( Column, + Date, DateTime, func, Integer, @@ -53,7 +54,6 @@ declared_attr, Mapped, mapped_column, - relationship, ) from sqlalchemy_continuum import make_versioned from sqlalchemy_searchable import make_searchable @@ -95,12 +95,47 @@ def pascal_to_snake(name): # ============= Common Mixins ============================================= class ReleaseMixin: - """Mixin to add release status to a model.""" + """Mixin to add release state to a model. + + Two axes, deliberately separate (ADR5): + + * ``release_status`` is the release *level* -- who may see the record at + all (draft, public, private, ...), lexicon-backed. + * ``data_maturity`` is how far through review the record is, on USGS + terms: provisional, in review, approved. It is orthogonal to the level: + San Acacia data is published and provisional at the same time, and one + column could not say both, because ``release_status``'s lexicon lists + those as siblings. + + ``data_maturity`` is nullable and NULL means not stated, which is honest + for the rows that predate it. The vocabulary and the column originated on + ``TransducerObservation``; this mixin is where it now lives so that every + released record can carry the second axis. + + ``release_at`` is the third, and it is *intent* rather than enforcement: + the date an ``embargoed`` record becomes ``public``. Nothing on the read + path consults it. ``services/release_schedule.py`` flips the level when + the date arrives, and ``release_status`` stays the only thing the OGC + views filter on. See ``docs/data-embargo.md``. + + NULL means no embargo, and that default is load-bearing rather than + incidental: migration ``w1x2y3z4a5b6`` records three NGWMN exports that + a release predicate emptied outright because the column it tested + defaulted to something other than "released". + """ @declared_attr def release_status(self): return lexicon_term(default="draft") + @declared_attr + def data_maturity(self): + return lexicon_term(nullable=True) + + @declared_attr + def release_at(self): + return mapped_column(Date, nullable=True) + class AuditMixin: """Mixin to add standard audit columns to a model.""" @@ -176,28 +211,6 @@ def properties(self): ) -# ============= Polymorphic Helper Mixins ============================================= - - -class PermissionMixin: - """ - Mixin for models that can have permissions (e.g., Thing, Location). - It automatically creates a polymorphic One-to-Many relationship to the - Permission table. - """ - - @declared_attr - def permissions(self): - # One-to-Many polymorphic relationship - return relationship( - "Permission", - primaryjoin=f"and_({self.__name__}.id==foreign(Permission.permissible_id), " - f"Permission.permissible_type=='{self.__name__}')", - lazy="selectin", - viewonly=True, - ) - - class User(Base): """Represents a user in the system.""" diff --git a/db/contact.py b/db/contact.py index 0fb594732..f1d68c410 100644 --- a/db/contact.py +++ b/db/contact.py @@ -27,7 +27,7 @@ from db.field import FieldEventParticipant, FieldEvent from db.thing import Thing from db.publication import Author, AuthorContactAssociation - from db.permission_history import PermissionHistory + from db.field_access_consent import FieldAccessConsent class ThingContactAssociation(Base, AutoBaseMixin): @@ -75,8 +75,8 @@ class Contact(Base, AutoBaseMixin, ReleaseMixin, NotesMixin): ) # One-To-Many: A Contact can grant many Permissions. - permissions: Mapped[List["PermissionHistory"]] = relationship( - "PermissionHistory", + permissions: Mapped[List["FieldAccessConsent"]] = relationship( + "FieldAccessConsent", back_populates="contact", cascade="all, delete, delete-orphan", ) diff --git a/db/destination.py b/db/destination.py new file mode 100644 index 000000000..c035fad15 --- /dev/null +++ b/db/destination.py @@ -0,0 +1,55 @@ +# =============================================================================== +# Copyright 2025 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +db/destination.py + +The registry of places data goes: the anonymous public web, a federal +harvester, a partner agency's standing connection. + +A destination is not a user and not a role. It is the other half of a +publication consent row -- the "to whom" a landowner agreed to. Adding a new +one is a registry entry plus a set of consent rows, which is the point of +having a registry at all (ADR5, Part IV). +""" + +from typing import Optional + +from sqlalchemy import String, Text, Boolean +from sqlalchemy.orm import Mapped, mapped_column + +from db.base import Base, AutoBaseMixin, lexicon_term + + +class Destination(Base, AutoBaseMixin): + """One place published data is offered to.""" + + # Stable, URL-safe handle. Routes and consent records refer to a + # destination by slug rather than id so a fixture, a config file, and an + # operator all name it the same way. + slug: Mapped[str] = mapped_column(String(50), nullable=False, unique=True) + name: Mapped[str] = mapped_column(String(255), nullable=False) + destination_kind: Mapped[str] = lexicon_term(nullable=False) + description: Mapped[Optional[str]] = mapped_column(Text, nullable=True) + + # Retiring a destination stops it being offered without deleting the + # consent history that says what was once agreed to. + active: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) + + def __str__(self): + return self.slug + + +# ============= EOF ============================================= diff --git a/db/permission_history.py b/db/field_access_consent.py similarity index 60% rename from db/permission_history.py rename to db/field_access_consent.py index 0e1526e2a..3a0ef09d7 100644 --- a/db/permission_history.py +++ b/db/field_access_consent.py @@ -1,10 +1,17 @@ """ -models/permission.py +db/field_access_consent.py -This model defines the `Permission` table, a polymorphic table that tracks -all legal and administrative agreements related to site access and activity. -Its purpose is to track who granted permission, what activities they authorized, -which entity the permission applies to, and for what period of time. +Landowner consent to physical site access, recorded per Thing or Location. + +This is a domain fact, not authorization. A row says a contact agreed to let +the Bureau do something at their well -- sample it, install equipment -- for +some period of time. Nothing here decides what an API caller may see or write; +that is the grant model described in ADR5, which is a separate table with +separate governance. + +The table was named `permission_history` until ADR5. The `permission_type` +and `permission_allowed` column names are kept because `permission_type` is +lexicon-backed and both are published in Thing responses as-is. """ from datetime import date @@ -21,10 +28,12 @@ from db.location import Location -class PermissionHistory(Base, AutoBaseMixin, ReleaseMixin): +class FieldAccessConsent(Base, AutoBaseMixin, ReleaseMixin): """ - Represents a specific grant of permission from a Contact for a - specific entity (e.g., a Thing or Location). + One consent record: a Contact agreed (or declined) to a type of field + activity at a specific entity (a Thing or a Location), over a date range. + + Not an access-control grant. See ADR5. """ # --- Foreign Keys --- @@ -52,14 +61,14 @@ class PermissionHistory(Base, AutoBaseMixin, ReleaseMixin): # They tell SQLAlchemy exactly how to find the specific parent record for a given child. _thing_target: Mapped["Thing"] = relationship( "Thing", - primaryjoin="and_(foreign(PermissionHistory.target_id) == Thing.id, " - "PermissionHistory.target_table == 'thing')", + primaryjoin="and_(foreign(FieldAccessConsent.target_id) == Thing.id, " + "FieldAccessConsent.target_table == 'thing')", viewonly=True, ) _location_target: Mapped["Location"] = relationship( "Location", - primaryjoin="and_(foreign(PermissionHistory.target_id) == Location.id, " - "PermissionHistory.target_table == 'location')", + primaryjoin="and_(foreign(FieldAccessConsent.target_id) == Location.id, " + "FieldAccessConsent.target_table == 'location')", viewonly=True, ) @@ -73,22 +82,22 @@ def target(self): return getattr(self, f"_{self.target_table}_target") -class PermissionHistoryMixin: +class FieldAccessConsentMixin: """ - Mixin for models that can have permissions (e.g., Thing, Location). + Mixin for models a landowner can consent about (e.g., Thing, Location). It automatically creates a polymorphic One-to-Many relationship to the - Permission table. + field_access_consent table. """ @declared_attr - def permission_history(cls): + def field_access_consent(cls): # One-to-Many polymorphic relationship return relationship( - "PermissionHistory", + "FieldAccessConsent", primaryjoin=( and_( - cls.id == foreign(PermissionHistory.target_id), - PermissionHistory.target_table == cls.__tablename__, + cls.id == foreign(FieldAccessConsent.target_id), + FieldAccessConsent.target_table == cls.__tablename__, ) ), lazy="selectin", diff --git a/db/notes.py b/db/notes.py index 3c238fbf3..390dc8607 100644 --- a/db/notes.py +++ b/db/notes.py @@ -110,6 +110,8 @@ def add_note( content: str, note_type: str, release_status: str = "draft", + data_maturity: str = None, + release_at=None, created_by: str = None, ) -> Notes: """ @@ -123,6 +125,8 @@ def add_note( target_id=self.id, target_table=self.__class__.__tablename__, release_status=release_status, + data_maturity=data_maturity, + release_at=release_at, ) def _get_notes(self, note_type: str) -> list[Notes]: diff --git a/db/permission.py b/db/permission.py deleted file mode 100644 index c4ea2c85c..000000000 --- a/db/permission.py +++ /dev/null @@ -1,81 +0,0 @@ -""" -models/permission.py - -This model defines the `Permission` table, a polymorphic table that tracks -all legal and administrative agreements related to site access and activity. -Its purpose is to track who granted permission, what activities they authorized, -which entity the permission applies to, and for what period of time. -""" - -from typing import TYPE_CHECKING - -from sqlalchemy import ( - Integer, - ForeignKey, - String, - Boolean, - Date, - Text, -) -from sqlalchemy.orm import relationship, Mapped, mapped_column - -from db.base import Base, AutoBaseMixin, ReleaseMixin - -if TYPE_CHECKING: - from db.contact import Contact - from db.thing import Thing - from db.location import Location - - -class Permission(Base, AutoBaseMixin, ReleaseMixin): - """ - Represents a specific grant of permission from a Contact for a - specific entity (e.g., a Thing or Location). - """ - - # --- Foreign Keys --- - contact_id: Mapped[int] = mapped_column( - Integer, ForeignKey("contact.id"), nullable=False - ) - - # --- Columns --- - allow_sampling: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) - allow_installation: Mapped[bool] = mapped_column( - Boolean, nullable=False, default=False - ) - start_date: Mapped[Date] = mapped_column(Date, nullable=True) - end_date: Mapped[Date] = mapped_column(Date, nullable=True) - notes: Mapped[str] = mapped_column(Text, nullable=True) - - # --- Polymorphic Columns --- - permissible_id: Mapped[int] = mapped_column(Integer, nullable=False) - permissible_type: Mapped[str] = mapped_column(String(50), nullable=False) - - # --- Relationships --- - # Many-To-One: A Permission is granted by one Contact. - contact: Mapped["Contact"] = relationship("Contact", back_populates="permissions") - - # --- Polymorphic Parent Relationships (Internal) --- - # These are view-only relationships used by the 'target' property below. - # They tell SQLAlchemy exactly how to find the specific parent record for a given child. - _thing_target: Mapped["Thing"] = relationship( - "Thing", - primaryjoin="and_(foreign(Permission.permissible_id) == Thing.id, " - "Permission.permissible_type == 'Thing')", - viewonly=True, - ) - _location_target: Mapped["Location"] = relationship( - "Location", - primaryjoin="and_(foreign(Permission.permissible_id) == Location.id, " - "Permission.permissible_type == 'Location')", - viewonly=True, - ) - - @property - def target(self): - """ - A generic property to get the parent object (Thing, Location, etc.). - This is useful for simplifying application code by providing a single, - consistent way to access the parent of a polymorphic record. - """ - return getattr(self, f"_{self.permissible_type.lower()}_target") diff --git a/db/permission_grant.py b/db/permission_grant.py new file mode 100644 index 000000000..daa19c7ce --- /dev/null +++ b/db/permission_grant.py @@ -0,0 +1,98 @@ +# =============================================================================== +# Copyright 2025 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +db/permission_grant.py + +Internal authorization: may this principal exercise this capability within +this scope? (ADR5, Part III.) + +This is institutional trust, decided per person by data services staff. The +landowner's half -- what an owner agreed to publish about their well -- is +``db/publication_consent.py``, a separate table with separate governance and +the same grammar. Both are evaluated by ``services/visibility.py``. + +Invariants, enforced in ``domain/access.py`` before a row is written: + +* ``data_type`` is never null and there is no term meaning "all", so a data + type added later is not covered by an existing grant. +* A ``global`` grant carries no ``scope_id``; a ``group`` or ``thing`` grant + requires one. +* Expiry is read at use. Nothing sweeps this table, so a missed job cannot + leave a grant standing past its end date. +""" + +from datetime import date, datetime +from typing import Optional + +from sqlalchemy import DateTime, Index, Integer, String, Text +from sqlalchemy.orm import Mapped, mapped_column + +from db.base import Base, AutoBaseMixin, lexicon_term + + +class PermissionGrant(Base, AutoBaseMixin): + """One grant: principal x capability x scope, over one data type.""" + + # --- Principal --- + # principal_id is a stable identifier whose meaning depends on the type: + # an Authentik subject for `user`, a group name for `role`, a key label + # for `api key`. It is a string, not a foreign key, because Authentik owns + # identity and Ocotillo owns authorization (ADR5, A.4). + principal_type: Mapped[str] = lexicon_term(nullable=False) + principal_id: Mapped[str] = mapped_column(String(255), nullable=False) + + # --- What and where --- + capability: Mapped[str] = lexicon_term(nullable=False) + scope_type: Mapped[str] = lexicon_term(nullable=False) + scope_id: Mapped[Optional[int]] = mapped_column(Integer, nullable=True) + data_type: Mapped[str] = lexicon_term(nullable=False) + + # --- When --- + starts_at: Mapped[date] = mapped_column(nullable=False) + ends_at: Mapped[Optional[date]] = mapped_column(nullable=True) + + # --- On whose authority --- + # "Who granted that, and when" is the first question after an incident, so + # it is a column rather than something to reconstruct from the audit log. + granted_by: Mapped[str] = mapped_column(String(255), nullable=False) + reason: Mapped[Optional[str]] = mapped_column(Text, nullable=True) + + # --- Revocation --- + # Revoking sets these rather than deleting the row: the record that access + # once existed is the point. Effective at the next read, never backdated. + revoked_at: Mapped[Optional[datetime]] = mapped_column( + DateTime(timezone=True), nullable=True + ) + revoked_by: Mapped[Optional[str]] = mapped_column(String(255), nullable=True) + + __table_args__ = ( + # The read path always starts from "who is asking". + Index( + "ix_permission_grant_principal", + "principal_type", + "principal_id", + "capability", + ), + ) + + def __str__(self): + return ( + f"{self.principal_type}:{self.principal_id} may {self.capability} " + f"{self.data_type} ({self.scope_type})" + ) + + +# ============= EOF ============================================= diff --git a/db/publication_consent.py b/db/publication_consent.py new file mode 100644 index 000000000..2ce59594f --- /dev/null +++ b/db/publication_consent.py @@ -0,0 +1,102 @@ +# =============================================================================== +# Copyright 2025 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +db/publication_consent.py + +The landowner's half of ADR5: did the owner of this well agree to publish this +data type to this destination? + +Kept apart from ``db/permission_grant.py`` on purpose. Both reduce to the same +grammar and both are evaluated by ``services/visibility.py``, but they are +decided by different people on different authority and revoked by different +events -- a phone call here, an HR-shaped event there. One engine, two tables. + +An owner willing to share water levels but not chemistry is one row, not a +workaround. +""" + +from datetime import date, datetime +from typing import TYPE_CHECKING, Optional + +from sqlalchemy import DateTime, ForeignKey, Index, Integer, String, Text, text +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from db.base import Base, AutoBaseMixin, lexicon_term + +if TYPE_CHECKING: + from db.contact import Contact + from db.destination import Destination + from db.thing import Thing + + +class PublicationConsent(Base, AutoBaseMixin): + """One consent: this thing's data type is offered to this destination.""" + + # --- What is published, and where --- + thing_id: Mapped[int] = mapped_column( + Integer, ForeignKey("thing.id", ondelete="CASCADE"), nullable=False + ) + destination_id: Mapped[int] = mapped_column( + Integer, ForeignKey("destination.id"), nullable=False + ) + data_type: Mapped[str] = lexicon_term(nullable=False) + + # --- Who agreed --- + # Nullable because the Bureau owns some of the wells it monitors, and + # inventing a consenting contact for those would be a lie. NULL means the + # decision was institutional; `recorded_by` still says who made it. + contact_id: Mapped[Optional[int]] = mapped_column( + Integer, ForeignKey("contact.id"), nullable=True + ) + recorded_by: Mapped[str] = mapped_column(String(255), nullable=False) + notes: Mapped[Optional[str]] = mapped_column(Text, nullable=True) + + # --- When --- + starts_at: Mapped[date] = mapped_column(nullable=False) + ends_at: Mapped[Optional[date]] = mapped_column(nullable=True) + + # --- Revocation --- + # "Unpublish" means "stop offering". Copies a harvester already took live + # in someone else's system, and owners are told so rather than promised a + # recall the Bureau does not have (ADR5, 3.6). + revoked_at: Mapped[Optional[datetime]] = mapped_column( + DateTime(timezone=True), nullable=True + ) + revoked_by: Mapped[Optional[str]] = mapped_column(String(255), nullable=True) + + # --- Relationships --- + thing: Mapped["Thing"] = relationship("Thing", viewonly=True) + destination: Mapped["Destination"] = relationship("Destination", viewonly=True) + contact: Mapped[Optional["Contact"]] = relationship("Contact", viewonly=True) + + __table_args__ = ( + # One live row per (thing, destination, data type). Revoked rows stay + # for the history, so the constraint has to ignore them. + Index( + "uq_publication_consent_live", + "thing_id", + "destination_id", + "data_type", + unique=True, + postgresql_where=text("revoked_at IS NULL"), + ), + ) + + def __str__(self): + return f"thing {self.thing_id} -> destination {self.destination_id} ({self.data_type})" + + +# ============= EOF ============================================= diff --git a/db/thing.py b/db/thing.py index 01aed09ed..8f15cf066 100644 --- a/db/thing.py +++ b/db/thing.py @@ -30,7 +30,7 @@ ) from db.data_provenance import DataProvenanceMixin from db.measuring_point_history import MeasuringPointHistory -from db.permission_history import PermissionHistoryMixin +from db.field_access_consent import FieldAccessConsentMixin from db.status_history import StatusHistoryMixin from services.util import retrieve_latest_polymorphic_history_table_record @@ -63,7 +63,7 @@ class Thing( AutoBaseMixin, ReleaseMixin, StatusHistoryMixin, - PermissionHistoryMixin, + FieldAccessConsentMixin, DataProvenanceMixin, NotesMixin, ): @@ -598,8 +598,8 @@ def permissions(self) -> list: associated permissions, an empty list is returned instead of None to allow the API to serialize correctly (see schemas/thing.py). """ - if self.permission_history: - return self.permission_history + if self.field_access_consent: + return self.field_access_consent else: return [] diff --git a/db/transducer.py b/db/transducer.py index e129cfe60..699f7c650 100644 --- a/db/transducer.py +++ b/db/transducer.py @@ -180,14 +180,9 @@ class TransducerObservation(Base, AutoBaseMixin, ReleaseMixin): comment="Per-reading correction annotation; NULL means the value is as measured", ) - # How far through review this reading is, on USGS terms: provisional, - # in review, approved. Orthogonal to `release_status`, which says who may - # see it -- a reading can be public and provisional at once, which one - # column could not express because its lexicon lists those as siblings. - # - # Nullable because legacy rows predate it and nobody has established - # whether they are approved. NULL means not stated, which is honest. - data_maturity: Mapped[str] = lexicon_term(nullable=True) + # data_maturity (provisional / in review / approved) comes from + # ReleaseMixin, which is where the axis was generalized in ADR5. It started + # here; the semantics are unchanged. nma_waterlevelscontinuous_pressure_conddl_ms_cm: Mapped[float] = mapped_column( Float, nullable=True ) diff --git a/docs/access-field-projection.md b/docs/access-field-projection.md new file mode 100644 index 000000000..944ec5e00 --- /dev/null +++ b/docs/access-field-projection.md @@ -0,0 +1,124 @@ +# Field projection: what an audience actually receives + +Read this before editing `core/field-allowlists.yml`, adding a projectable +entity, or wiring a new consumer into published payloads. + +The decision behind it is [ADR5](../ADR5.md), sections 3.5 and A.2. + +## The rule + +Fields are published by **allowlist, per audience, at one chokepoint**. + +- Only fields named for an audience appear in that audience's payload. + Omission produces silence, not leakage: a column added next year is + invisible to everyone outside the Bureau until someone lists it. +- An audience with no entry receives an **empty record**, not the whole one. +- Protection includes **transformation**, not only removal. A public record can + carry a coordinate rounded to protect the landowner while the precise value + stays internal. +- The **never-public list wins** over every allowlist, and is applied twice: + when the configuration loads, and again when a record is projected. + +## Where the pieces are + +| Piece | Path | Job | +| --- | --- | --- | +| Rules | `domain/field_projection.py` | Omit, transform, validate. Plain dicts; no database, no config parsing. | +| Configuration + chokepoint | `services/field_projection.py` | Parse and validate the YAML, build a record from a model row, project it. | +| Configuration | `core/field-allowlists.yml` | The allowlists and the never-public list. | +| Destination payloads | `services/visibility.py` (`published_things`) | Builds them through `project_entity`. | +| Public OGC collections | `core/feature_provider.py` | Reads `ogc_allowlist(table)` at provider construction. | + +`api/access.py` never projects anything itself. That is the point: the +projection sits *below* the routes, so a new route or a new output format +cannot skip it by forgetting to call it. + +## Adding a field for an audience + +1. Add it to that audience's `fields` list in `core/field-allowlists.yml`, + under `audiences.by_kind.` or `audiences.by_slug.`. +2. Run the tests. The configuration is validated on load, so a typo, a field + the entity does not have, or a field on the never-public list fails + immediately rather than quietly withholding or exposing data. + +A `by_slug` entry **replaces** the rules for that destination's kind rather +than extending them, so what one partner receives is readable in one place. + +## Adding a transform + +Transforms live in `TRANSFORMS` in `domain/field_projection.py` and are named +in the YAML as `field: {transform: argument}`. Only `round` exists today: + +```yaml +transforms: + latitude: + round: 2 +``` + +Two decimal places is roughly a kilometre; four is roughly ten metres. A +transform on a field that is not in the same allowlist raises at load, because +it would never run. + +## The never-public list + +`never_public` in the YAML is the list no configuration can override. +Engineering guarantees that whatever is listed is enforced everywhere. +Engineering **cannot decide what belongs on it** — that needs a named data +owner, and ADR5 records that nobody has been named yet. + +What that means in practice: + +- **Adding** a field to `never_public` is safe at any time and needs no + ceremony. +- **Removing** one is not an engineering call. If a removal seems necessary, + the question goes to the data owner, not into a pull request. + +Currently listed: provenance columns (`created_by_*`, `updated_by_*`), legacy +AMPAPI primary keys, and the free-text location and coordinate note columns — +the permissions interview was explicit that gate codes, lock combinations and +candid landowner notes must never leave the Bureau, and that is the text those +columns have collected. + +## The OGC collections + +The public `/ogcapi` mount serves 27 collections from three places: fifteen +declared in `core/pygeoapi-config.yml`, ten built from `THING_COLLECTIONS` in +`core/pygeoapi.py`, and two EDR collections wired there too. Missing that third +source is how eleven collections published `nma_pk_welldata` unnoticed. + +`DescribedPostgreSQLProvider` looks each table up in the `ogc:` block of +`core/field-allowlists.yml` and hands the result to pygeoapi as the provider's +`properties`. That is what `_select_properties` builds the SELECT from, so an +unlisted column is never read out of Postgres at all: it cannot appear in a +feature, in `/schema`, or in `/queryables`, and a filter cannot probe for it. +`get_fields` is narrowed to match, since the reflection sees the whole view. + +**Adding a collection** means adding its table to `ogc.collections`. Without an +entry it publishes *no* properties — default deny, logged as a warning — and +`tests/test_ogc_projection.py` fails, so a missing entry shows up in CI rather +than in production. + +**The internal mount is not projected.** `ogc_internal_*` tables return `None` +from `ogc_allowlist` and pass through whole. It serves authenticated Bureau +staff, and per-role internal field rules are the part of ADR5 3.5 that is not +built. + +The lists were generated from what each view published on 2026-08-24 minus the +never-public fields, so no consumer lost a field it was using. That makes the +file a record of what is public rather than an aspiration — narrowing it is +the next deliberate step, and it should happen. + +## What this does *not* cover + +The desktop-GIS layer files in `core/gis-curated-layers.yml` are not a separate +publication path — they point clients at the OGC collections above and so +inherit the projection. + +Only `thing` and `location` are projectable entities for destination payloads. Adding another means +adding it to `ENTITY_MODELS` (and `DERIVED_FIELDS` if the payload carries +values that are not columns, the way `latitude` and `longitude` stand in for +the PostGIS `point`). + +Field rules are per *audience* today. ADR5 also asks for field rules between +internal roles — contact information being AMP-only is one — and that case is +not implemented. diff --git a/docs/data-embargo.md b/docs/data-embargo.md new file mode 100644 index 000000000..262f4c9bc --- /dev/null +++ b/docs/data-embargo.md @@ -0,0 +1,152 @@ +# Embargoing data until a date + +An embargo is a record withheld from public release until a date decided in +advance. It is two columns on every `ReleaseMixin` table: + +| column | meaning | +| --- | --- | +| `release_status = 'embargoed'` | the record is being withheld | +| `release_at` | the date it becomes public | + +When the date arrives, `oco release-embargoed --apply` flips the level to +`public`. The record then appears in the public OGC collections on their next +refresh. + +## The two rules worth knowing before you touch this + +**`release_at` is intent. `release_status` is enforcement.** Nothing on the +read path consults `release_at` — not the OGC views, not the API, not the +visibility layer. It exists so a job can change `release_status` on the right +day. If you find yourself adding a date comparison to a read path, stop: that +is the distributed filtering ADR5 exists to prevent, and migration +`baba91fe5e83` is what it already cost this repository once. + +**An embargo only ever widens visibility.** The scheduled path turns +`embargoed` into `public`, and does nothing else. Withdrawing something already +published is an immediate change to `release_status`, made by a person — never +scheduled. This mirrors `domain/access.py`, where a revocation takes effect at +once and is never backdated: a promise to hide something *later* is not a +promise anyone should rely on. + +## Why a job and not a predicate in the views + +The obvious design is `WHERE release_at IS NULL OR release_at <= current_date` +in the OGC views, so that no job is needed and the embargo lifts itself at read +time. It was rejected. + +Seven of the public collections are **materialized** views, refreshed by one +pg_cron job at 09:00 UTC (`docs/pg_cron-nightly-refresh.md`). `current_date` +inside a materialized view is frozen at refresh time. A date predicate would +therefore buy nothing on more than half the public surface, while costing a +recreation of every relation that carries it. The refresh already sets the +granularity, so the cheaper mechanism with the same behaviour wins. + +Consequence to be honest about: **an embargo lifts up to a day late**, bounded +by when the release job and the refresh run. It never lifts early. + +## Running it + +```bash +oco release-embargoed +``` + +Previews by default, like `oco seed-access-grants`; add `--apply` to write. +Idempotent — a released record is no longer embargoed, so a second run finds +nothing. Every flip writes an `authorization_audit` row recording the date the +embargo was set for alongside the date it was actually lifted, so "was this +released early" stays answerable. + +**Ordering matters.** Run it *before* the 09:00 UTC materialized-view refresh. +A record released after the refresh waits another day to appear. + +### It is not scheduled yet + +The CLI exists; nothing runs it on a timer. Until that is wired, an embargo +lifts when someone runs the command. Two ways to wire it, neither free: + +- **App Engine cron / Cloud Scheduler → an HTTP route.** Fits the existing + deployment, but means adding a route whose effect is publishing data. That + route needs its own authorization decision — `X-Appengine-Cron` alone is a + header, not an authenticator, once anything else can reach the app. +- **pg_cron, next to the refresh job.** The scheduler is already there and the + ordering would be trivial to guarantee. But pg_cron cannot call Python, so + the flip would have to be reimplemented in SQL — a second copy of the rule, + which is the drift this repository has already paid for once. + +Failure in either direction is safe: if the job does not run, embargoed records +stay embargoed. + +## What is enforced, and where + +Migration `b4c5d6e7f8a9` adds the embargo clause to the four public relations +that read the observation chain: + +- `ogc_water_well_summary` +- `ogc_water_elevation_wells` +- `ogc_depth_to_water_trend_wells` +- `ogc_latest_depth_to_water_wells` + +The clause is `release_status IS DISTINCT FROM 'embargoed'`, applied at each +level of the chain — observation, sample, field activity, field event — so an +embargoed reading cannot be reached through a public parent. `IS DISTINCT +FROM` rather than `<>` because `release_status` is nullable and `NULL <> +'embargoed'` is NULL, which would drop the row. + +`ogc_well_water_column` and the Group A thing views need no change: they +already filter their observations on `release_status = 'public'`, which +excludes `embargoed` for free. Whole-thing embargoes need no change anywhere, +for the same reason. + +The `ogc_internal_*` mount is deliberately untouched. It serves Bureau staff +and has never filtered on release level; seeing embargoed data before it is +published is what it is for. + +### Why the clause is not `= 'public'` + +Matching the other relations would be tidier, and is deliberately not done. It +would also drop every observation sitting at `draft`, `provisional`, or NULL — +a release-policy change with its own row counts to check, not an embargo. As +written, the four relations returned byte-identical results the day the +migration landed, because nothing was embargoed yet. + +Migration `w1x2y3z4a5b6` is the record of what the tidier version costs when +the row states are not what you assumed: three NGWMN exports emptied outright, +3005/3005 rows on one of them. If someone later decides the public relations +should require `= 'public'` all the way down, that is its own change, with row +counts taken on a prod clone before and after. + +## Chemistry cannot be embargoed per record + +`ogc_major_chemistry_results`, `ogc_minor_chemistry_wells`, +`ogc_avg_tds_wells` and `ogc_latest_tds_wells` do not read `observation` and +`sample`. They read the legacy `NMA_Chemistry_SampleInfo`, +`NMA_MajorChemistry` and `NMA_MinorTraceChemistry` mirror tables, which are +plain `Base` models carrying no release columns at all — no `release_status`, +no `data_maturity`, no `release_at`. Their only gate is the joined thing. + +So chemistry can be embargoed **per well**, by embargoing the thing, and not +per result. Since chemistry is the usual reason anyone wants an embargo, this +is the gap to close first. Three ways, none cheap: + +1. **Release columns on the `NMA_*` mirrors.** They are deprecated and frozen + (`transfers/README.md`), and repopulated by a deprecated driver that would + have to learn to preserve a governance column it knows nothing about. Key + anything added there on `nma_global_id`, which is stable and UNIQUE — never + the autoincrement `id`. +2. **An embargo side-table** keyed by `nma_global_id`, joined into the four + views. Leaves the frozen tables alone, at the cost of governance living in + two places, which is what ADR5 exists to stop. +3. **Wait for chemistry to move onto the Ocotillo `observation` model.** Then + it joins the chain above and costs nothing. Not currently scheduled. + +## Files + +| file | what it holds | +| --- | --- | +| `domain/release.py` | the rules, over plain values, no database | +| `services/release_schedule.py` | loads due rows, flips them, writes the audit | +| `cli/cli.py` (`release-embargoed`) | the command | +| `db/base.py` (`ReleaseMixin`) | the `release_at` column | +| `alembic/versions/a3b4c5d6e7f8_*` | the column, on 35 tables and 7 version tables | +| `alembic/versions/b4c5d6e7f8a9_*` | the embargo clause in the four public views | +| `tests/test_ogc_embargo.py` | the claim, checked against the relations themselves | diff --git a/domain/access.py b/domain/access.py new file mode 100644 index 000000000..9bdd3bde9 --- /dev/null +++ b/domain/access.py @@ -0,0 +1,283 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Access-control rules from ADR5, as plain functions over plain values. + +Two questions, one grammar: + +* May this principal exercise this capability within this scope? + (``permission_grant`` -- institutional trust, decided per person.) +* Did the owner agree to publish this data type of this well to this + destination? (``publication_consent`` -- landowner consent, decided per well.) + +Two record types with separate governance, evaluated the same way. The rules +live here so they can be exercised without a database; ``services/visibility.py`` +loads the rows and asks. + +Three invariants this module exists to keep: + +* **Default deny.** No grant means no. An empty sequence answers False, and + every helper returns False rather than raising when it cannot say yes. +* **No wildcards.** A grant names its data type. There is no term meaning + "all", so a data type added next year is never covered by an existing row. +* **Expiry is checked at use.** Nothing sweeps expired rows; every check + compares against the date it is asked about. +""" + +from dataclasses import dataclass, field +from datetime import date, datetime + +# Scope types. A grant reaches everything below its scope, so `global` covers +# every thing, `group` covers the things in one group (a project or network), +# and `thing` covers exactly one. +SCOPE_GLOBAL = "global" +SCOPE_GROUP = "group" +SCOPE_THING = "thing" +SCOPE_TYPES = frozenset({SCOPE_GLOBAL, SCOPE_GROUP, SCOPE_THING}) + +# Capabilities. +CAPABILITY_READ = "read" +CAPABILITY_ENTER = "enter" +CAPABILITY_CORRECT = "correct" +CAPABILITY_ADMINISTER = "administer" +CAPABILITIES = frozenset( + {CAPABILITY_READ, CAPABILITY_ENTER, CAPABILITY_CORRECT, CAPABILITY_ADMINISTER} +) + +# Principal types. A destination is not here: publishing to one is recorded as +# consent, not as a grant, which is the two-table half of ADR5. +PRINCIPAL_USER = "user" +PRINCIPAL_ROLE = "role" +# Spelled as the lexicon spells it. The lexicon is the source of truth for +# every controlled term, and a constant that disagreed with it made this +# principal type unwritable: the route validated "api key" against "api_key" +# and rejected every API-key grant with a 422. tests/test_domain_access.py +# pins the two together so it cannot drift again. +PRINCIPAL_API_KEY = "api key" +PRINCIPAL_TYPES = frozenset({PRINCIPAL_USER, PRINCIPAL_ROLE, PRINCIPAL_API_KEY}) + + +class AccessRuleError(ValueError): + """Base for rule violations. A ValueError, per ADR4.""" + + +class UnknownScopeType(AccessRuleError): + pass + + +class UnknownCapability(AccessRuleError): + pass + + +class UnknownPrincipalType(AccessRuleError): + pass + + +class MissingDataType(AccessRuleError): + pass + + +class ScopeIdMismatch(AccessRuleError): + pass + + +class BackwardsDateRange(AccessRuleError): + pass + + +@dataclass(frozen=True) +class Grant: + """One row of ``permission_grant``, as plain values.""" + + principal_type: str + principal_id: str + capability: str + scope_type: str + scope_id: int | None + data_type: str + starts_at: date + ends_at: date | None = None + revoked_at: datetime | None = None + + +@dataclass(frozen=True) +class Consent: + """One row of ``publication_consent``, as plain values.""" + + thing_id: int + destination_id: int + data_type: str + starts_at: date + ends_at: date | None = None + revoked_at: datetime | None = None + + +@dataclass(frozen=True) +class AccessRequest: + """What a caller is asking for. + + ``principals`` is every identity the caller presents at once -- their user + subject and each role they hold -- because a grant may name any of them. + ``group_ids`` are the groups the target thing belongs to, which is what + makes a project-scoped grant reach it. + """ + + capability: str + data_type: str + principals: tuple[tuple[str, str], ...] = () + thing_id: int | None = None + group_ids: tuple[int, ...] = field(default_factory=tuple) + + +def validate_grant( + principal_type: str, + capability: str, + scope_type: str, + scope_id: int | None, + data_type: str | None, + starts_at: date, + ends_at: date | None, +) -> None: + """Reject a grant that could not be evaluated honestly. + + Raised before a row is written, so the invariants hold in the table rather + than in the reader. + """ + if principal_type not in PRINCIPAL_TYPES: + raise UnknownPrincipalType( + f"'{principal_type}' is not a principal type " + f"({', '.join(sorted(PRINCIPAL_TYPES))})." + ) + if capability not in CAPABILITIES: + raise UnknownCapability( + f"'{capability}' is not a capability ({', '.join(sorted(CAPABILITIES))})." + ) + if scope_type not in SCOPE_TYPES: + raise UnknownScopeType( + f"'{scope_type}' is not a scope type ({', '.join(sorted(SCOPE_TYPES))})." + ) + if scope_type == SCOPE_GLOBAL and scope_id is not None: + raise ScopeIdMismatch("A global grant names no scope_id.") + if scope_type != SCOPE_GLOBAL and scope_id is None: + raise ScopeIdMismatch(f"A {scope_type}-scoped grant needs a scope_id.") + if not data_type: + # The no-wildcard rule. There is deliberately no term meaning "all": + # a blanket grant is what published data nobody had agreed to publish. + raise MissingDataType( + "A grant names its data type. There is no wildcard, so a new data " + "type is never covered by an existing grant." + ) + require_forward_range(starts_at, ends_at) + + +def require_forward_range(starts_at: date, ends_at: date | None) -> None: + if ends_at is not None and starts_at is not None and ends_at < starts_at: + raise BackwardsDateRange( + f"end date {ends_at.isoformat()} precedes start date " + f"{starts_at.isoformat()}." + ) + + +def is_active( + starts_at: date | None, + ends_at: date | None, + revoked_at: datetime | None, + on_date: date, +) -> bool: + """Whether a row is in force on ``on_date``. + + Revocation wins immediately and is not backdated: a revoked row is dead + from the moment it is revoked, which is the promise made to a landowner who + calls to change their mind. + """ + if revoked_at is not None: + return False + if starts_at is not None and on_date < starts_at: + return False + if ends_at is not None and on_date > ends_at: + return False + return True + + +def scope_covers( + scope_type: str, + scope_id: int | None, + thing_id: int | None, + group_ids: tuple[int, ...] = (), +) -> bool: + """Whether a grant's scope reaches the thing being asked about.""" + if scope_type == SCOPE_GLOBAL: + return True + if scope_type == SCOPE_THING: + return thing_id is not None and scope_id == thing_id + if scope_type == SCOPE_GROUP: + return scope_id in set(group_ids) + # An unrecognized scope type denies rather than raises: a row written by a + # newer version of the code must not read as permission to an older one. + return False + + +def grant_covers(grant: Grant, request: AccessRequest, on_date: date) -> bool: + """Whether one grant answers one request. Every axis must match.""" + if (grant.principal_type, grant.principal_id) not in request.principals: + return False + if grant.capability != request.capability: + return False + if grant.data_type != request.data_type: + return False + if not is_active(grant.starts_at, grant.ends_at, grant.revoked_at, on_date): + return False + return scope_covers( + grant.scope_type, grant.scope_id, request.thing_id, request.group_ids + ) + + +def any_grant_allows(grants, request: AccessRequest, on_date: date) -> bool: + """Default deny: an empty sequence is a no.""" + return any(grant_covers(grant, request, on_date) for grant in grants) + + +def consent_covers( + consent: Consent, + thing_id: int, + destination_id: int, + data_type: str, + on_date: date, +) -> bool: + """Whether one consent row publishes this data type of this well here.""" + return ( + consent.thing_id == thing_id + and consent.destination_id == destination_id + and consent.data_type == data_type + and is_active(consent.starts_at, consent.ends_at, consent.revoked_at, on_date) + ) + + +def any_consent_publishes( + consents, + thing_id: int, + destination_id: int, + data_type: str, + on_date: date, +) -> bool: + """Default deny, for the publication half.""" + return any( + consent_covers(consent, thing_id, destination_id, data_type, on_date) + for consent in consents + ) + + +# ============= EOF ============================================= diff --git a/domain/field_projection.py b/domain/field_projection.py new file mode 100644 index 000000000..8ddb09f2e --- /dev/null +++ b/domain/field_projection.py @@ -0,0 +1,144 @@ +# =============================================================================== +# Copyright 2025 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Field projection: what an audience actually receives, field by field (ADR5, 3.5). + +Record-level grants are not enough, because the Bureau's promises are +field-shaped. The owner's name and phone number sit on the same well record as +the water levels the owner agreed to share. + +The rule is an allowlist, per audience: + +* **Omission produces silence, not leakage.** Only fields explicitly approved + for an audience appear in that audience's payload, so a column added next + year is invisible until someone decides otherwise. +* **Protection includes transformation.** A public record can carry a + coordinate rounded to protect the landowner while the precise value stays + internal. Dropping the well from the map entirely is not the only option. +* **The never-public list wins over any allowlist.** Engineering guarantees a + listed field is enforced everywhere; a named data owner decides what is on + the list. A configuration that names one is a mistake, and it is raised at + load rather than honored at request time. + +Plain values only: dicts in, dicts out. ``services/field_projection.py`` reads +the configuration and supplies them. +""" + +from dataclasses import dataclass, field as dataclass_field + + +class FieldProjectionError(ValueError): + """Base for projection configuration problems.""" + + +class NeverPublicFieldAllowed(FieldProjectionError): + """An allowlist named a field no configuration may expose.""" + + +class UnknownField(FieldProjectionError): + """An allowlist named a field the entity does not have.""" + + +class UnknownTransform(FieldProjectionError): + """A transform nobody implements.""" + + +@dataclass(frozen=True) +class EntityProjection: + """The rule for one entity, for one audience.""" + + fields: frozenset + # field name -> (transform name, argument), e.g. "latitude" -> ("round", 2) + transforms: dict = dataclass_field(default_factory=dict) + + +def round_to(value, places: int): + """Reduce coordinate precision. Two decimal places is roughly a kilometre.""" + if value is None: + return None + return round(float(value), places) + + +# Transformations a configuration may ask for. Anything else raises at load, +# so a typo cannot silently degrade to "publish the value untouched". +TRANSFORMS = {"round": round_to} + + +def validate_projection( + entity: str, + fields, + transforms: dict, + known_fields, + never_public, +) -> None: + """Reject a projection that could not be honored, before it is used.""" + unknown = sorted(set(fields) - set(known_fields)) + if unknown: + raise UnknownField( + f"{entity} has no field(s) {', '.join(unknown)}. " + "An allowlist naming a field that does not exist is a typo, and a " + "typo in an allowlist silently withholds data." + ) + + forbidden = sorted(set(fields) & set(never_public)) + if forbidden: + raise NeverPublicFieldAllowed( + f"{entity}.{forbidden[0]} is on the never-public list and cannot " + "be added to an audience. Removing it from that list is a policy " + "decision with a named owner, not a configuration change." + ) + + for field_name, (transform_name, _) in transforms.items(): + if field_name not in fields: + raise UnknownField( + f"{entity}.{field_name} has a transform but is not in the " + "allowlist, so the transform would never run." + ) + if transform_name not in TRANSFORMS: + raise UnknownTransform( + f"'{transform_name}' is not a transform " + f"({', '.join(sorted(TRANSFORMS))})." + ) + + +def project(record: dict, projection: EntityProjection, never_public=frozenset()): + """One record, as this audience receives it. + + Default deny: an audience with no rule gets an empty dict, not the record. + The never-public list is applied here as well as at load, so a field on it + stays out even if a projection was built without validation. + """ + if projection is None: + return {} + + projected = {} + for field_name, value in record.items(): + if field_name not in projection.fields: + continue + if field_name in never_public: + continue + + transform = projection.transforms.get(field_name) + if transform is not None: + transform_name, argument = transform + value = TRANSFORMS[transform_name](value, argument) + + projected[field_name] = value + + return projected + + +# ============= EOF ============================================= diff --git a/domain/release.py b/domain/release.py new file mode 100644 index 000000000..ab4e60eff --- /dev/null +++ b/domain/release.py @@ -0,0 +1,109 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +Embargo rules, as plain functions over plain values. + +An embargo is a record held back from public release until a date that was +decided in advance: `release_status = 'embargoed'` plus a `release_at`. When +the date arrives, `services/release_schedule.py` flips the level to `public` +and the record appears in the OGC views on their next refresh. + +Three invariants this module exists to keep: + +* **Intent is not enforcement.** `release_status` is the only thing the views + filter on. `release_at` says when that level is due to change and is read by + the release job alone. This module therefore publishes no "is this visible" + predicate: one that compared dates would disagree with the views for up to a + day and would eventually be used as a read-path filter, which is the + distributed filtering ADR5 exists to prevent (see migration `baba91fe5e83`). + +* **An embargo only ever widens visibility.** The scheduled path turns + `embargoed` into `public` and does nothing else. Making a record *less* + visible is an immediate change to `release_status`, never a scheduled one, + for the same reason a revocation in `domain/access.py` is never backdated: + a promise to hide something later is not a promise anybody should rely on. + +* **An embargo names its date.** `embargoed` without a `release_at` would be + a hold nothing ever lifts, and a `release_at` on any other level would be a + date nothing ever reads. Both are rejected before the row is written. An + indefinite hold is `private`, which is honest about never lifting itself. +""" + +from datetime import date + +# The level a record sits at while it is being withheld, and the one the +# scheduled release moves it to. Spelled as the lexicon spells them; +# tests/test_domain_release.py pins these against core/lexicon.json so a +# rename there cannot silently strand the release job. +STATUS_EMBARGOED = "embargoed" +STATUS_PUBLIC = "public" +# The level for a hold that nothing lifts. Named here only so the error +# message can point at it. +STATUS_PRIVATE = "private" + + +class ReleaseRuleError(ValueError): + """Base for embargo rule violations. A ValueError, per ADR4.""" + + +class MissingReleaseDate(ReleaseRuleError): + pass + + +class UnscheduledReleaseDate(ReleaseRuleError): + pass + + +def validate_release(release_status: str | None, release_at: date | None) -> None: + """Reject an embargo that could not be lifted honestly. + + Raised before a row is written, so the invariant holds in the table + rather than in the reader. + """ + if release_status == STATUS_EMBARGOED and release_at is None: + raise MissingReleaseDate( + "An embargoed record names the date it is released. For a hold " + f"with no end, use '{STATUS_PRIVATE}' instead." + ) + if release_at is not None and release_status != STATUS_EMBARGOED: + raise UnscheduledReleaseDate( + f"release_at is only read for '{STATUS_EMBARGOED}' records, so " + f"setting it on a '{release_status}' record schedules nothing." + ) + + +def due_for_release( + release_status: str | None, + release_at: date | None, + on_date: date, +) -> bool: + """Whether this record's embargo has run out by ``on_date``. + + The only date comparison in the embargo feature, and it belongs to the + release job. A record is due on its ``release_at`` itself, not the day + after: the date is when the record becomes public, not the last day it is + held. + """ + if release_status != STATUS_EMBARGOED: + return False + if release_at is None: + # Unreachable through validate_release, but a row written before this + # module existed, or by hand, must not crash the nightly job. + return False + return on_date >= release_at + + +# ============= EOF ============================================= diff --git a/entrypoint.sh b/entrypoint.sh index c89c621c1..8eb845d3d 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -19,6 +19,13 @@ echo "PostgreSQL is ready!" if [ "$RUN_MIGRATIONS" = "true" ]; then echo "Applying migrations..." alembic upgrade head + + # Lexicon terms (principal_type, capability, ...) are seeded from + # core/lexicon.json separately from the migration that adds the tables + # referencing them -- see alembic/versions/79a3ab24627e_add_access_control_tables.py. + # Idempotent (on_conflict_do_nothing), safe to run on every start. + echo "Seeding lexicon..." + oco initialize-lexicon fi echo "Starting the application..." diff --git a/schemas/__init__.py b/schemas/__init__.py index 5a1d85afb..a556705d1 100644 --- a/schemas/__init__.py +++ b/schemas/__init__.py @@ -16,12 +16,14 @@ from datetime import datetime, timezone, date from typing import Annotated -from core.enums import ReleaseStatus +from core.enums import DataMaturity, ReleaseStatus +from domain.release import validate_release from pydantic import ( BaseModel, ConfigDict, AwareDatetime, field_validator, + model_validator, ) from pydantic.functional_validators import AfterValidator from pydantic.json_schema import JsonSchemaValue @@ -36,6 +38,12 @@ class ResourceNotFoundResponse(BaseModel): class BaseCreateModel(BaseModel): release_status: ReleaseStatus = "draft" + # Orthogonal to release_status: data can be published and provisional at + # the same time (ADR5). NULL means not stated. + data_maturity: DataMaturity | None = None + # The date an embargoed record becomes public. NULL for everything that is + # not embargoed. See docs/data-embargo.md. + release_at: date | None = None @field_validator("release_status", mode="before") @classmethod @@ -47,10 +55,37 @@ def coerce_release_status(cls, v): raise ValueError(f"Invalid release_status: {v}") return v + @model_validator(mode="after") + def check_release_schedule(self): + """An embargo names its date, and a date means an embargo. + + Sound here because a create payload is the whole row. The same pair is + enforced by a CHECK constraint on every ReleaseMixin table, which is + what covers the paths this validator cannot see -- a PATCH that sets + one half, the CLI, a transfer, raw SQL. + """ + # release_status arrives as the lexicon-backed enum or as the plain + # string, depending on whether the subclass narrowed the annotation. + status = self.release_status + validate_release(getattr(status, "value", status), self.release_at) + return self + class BaseUpdateModel(BaseCreateModel): release_status: ReleaseStatus | None = None + @model_validator(mode="after") + def check_release_schedule(self): + """Deliberately not checked here. + + An update payload is a fragment: `{"release_status": "embargoed"}` on a + row that already carries a release_at is legitimate, and so is clearing + an embargo by setting the level back. The invariant is about the + resulting row, which this schema cannot see, so it is left to the CHECK + constraint that guards the table itself. + """ + return self + def past_or_today_validator( value: date | datetime | None, @@ -113,6 +148,8 @@ class BaseResponseModel(BaseModel): id: int # every ORM model should have an id field created_at: UTCAwareDatetime release_status: ReleaseStatus + data_maturity: DataMaturity | None = None + release_at: date | None = None model_config = ConfigDict( from_attributes=True, diff --git a/schemas/access.py b/schemas/access.py new file mode 100644 index 000000000..082a0e811 --- /dev/null +++ b/schemas/access.py @@ -0,0 +1,153 @@ +# =============================================================================== +# Copyright 2025 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +schemas/access.py + +Request and response shapes for the ADR5 access-control routes: destinations, +permission grants, publication consent. + +These do not inherit ``BaseCreateModel`` / ``BaseResponseModel``. Those carry +``release_status`` and ``data_maturity``, which describe released *data*. A +grant is not data anybody releases -- it is the rule about who sees it. +""" + +from datetime import date + +from pydantic import BaseModel, ConfigDict, Field + +from core.enums import ( + AccessDataType, + Capability, + DestinationKind, + GrantScopeType, + PrincipalType, +) +from schemas import UTCAwareDatetime + + +# ------ DESTINATION ---------- +class CreateDestination(BaseModel): + slug: str = Field(max_length=50, examples=["ngwmn"]) + name: str = Field( + max_length=255, examples=["National Ground-Water Monitoring Network"] + ) + destination_kind: DestinationKind + description: str | None = None + + +class DestinationResponse(BaseModel): + model_config = ConfigDict(from_attributes=True) + + id: int + slug: str + name: str + destination_kind: DestinationKind + description: str | None + active: bool + + +# ------ PERMISSION GRANT ---------- +class CreatePermissionGrant(BaseModel): + """One grant. Every axis is named; there is no wildcard data type.""" + + principal_type: PrincipalType + principal_id: str = Field(max_length=255) + capability: Capability + scope_type: GrantScopeType + # Null for a global grant, required for a group- or thing-scoped one. The + # rule is enforced in domain/access.py rather than here, so it holds for + # every writer, not only this route. + scope_id: int | None = None + data_type: AccessDataType + starts_at: date + ends_at: date | None = None + reason: str | None = None + + +class PermissionGrantResponse(BaseModel): + model_config = ConfigDict(from_attributes=True) + + id: int + principal_type: PrincipalType + principal_id: str + capability: Capability + scope_type: GrantScopeType + scope_id: int | None + data_type: AccessDataType + starts_at: date + ends_at: date | None + granted_by: str + reason: str | None + revoked_at: UTCAwareDatetime | None + revoked_by: str | None + + +class AccessDecision(BaseModel): + """The visibility layer's answer, and what was asked.""" + + allowed: bool + capability: Capability + data_type: AccessDataType + thing_id: int | None + principals: list[str] + + +# ------ PUBLICATION CONSENT ---------- +class CreatePublicationConsent(BaseModel): + thing_id: int + destination_slug: str = Field(max_length=50) + data_type: AccessDataType + starts_at: date + ends_at: date | None = None + # Null when the Bureau owns the well: the decision was institutional, and + # inventing a consenting contact would be a lie. + contact_id: int | None = None + notes: str | None = None + + +class PublicationConsentResponse(BaseModel): + model_config = ConfigDict(from_attributes=True) + + id: int + thing_id: int + destination_id: int + data_type: AccessDataType + contact_id: int | None + recorded_by: str + notes: str | None + starts_at: date + ends_at: date | None + revoked_at: UTCAwareDatetime | None + revoked_by: str | None + + +class PublishedThing(BaseModel): + """One thing as a destination sees it. + + ``thing_id`` and ``data_types`` are the envelope for staff reading this + route; ``properties`` and ``location`` are what the destination itself + receives, already projected through the per-audience allowlist. A field + nobody approved for this audience is absent rather than null, and a + coordinate may arrive rounded. + """ + + thing_id: int + data_types: list[AccessDataType] + properties: dict + location: dict + + +# ============= EOF ============================================= diff --git a/schemas/permission_history.py b/schemas/field_access_consent.py similarity index 77% rename from schemas/permission_history.py rename to schemas/field_access_consent.py index d8f1f3ef2..45d83d750 100644 --- a/schemas/permission_history.py +++ b/schemas/field_access_consent.py @@ -5,8 +5,11 @@ # ------ RESPONSE ---------- -class PermissionHistoryResponse(BaseModel): +class FieldAccessConsentResponse(BaseModel): """ + Landowner field-access consent, as published on a Thing. Not an + access-control grant (ADR5). + Even though permission_allowed and start_date are not-nullable in the database, they are nullable here to accommodate cases where no permission record exists for a given permission type. diff --git a/schemas/thing.py b/schemas/thing.py index c2798b5fd..86942e377 100644 --- a/schemas/thing.py +++ b/schemas/thing.py @@ -35,7 +35,7 @@ from schemas.group import GroupResponse from schemas.location import LocationGeoJSONResponse from schemas.notes import NoteResponse, CreateNote -from schemas.permission_history import PermissionHistoryResponse +from schemas.field_access_consent import FieldAccessConsentResponse # -------- VALIDATE ---------- @@ -274,7 +274,7 @@ class WellResponse(BaseThingResponse): water_notes: list[NoteResponse] = [] construction_notes: list[NoteResponse] = [] contacts: list[WellContactSummaryResponse] = [] - permissions: list[PermissionHistoryResponse] + permissions: list[FieldAccessConsentResponse] formation_completion_code: FormationCode | None nma_formation_zone: str | None well_location_note: list[str] = [] @@ -299,7 +299,7 @@ def populate_well_casing_materials_with_strings(cls, well_casing_materials): return materials @field_validator("permissions", mode="before") - def populate_permission_history_with_latest_records(cls, permissions): + def populate_field_access_consent_with_latest_records(cls, permissions): """ Populate the permission history with the latest records for each type of permission. If multiple records exist for the same permission type @@ -326,7 +326,7 @@ def populate_permission_history_with_latest_records(cls, permissions): permissions_to_return.append(latest_record) else: permissions_to_return.append( - PermissionHistoryResponse( + FieldAccessConsentResponse( permission_type=permission_type, permission_allowed=None, start_date=None, diff --git a/services/access_admin.py b/services/access_admin.py new file mode 100644 index 000000000..2d94946ba --- /dev/null +++ b/services/access_admin.py @@ -0,0 +1,268 @@ +# =============================================================================== +# Copyright 2025 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +services/access_admin.py + +Writes to the access-control tables: register a destination, grant, revoke, +record a landowner's consent, withdraw it. + +Every write lands an ``authorization_audit`` row in the same transaction. That +is the point of the module: there is no path that changes who may see what and +leaves no trace, because the question after an incident is "who granted that, +and when" (ADR5, 4.3). + +Rules live in ``domain/access.py`` and are checked before the row is written, +so an unevaluable grant -- no data type, a global grant with a scope id -- +cannot reach the table. Domain errors subclass ValueError; the routes turn +them into 422s. +""" + +from datetime import date, datetime, timezone + +from db.authorization_audit import ( + AuthorizationAudit, + CONSENT_RECORDED, + CONSENT_REVOKED, + DESTINATION_REGISTERED, + GRANT_CREATED, + GRANT_REVOKED, +) +from db.destination import Destination +from db.permission_grant import PermissionGrant +from db.publication_consent import PublicationConsent +from domain.access import require_forward_range, validate_grant + +# Used when the caller's token carries no subject -- the development bypass, +# or a test override. Recorded rather than left null: "we do not know who" +# is itself worth knowing when reading the log back. +UNKNOWN_ACTOR = "unknown" + + +class AlreadyRevoked(ValueError): + """Raised when revoking something that is already revoked.""" + + +def actor_from_payload(payload) -> str: + """The identifier to record as having done this.""" + if not isinstance(payload, dict): + return UNKNOWN_ACTOR + return str(payload.get("sub") or payload.get("preferred_username") or UNKNOWN_ACTOR) + + +def _audit(session, actor, event_type, subject_table, subject_id, detail): + session.add( + AuthorizationAudit( + event_type=event_type, + actor=actor, + subject_table=subject_table, + subject_id=subject_id, + detail=detail, + ) + ) + + +def register_destination( + session, + actor: str, + slug: str, + name: str, + destination_kind: str, + description: str = None, +) -> Destination: + destination = Destination( + slug=slug, + name=name, + destination_kind=destination_kind, + description=description, + active=True, + ) + session.add(destination) + session.flush() + _audit( + session, + actor, + DESTINATION_REGISTERED, + Destination.__tablename__, + destination.id, + {"slug": slug, "name": name, "destination_kind": destination_kind}, + ) + session.commit() + session.refresh(destination) + return destination + + +def create_grant( + session, + actor: str, + principal_type: str, + principal_id: str, + capability: str, + scope_type: str, + scope_id: int, + data_type: str, + starts_at: date, + ends_at: date = None, + reason: str = None, +) -> PermissionGrant: + validate_grant( + principal_type=principal_type, + capability=capability, + scope_type=scope_type, + scope_id=scope_id, + data_type=data_type, + starts_at=starts_at, + ends_at=ends_at, + ) + + grant = PermissionGrant( + principal_type=principal_type, + principal_id=principal_id, + capability=capability, + scope_type=scope_type, + scope_id=scope_id, + data_type=data_type, + starts_at=starts_at, + ends_at=ends_at, + granted_by=actor, + reason=reason, + ) + session.add(grant) + session.flush() + _audit( + session, + actor, + GRANT_CREATED, + PermissionGrant.__tablename__, + grant.id, + { + "principal_type": principal_type, + "principal_id": principal_id, + "capability": capability, + "scope_type": scope_type, + "scope_id": scope_id, + "data_type": data_type, + "starts_at": starts_at.isoformat(), + "ends_at": ends_at.isoformat() if ends_at else None, + "reason": reason, + }, + ) + session.commit() + session.refresh(grant) + return grant + + +def revoke_grant(session, actor: str, grant: PermissionGrant) -> PermissionGrant: + """Revoke now. Effective at the next read, never backdated.""" + if grant.revoked_at is not None: + raise AlreadyRevoked(f"grant {grant.id} was already revoked.") + + grant.revoked_at = datetime.now(timezone.utc) + grant.revoked_by = actor + _audit( + session, + actor, + GRANT_REVOKED, + PermissionGrant.__tablename__, + grant.id, + { + "principal_type": grant.principal_type, + "principal_id": grant.principal_id, + "capability": grant.capability, + "data_type": grant.data_type, + }, + ) + session.commit() + session.refresh(grant) + return grant + + +def record_consent( + session, + actor: str, + thing_id: int, + destination_id: int, + data_type: str, + starts_at: date, + ends_at: date = None, + contact_id: int = None, + notes: str = None, +) -> PublicationConsent: + """Record that an owner agreed to publish this data type here.""" + require_forward_range(starts_at, ends_at) + + consent = PublicationConsent( + thing_id=thing_id, + destination_id=destination_id, + data_type=data_type, + contact_id=contact_id, + recorded_by=actor, + notes=notes, + starts_at=starts_at, + ends_at=ends_at, + ) + session.add(consent) + session.flush() + _audit( + session, + actor, + CONSENT_RECORDED, + PublicationConsent.__tablename__, + consent.id, + { + "thing_id": thing_id, + "destination_id": destination_id, + "data_type": data_type, + "contact_id": contact_id, + "starts_at": starts_at.isoformat(), + "ends_at": ends_at.isoformat() if ends_at else None, + }, + ) + session.commit() + session.refresh(consent) + return consent + + +def revoke_consent( + session, actor: str, consent: PublicationConsent +) -> PublicationConsent: + """Stop offering this. + + Not a recall: copies a harvester already took live in someone else's + system, and the owner should be told that rather than promised otherwise. + """ + if consent.revoked_at is not None: + raise AlreadyRevoked(f"consent {consent.id} was already revoked.") + + consent.revoked_at = datetime.now(timezone.utc) + consent.revoked_by = actor + _audit( + session, + actor, + CONSENT_REVOKED, + PublicationConsent.__tablename__, + consent.id, + { + "thing_id": consent.thing_id, + "destination_id": consent.destination_id, + "data_type": consent.data_type, + }, + ) + session.commit() + session.refresh(consent) + return consent + + +# ============= EOF ============================================= diff --git a/services/access_seed.py b/services/access_seed.py new file mode 100644 index 000000000..fdf4e3b09 --- /dev/null +++ b/services/access_seed.py @@ -0,0 +1,178 @@ +# =============================================================================== +# Copyright 2025 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +services/access_seed.py + +The day-one baseline from ADR5, 5.2: existing Authentik roles become role +principals holding broad grants, so nobody's access changes on the day the +grant table starts being consulted. + +Without this the grant table is empty, and an empty grant table means default +deny -- correct, and useless: ``/access/decision`` says no to everyone, +including the people who administer the system. + +Three properties this seeder has to have: + +* **Idempotent.** Running it twice creates nothing the second time. +* **Not a resurrection.** A seeded grant somebody revoked stays revoked. The + skip check looks at every seeded row, not only the live ones, so re-running + after a deliberate revocation does not quietly undo it. +* **Marked.** Every row it writes carries ``granted_by`` set to the seed actor + and a reason naming the ADR, so a grant that exists because of institutional + history is distinguishable from one somebody decided on. + +The mapping mirrors the role families in ``core/dependencies.py``. It is a +starting point, not a statement about what each role should have: narrowing it +is the point of the whole exercise, and every narrowing is a revocation +somebody makes deliberately. +""" + +from dataclasses import dataclass, field +from datetime import date + +from sqlalchemy import select + +from core.enums import AccessDataType +from db.permission_grant import PermissionGrant +from domain.access import ( + CAPABILITY_ADMINISTER, + CAPABILITY_CORRECT, + CAPABILITY_ENTER, + CAPABILITY_READ, + PRINCIPAL_ROLE, + SCOPE_GLOBAL, +) +from services.access_admin import create_grant + +# Recorded as the granting actor. Not a person, and deliberately obvious in +# the audit log: these grants exist because of what access already was, not +# because someone weighed them. +SEED_ACTOR = "system:day-one-baseline" +SEED_REASON = ( + "Day-one baseline (ADR5, 5.2): preserves the access this Authentik role " + "already had before grants were consulted. Narrow it deliberately." +) + +READ_ONLY = (CAPABILITY_READ,) +EDIT = (CAPABILITY_READ, CAPABILITY_ENTER, CAPABILITY_CORRECT) +FULL = EDIT + (CAPABILITY_ADMINISTER,) + +# Authentik group -> capabilities, mirroring core/dependencies.py. The tiers +# nest within a family, so an Admin's row set is a superset of an Editor's. +# +# Lexicon* is absent: it gates vocabulary, not data. AMP.Staging is absent +# because it gates a workbench that ships dark, and seeding it would be the +# one thing nobody intended -- granting access to something still being +# validated. +ROLE_BASELINE = { + "Viewer": READ_ONLY, + "Editor": EDIT, + "Admin": FULL, + "AMPViewer": READ_ONLY, + "AMPEditor": EDIT, + "AMPAdmin": FULL, + # The desktop-GIS mount reads; it has never written. + "OGCInternal": READ_ONLY, +} + + +@dataclass +class SeedPlan: + """What seeding would do, or did.""" + + created: list = field(default_factory=list) + skipped: list = field(default_factory=list) + + def describe(self, entry) -> str: + role, capability, data_type = entry + return f"role:{role} may {capability} {data_type} (global)" + + +def data_types() -> tuple: + """Every access data type there is, named one by one. + + No wildcard: this is a list of rows, so a data type added later is not + covered until somebody seeds or grants it. + """ + return tuple(member.value for member in AccessDataType) + + +def planned_entries() -> list: + return [ + (role, capability, data_type) + for role, capabilities in ROLE_BASELINE.items() + for capability in capabilities + for data_type in data_types() + ] + + +def _already_seeded(session) -> set: + """Every (role, capability, data type) this seeder has ever written. + + Revoked rows count. A grant somebody took away is not re-created by + running the seeder again. + """ + rows = session.execute( + select( + PermissionGrant.principal_id, + PermissionGrant.capability, + PermissionGrant.data_type, + ).where( + PermissionGrant.principal_type == PRINCIPAL_ROLE, + PermissionGrant.granted_by == SEED_ACTOR, + ) + ).all() + return {(role, capability, data_type) for role, capability, data_type in rows} + + +def seed_role_grants(session, starts_at: date = None, apply: bool = True) -> SeedPlan: + """Create the missing baseline grants. Safe to run repeatedly. + + With ``apply=False`` nothing is written and the plan describes what would + be. Grants are security state, so the CLI previews by default. + """ + starts_at = starts_at or date.today() + seeded = _already_seeded(session) + + plan = SeedPlan() + for entry in planned_entries(): + if entry in seeded: + plan.skipped.append(entry) + continue + + plan.created.append(entry) + if not apply: + continue + + role, capability, data_type = entry + create_grant( + session, + SEED_ACTOR, + principal_type=PRINCIPAL_ROLE, + principal_id=role, + capability=capability, + scope_type=SCOPE_GLOBAL, + scope_id=None, + data_type=data_type, + starts_at=starts_at, + ends_at=None, + reason=SEED_REASON, + ) + + return plan + + +# ============= EOF ============================================= diff --git a/services/field_projection.py b/services/field_projection.py new file mode 100644 index 000000000..358ee36cd --- /dev/null +++ b/services/field_projection.py @@ -0,0 +1,227 @@ +# =============================================================================== +# Copyright 2025 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +services/field_projection.py + +Loads ``core/field-allowlists.yml`` and turns a database row into the record an +audience actually receives. + +This is the chokepoint from ADR5 A.2. Every published payload is built here, so +a new route or a new output format cannot skip the rule by omission -- it sits +below them. The rules themselves are in ``domain/field_projection.py``. + +Two consumers, because there are two publication paths: + +* ``services/visibility.py`` builds destination payloads from ORM rows and + calls :func:`project_entity`. +* ``core/feature_provider.py`` serves the public OGC collections and calls + :func:`ogc_allowlist` at provider construction, so the columns nobody + approved are never selected from Postgres at all. + +The whole file is validated the first time it is read: an unknown field name, a +transform on a field nobody publishes, or an allowlist naming a never-public +field raises. Failing at load rather than at request time is deliberate; the +alternative is discovering a typo by noticing data that should have been there, +or worse, data that should not have been. +""" + +from functools import lru_cache +from pathlib import Path + +import yaml + +from db.location import Location +from db.thing import Thing +from domain.field_projection import ( + EntityProjection, + NeverPublicFieldAllowed, + project, + validate_projection, +) + +CONFIG_PATH = Path(__file__).parent.parent / "core" / "field-allowlists.yml" + +THING = "thing" +LOCATION = "location" + +# Fields a projection may name, per entity. Model columns plus the derived +# values a payload carries instead of raw storage: `point` is a PostGIS +# geometry nobody consumes directly, so `latitude` and `longitude` stand in +# for it and can be rounded independently. +DERIVED_FIELDS = { + THING: frozenset(), + LOCATION: frozenset({"latitude", "longitude"}), +} +ENTITY_MODELS = {THING: Thing, LOCATION: Location} + + +def known_fields(entity: str) -> frozenset: + model = ENTITY_MODELS[entity] + columns = {column.name for column in model.__table__.columns} + return frozenset(columns | DERIVED_FIELDS[entity]) + + +@lru_cache(maxsize=1) +def _configuration(path: str = None) -> dict: + """Parse and validate the allowlists once per process.""" + raw = yaml.safe_load(Path(path or CONFIG_PATH).read_text(encoding="utf-8")) + + never_public = { + entity: frozenset(fields or ()) + for entity, fields in (raw.get("never_public") or {}).items() + } + + audiences = {} + for keyed_by in ("by_kind", "by_slug"): + for audience, entities in ( + (raw.get("audiences") or {}).get(keyed_by) or {} + ).items(): + for entity, rule in (entities or {}).items(): + projection = _build_projection(entity, rule, never_public) + audiences[(keyed_by, audience, entity)] = projection + + ogc = raw.get("ogc") or {} + ogc_never = frozenset(ogc.get("never_public") or ()) + ogc_collections = {} + for table, columns in (ogc.get("collections") or {}).items(): + columns = frozenset(columns or ()) + forbidden = sorted(columns & ogc_never) + if forbidden: + # The same rule as the audience allowlists: a never-public field + # cannot be re-admitted by naming it somewhere else. + raise NeverPublicFieldAllowed( + f"{table}.{forbidden[0]} is on the OGC never-public list and " + "cannot be published by a collection." + ) + ogc_collections[table] = columns + + return { + "never_public": never_public, + "audiences": audiences, + "ogc_never_public": ogc_never, + "ogc_collections": ogc_collections, + } + + +def _build_projection(entity: str, rule, never_public: dict) -> EntityProjection: + if entity not in ENTITY_MODELS: + raise KeyError( + f"'{entity}' is not a projectable entity " + f"({', '.join(sorted(ENTITY_MODELS))})." + ) + + # A bare list is the common case; the mapping form adds transforms. + if isinstance(rule, list): + fields, raw_transforms = rule, {} + else: + fields = (rule or {}).get("fields") or [] + raw_transforms = (rule or {}).get("transforms") or {} + + transforms = {} + for field_name, spec in raw_transforms.items(): + # One transform per field: {"round": 2} -> ("round", 2). + ((transform_name, argument),) = spec.items() + transforms[field_name] = (transform_name, argument) + + validate_projection( + entity=entity, + fields=fields, + transforms=transforms, + known_fields=known_fields(entity), + never_public=never_public.get(entity, frozenset()), + ) + return EntityProjection(fields=frozenset(fields), transforms=transforms) + + +def projection_for(destination, entity: str) -> EntityProjection | None: + """The rule for this destination, or None -- which means nothing is sent. + + A per-destination entry replaces its kind's rules rather than extending + them, so what one audience receives is readable in one place. + """ + configuration = _configuration()["audiences"] + by_slug = configuration.get(("by_slug", destination.slug, entity)) + if by_slug is not None: + return by_slug + return configuration.get(("by_kind", destination.destination_kind, entity)) + + +def never_public_fields(entity: str) -> frozenset: + return _configuration()["never_public"].get(entity, frozenset()) + + +# ============= OGC collections ============================================= + + +def ogc_never_public() -> frozenset: + """Fields no public OGC collection may publish.""" + return _configuration()["ogc_never_public"] + + +def ogc_allowlist(table: str) -> frozenset | None: + """Columns this public collection publishes, or None if it is not one. + + None means "not projected" and covers the internal mount, whose tables are + ``ogc_internal_*``. An empty frozenset means "listed nowhere", which is + default deny: the collection publishes no properties until someone says + what it may publish. + """ + if table is None or table.startswith("ogc_internal_"): + return None + + collections = _configuration()["ogc_collections"] + if table not in collections: + return frozenset() + return collections[table] + + +def ogc_collection_tables() -> frozenset: + """Every public table with an allowlist entry.""" + return frozenset(_configuration()["ogc_collections"]) + + +def thing_record(thing) -> dict: + """Every stored field of a thing, before projection.""" + return { + column.name: getattr(thing, column.name) + for column in Thing.__table__.columns + if column.name != "search_vector" + } + + +def location_record(location) -> dict: + """Every stored field of a location, with the geometry as lat/lon.""" + record = { + column.name: getattr(location, column.name) + for column in Location.__table__.columns + if column.name != "point" + } + latitude, longitude = location.latlon + record["latitude"] = latitude + record["longitude"] = longitude + return record + + +def project_entity(destination, entity: str, record: dict) -> dict: + """Apply this destination's rule to one record.""" + return project( + record, + projection_for(destination, entity), + never_public=never_public_fields(entity), + ) + + +# ============= EOF ============================================= diff --git a/services/release_schedule.py b/services/release_schedule.py new file mode 100644 index 000000000..308487f80 --- /dev/null +++ b/services/release_schedule.py @@ -0,0 +1,170 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +services/release_schedule.py + +Lifts embargoes whose date has arrived: `release_status = 'embargoed'` becomes +`'public'` once `release_at` is reached. + +**Why a job rather than a predicate in the views.** Seven of the public OGC +collections are materialized views refreshed by one pg_cron job at 09:00 UTC +(migration ``b6c7d8e9f0a1``). ``current_date`` inside a matview is frozen at +refresh time, so a date predicate in the view SQL would buy nothing on more +than half the public surface while costing a recreation of every relation. +The refresh already sets the granularity; this job runs before it and leaves +``release_status`` as the only thing the views test. + +**Fail closed.** If this job does not run, embargoed records stay embargoed. +The failure mode is data staying private a day longer than promised, which is +the direction that does not require an apology. + +**One direction.** Nothing here makes a record less visible. Withdrawing +something already published is an immediate `release_status` change made by a +person, for the reason ``domain/access.py`` never backdates a revocation. + +Every flip lands an ``authorization_audit`` row in the same transaction, and +records the date the embargo was set for, so the log answers "was this +released early" and not only "was this released". +""" + +from dataclasses import dataclass, field +from datetime import date + +from sqlalchemy import select + +from db.authorization_audit import RELEASE_LIFTED, AuthorizationAudit +from db.field import FieldActivity, FieldEvent +from db.location import Location +from db.observation import Observation +from db.sample import Sample +from db.thing import Thing +from domain.release import STATUS_EMBARGOED, STATUS_PUBLIC, due_for_release + +# Recorded as the actor. Not a person: nobody decided anything on the day the +# embargo lifted, which is the point of scheduling it. +RELEASE_ACTOR = "system:release-schedule" + +# The models a scheduled release may touch, named one by one. +# +# Every `ReleaseMixin` model carries `release_at`, so this could iterate the +# mapper registry instead. It does not, for the reason `domain/access.py` has +# no wildcard data type: a model added next year should not silently join the +# set of things a nightly job rewrites. Adding one here is a deliberate line +# in a diff. +# +# These six are the water-level chain the public OGC views actually filter on +# -- thing, location, and the field-data path down to the observation -- so an +# embargo on any of them changes what `/ogcapi` publishes. Chemistry is +# absent: those collections read the legacy `NMA_*` tables, which carry no +# release columns at all. See docs/data-embargo.md. +RELEASABLE_MODELS = ( + Thing, + Location, + FieldEvent, + FieldActivity, + Sample, + Observation, +) + + +@dataclass +class ReleasePlan: + """What lifting would do, or did.""" + + lifted: list = field(default_factory=list) + + def describe(self, entry) -> str: + table, row_id, release_at = entry + return f"{table}:{row_id} embargoed until {release_at.isoformat()}" + + def by_table(self) -> dict: + counts: dict = {} + for table, _, _ in self.lifted: + counts[table] = counts.get(table, 0) + 1 + return counts + + +def _due_rows(session, model, on_date: date) -> list: + """Embargoed rows of one model whose date has arrived. + + The date test is narrowed in SQL so the job does not load every embargoed + row in the database, and then applied again by ``due_for_release`` on each + candidate. The rule lives in the domain function; the WHERE clause is an + optimisation that is allowed to be looser than it, never tighter. + """ + rows = session.execute( + select(model).where( + model.release_status == STATUS_EMBARGOED, + model.release_at.is_not(None), + model.release_at <= on_date, + ) + ).scalars() + return [ + row + for row in rows + if due_for_release(row.release_status, row.release_at, on_date) + ] + + +def lift_due_embargoes( + session, on_date: date = None, apply: bool = True +) -> ReleasePlan: + """Publish every record whose embargo has run out. Safe to run repeatedly. + + With ``apply=False`` nothing is written and the plan describes what would + be. Release is security state, so the CLI previews by default. + + Idempotent by construction: a lifted row is no longer ``embargoed``, so the + next run does not see it. + """ + on_date = on_date or date.today() + plan = ReleasePlan() + + for model in RELEASABLE_MODELS: + table = model.__tablename__ + for row in _due_rows(session, model, on_date): + plan.lifted.append((table, row.id, row.release_at)) + if not apply: + continue + + embargoed_until = row.release_at + row.release_status = STATUS_PUBLIC + # Cleared so the row stops advertising a schedule it has already + # kept, and so `validate_release` still holds for it afterwards: + # a release_at on a public row schedules nothing. + row.release_at = None + session.add( + AuthorizationAudit( + event_type=RELEASE_LIFTED, + actor=RELEASE_ACTOR, + subject_table=table, + subject_id=row.id, + detail={ + "release_at": embargoed_until.isoformat(), + "released_on": on_date.isoformat(), + "from_status": STATUS_EMBARGOED, + "to_status": STATUS_PUBLIC, + }, + ) + ) + + if apply and plan.lifted: + session.commit() + + return plan + + +# ============= EOF ============================================= diff --git a/services/sample_helper.py b/services/sample_helper.py index 0f25dd9a1..6b4a5bb5a 100644 --- a/services/sample_helper.py +++ b/services/sample_helper.py @@ -41,7 +41,7 @@ GroupThingAssociation.group ), THING_RESPONSE_BASE.selectinload(Thing.notes), - THING_RESPONSE_BASE.selectinload(Thing.permission_history), + THING_RESPONSE_BASE.selectinload(Thing.field_access_consent), THING_RESPONSE_BASE.selectinload(Thing.data_provenance), THING_RESPONSE_BASE.selectinload(Thing.status_history), ) diff --git a/services/scoped_transfer.py b/services/scoped_transfer.py index 39749de28..6dac0aaf5 100644 --- a/services/scoped_transfer.py +++ b/services/scoped_transfer.py @@ -31,7 +31,7 @@ NMA_Chemistry_SampleInfo, Notes, Observation, - PermissionHistory, + FieldAccessConsent, Sample, Thing, ThingContactAssociation, @@ -1393,11 +1393,11 @@ def _execute_permissions(pointids: list[str]) -> ScopedFamilyResult: existing_permissions = { (target_id, contact_id, permission_type) for target_id, contact_id, permission_type in session.query( - PermissionHistory.target_id, - PermissionHistory.contact_id, - PermissionHistory.permission_type, + FieldAccessConsent.target_id, + FieldAccessConsent.contact_id, + FieldAccessConsent.permission_type, ) - .filter(PermissionHistory.target_table == "thing") + .filter(FieldAccessConsent.target_table == "thing") .all() } diff --git a/services/util.py b/services/util.py index dbf88f98e..ad1b8b84c 100644 --- a/services/util.py +++ b/services/util.py @@ -267,7 +267,7 @@ def retrieve_latest_polymorphic_history_table_record( DeclarativeBase | None The latest record from the specified polymorphic table with the defined type if it exists. """ - if polymorphic_relationship == "permission_history": + if polymorphic_relationship == "field_access_consent": type_field = "permission_type" elif polymorphic_relationship == "status_history": type_field = "status_type" diff --git a/services/visibility.py b/services/visibility.py new file mode 100644 index 000000000..57becb338 --- /dev/null +++ b/services/visibility.py @@ -0,0 +1,288 @@ +# =============================================================================== +# Copyright 2025 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +""" +services/visibility.py + +The one place that answers "what may this principal see, and what has this +destination been given" (ADR5, Part IV). + +Every consumer asks here. Not the public API filtering one way and a harvester +view another: three services filtering independently give three subtly +different answers to "is this published", and an outside party finds the +difference first. That is not hypothetical in this repository -- migration +baba91fe5e83 fixed exactly that between two OGC views. + +This module loads rows and hands them to ``domain/access.py``, which holds the +rules, and to ``services/field_projection.py``, which decides field by field +what an audience receives. It decides nothing itself. + +Grants are read at request time and never cached here. ADR5 A.5 allows a short +per-principal cache with explicit invalidation on revoke; an unbounded one +would silently defeat the immediate-revocation promise, so the first version +has none. +""" + +from datetime import date + +from sqlalchemy import select + +from db.destination import Destination +from db.group import GroupThingAssociation +from db.location import Location, LocationThingAssociation +from db.permission_grant import PermissionGrant +from db.publication_consent import PublicationConsent +from db.thing import Thing +from services.field_projection import ( + LOCATION, + THING, + location_record, + project_entity, + thing_record, +) +from domain.access import ( + AccessRequest, + Consent, + Grant, + PRINCIPAL_ROLE, + PRINCIPAL_USER, + any_grant_allows, + consent_covers, +) + +# Claims carrying the caller's identity and their Authentik groups. +SUBJECT_CLAIM = "sub" +GROUPS_CLAIM = "groups" + + +def principals_from_payload(payload) -> tuple[tuple[str, str], ...]: + """Every identity a caller presents, as (principal_type, principal_id). + + A caller is their subject *and* each role they hold, because a grant may + name any of them. + + The development bypass returns ``True`` rather than a token payload, and an + anonymous caller has no payload at all. Both yield no principals, which + means default deny: the bypass turns off authentication, not authorization. + """ + if not isinstance(payload, dict): + return () + + principals = [] + subject = payload.get(SUBJECT_CLAIM) + if subject: + principals.append((PRINCIPAL_USER, str(subject))) + for group in payload.get(GROUPS_CLAIM) or []: + principals.append((PRINCIPAL_ROLE, str(group))) + return tuple(principals) + + +def group_ids_for_thing(session, thing_id: int) -> tuple[int, ...]: + """The groups a thing belongs to, which is what a project grant reaches.""" + if thing_id is None: + return () + rows = session.execute( + select(GroupThingAssociation.group_id).where( + GroupThingAssociation.thing_id == thing_id + ) + ).all() + return tuple(row[0] for row in rows) + + +def load_grants(session, principals: tuple[tuple[str, str], ...]) -> list[Grant]: + """Live and expired grants for these principals, as domain values. + + Expired and revoked rows are loaded rather than filtered in SQL so the + date rule lives in one place -- ``domain.access.is_active`` -- instead of + being restated as a WHERE clause that can drift from it. + """ + if not principals: + return [] + + principal_ids = {principal_id for _, principal_id in principals} + rows = session.execute( + select(PermissionGrant).where(PermissionGrant.principal_id.in_(principal_ids)) + ).scalars() + + return [ + Grant( + principal_type=row.principal_type, + principal_id=row.principal_id, + capability=row.capability, + scope_type=row.scope_type, + scope_id=row.scope_id, + data_type=row.data_type, + starts_at=row.starts_at, + ends_at=row.ends_at, + revoked_at=row.revoked_at, + ) + for row in rows + ] + + +def may( + session, + principals: tuple[tuple[str, str], ...], + capability: str, + data_type: str, + thing_id: int = None, + on_date: date = None, +) -> bool: + """May these principals do this, to this data type, at this thing? + + Default deny. No principals, no grants, or nothing matching is a no. + """ + request = AccessRequest( + capability=capability, + data_type=data_type, + principals=tuple(principals), + thing_id=thing_id, + group_ids=group_ids_for_thing(session, thing_id), + ) + return any_grant_allows( + load_grants(session, request.principals), request, on_date or date.today() + ) + + +def destination_by_slug(session, slug: str) -> Destination | None: + return session.execute( + select(Destination).where(Destination.slug == slug) + ).scalar_one_or_none() + + +def _consent_rows(session, destination_id: int, data_type: str = None): + statement = select(PublicationConsent).where( + PublicationConsent.destination_id == destination_id + ) + if data_type: + statement = statement.where(PublicationConsent.data_type == data_type) + return session.execute(statement).scalars().all() + + +def published_data_types( + session, destination: Destination, thing_id: int, on_date: date = None +) -> list[str]: + """Data types this destination may read for one thing.""" + if not destination.active: + return [] + + on_date = on_date or date.today() + return sorted( + { + row.data_type + for row in _consent_rows(session, destination.id) + if row.thing_id == thing_id + and consent_covers( + Consent( + thing_id=row.thing_id, + destination_id=row.destination_id, + data_type=row.data_type, + starts_at=row.starts_at, + ends_at=row.ends_at, + revoked_at=row.revoked_at, + ), + thing_id, + destination.id, + row.data_type, + on_date, + ) + } + ) + + +def published_things( + session, destination: Destination, data_type: str = None, on_date: date = None +) -> list[dict]: + """What this destination gets: one entry per thing, with its data types. + + A retired destination gets nothing, without the caller having to remember + to check. + """ + if not destination.active: + return [] + + on_date = on_date or date.today() + by_thing: dict[int, set] = {} + for row in _consent_rows(session, destination.id, data_type): + covered = consent_covers( + Consent( + thing_id=row.thing_id, + destination_id=row.destination_id, + data_type=row.data_type, + starts_at=row.starts_at, + ends_at=row.ends_at, + revoked_at=row.revoked_at, + ), + row.thing_id, + destination.id, + row.data_type, + on_date, + ) + if covered: + by_thing.setdefault(row.thing_id, set()).add(row.data_type) + + if not by_thing: + return [] + + things = { + thing.id: thing + for thing in session.execute( + select(Thing).where(Thing.id.in_(by_thing.keys())) + ).scalars() + } + locations = _current_locations(session, by_thing.keys()) + + entries = [] + for thing_id, data_types in sorted(by_thing.items()): + thing = things.get(thing_id) + if thing is None: + # Consent outliving its thing is a data problem, not a reason to + # publish a record nobody can check. + continue + + location = locations.get(thing_id) + entries.append( + { + "thing_id": thing_id, + "data_types": sorted(data_types), + # Field by field, per audience, at the one chokepoint. A field + # nobody approved for this destination is absent, and a + # coordinate may arrive rounded rather than exact. + "properties": project_entity(destination, THING, thing_record(thing)), + "location": ( + project_entity(destination, LOCATION, location_record(location)) + if location is not None + else {} + ), + } + ) + + return entries + + +def _current_locations(session, thing_ids) -> dict: + """The location each thing sits at now, keyed by thing id.""" + rows = session.execute( + select(LocationThingAssociation.thing_id, Location) + .join(Location, Location.id == LocationThingAssociation.location_id) + .where( + LocationThingAssociation.thing_id.in_(thing_ids), + LocationThingAssociation.effective_end.is_(None), + ) + ).all() + return {thing_id: location for thing_id, location in rows} + + +# ============= EOF ============================================= diff --git a/services/well_inventory_csv.py b/services/well_inventory_csv.py index ccb2863b5..c041373d8 100644 --- a/services/well_inventory_csv.py +++ b/services/well_inventory_csv.py @@ -38,7 +38,7 @@ FieldEventParticipant, FieldActivity, Contact, - PermissionHistory, + FieldAccessConsent, Thing, ThingContactAssociation, Sample, @@ -425,11 +425,11 @@ def _make_well_permission( permission_type: str, permission_allowed: bool, start_date: date, -) -> PermissionHistory: +) -> FieldAccessConsent: """ - Makes a PermissionHistory record for the given well and contact. + Makes a FieldAccessConsent record for the given well and contact. If the contact has not been provided, but a permission is to be created, - no PermissionHistory record is created and a 400 error is raised. + no FieldAccessConsent record is created and a 400 error is raised. """ if contact is None: raise PydanticStyleException( @@ -444,7 +444,7 @@ def _make_well_permission( ], ) - permission = PermissionHistory( + permission = FieldAccessConsent( target_table="thing", target_id=well.id, contact=contact, diff --git a/tests/features/environment.py b/tests/features/environment.py index d3c1b47cd..c0af8423c 100644 --- a/tests/features/environment.py +++ b/tests/features/environment.py @@ -39,7 +39,7 @@ Deployment, TransducerObservationBlock, WellCasingMaterial, - PermissionHistory, + FieldAccessConsent, StatusHistory, ThingIdLink, WellPurpose, @@ -247,7 +247,7 @@ def add_contact(context, session): @add_context_object_container("permission_histories") -def add_permission_history( +def add_field_access_consent( context, session, contact_id, @@ -259,7 +259,7 @@ def add_permission_history( target_id, target_table, ): - permission_history = PermissionHistory( + field_access_consent = FieldAccessConsent( contact_id=contact_id, permission_type=permission_type, permission_allowed=permission_allowed, @@ -269,12 +269,12 @@ def add_permission_history( target_id=target_id, target_table=target_table, ) - session.add(permission_history) + session.add(field_access_consent) session.commit() - session.refresh(permission_history) + session.refresh(field_access_consent) - context.objects["permission_histories"].append(permission_history) - return permission_history + context.objects["permission_histories"].append(field_access_consent) + return field_access_consent @add_context_object_container("sensors") @@ -702,7 +702,7 @@ def before_all(context): "Water Level Sample", "Water Chemistry Sample", ]: - add_permission_history( + add_field_access_consent( context, session, contact_id=context.objects["contacts"][0].id, diff --git a/tests/features/steps/ogc-cleanup-sprint1.py b/tests/features/steps/ogc-cleanup-sprint1.py index 5ccf0b229..810c26dd9 100644 --- a/tests/features/steps/ogc-cleanup-sprint1.py +++ b/tests/features/steps/ogc-cleanup-sprint1.py @@ -425,10 +425,17 @@ def _ensure_head(context): @given("a clean database state before the Sprint 1 migration") def step_given_clean_database_state(context): - command.downgrade(_alembic_config(), PRE_A1_REVISION) + # Seed at head, then downgrade -- not the other way around. _seed_all() + # writes through the ORM, whose models carry every column added by + # migrations newer than PRE_A1_REVISION (data_maturity, for one), so + # seeding against the downgraded schema fails on the missing columns. + # Downgrading afterwards drops those columns but keeps the rows, which + # is exactly the pre-migration state these scenarios need. + command.upgrade(_alembic_config(), "head") with session_ctx() as session: context.seed_ids = _seed_all(session) context.add_cleanup(_teardown_a1_seed_data) + command.downgrade(_alembic_config(), PRE_A1_REVISION) @when("the Sprint 1 Alembic migration is applied") @@ -650,12 +657,15 @@ def _seed_already_consistent_layers(session): @given("the following layers were already filtering correctly before the migration:") def step_given_already_consistent_layers(context): - command.downgrade(_alembic_config(), PRE_A1_REVISION) - reset_pygeoapi_reflection() + # Seed at head before downgrading, for the reason given in + # step_given_clean_database_state. + command.upgrade(_alembic_config(), "head") with session_ctx() as session: _seed_already_consistent_layers(session) session.commit() context.add_cleanup(_teardown_a1_seed_data) + command.downgrade(_alembic_config(), PRE_A1_REVISION) + reset_pygeoapi_reflection() context.already_consistent_counts = {} for row in context.table: diff --git a/tests/features/steps/well-additional-information.py b/tests/features/steps/well-additional-information.py index c34f17b66..e3e88e244 100644 --- a/tests/features/steps/well-additional-information.py +++ b/tests/features/steps/well-additional-information.py @@ -14,7 +14,7 @@ def step_step_step(context): assert "permissions" in context.water_well_data permission_record = retrieve_latest_polymorphic_history_table_record( - context.objects["wells"][0], "permission_history", permission_type + context.objects["wells"][0], "field_access_consent", permission_type ) water_well_data_permissions = [ @@ -49,7 +49,7 @@ def step_then_the_response_should_include_whether_sampling_permission_is_granted assert "permissions" in context.water_well_data permission_record = retrieve_latest_polymorphic_history_table_record( - context.objects["wells"][0], "permission_history", permission_type + context.objects["wells"][0], "field_access_consent", permission_type ) water_well_data_permissions = [ @@ -84,7 +84,7 @@ def step_step_step_2(context): assert "permissions" in context.water_well_data permission_record = retrieve_latest_polymorphic_history_table_record( - context.objects["wells"][0], "permission_history", permission_type + context.objects["wells"][0], "field_access_consent", permission_type ) water_well_data_permissions = [ diff --git a/tests/test_access.py b/tests/test_access.py new file mode 100644 index 000000000..367b1df7c --- /dev/null +++ b/tests/test_access.py @@ -0,0 +1,465 @@ +# =============================================================================== +# Copyright 2025 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""The ADR5 visibility layer, end to end through its first tenant. + +These exercise the promise the model is for: an owner who shares water levels +but not chemistry, and a withdrawal that takes effect on the next read. +""" + +from datetime import date, timedelta + +import pytest +from sqlalchemy import delete, select + +from core.dependencies import admin_function, viewer_function +from main import app +from db.authorization_audit import AuthorizationAudit +from db.destination import Destination +from db.engine import session_ctx +from db.permission_grant import PermissionGrant +from db.publication_consent import PublicationConsent +from tests import client, override_authentication + +ADMIN_PAYLOAD = {"sub": "test-admin", "groups": ["Admin"]} +SLUG = "test-harvester" +TODAY = date.today() + + +@pytest.fixture(autouse=True) +def override_authentication_dependency_fixture(): + app.dependency_overrides[admin_function] = override_authentication( + default=ADMIN_PAYLOAD + ) + app.dependency_overrides[viewer_function] = override_authentication( + default=ADMIN_PAYLOAD + ) + + yield + + app.dependency_overrides = {} + + +@pytest.fixture +def destination(): + response = client.post( + "/access/destination", + json={ + "slug": SLUG, + "name": "Test Harvester", + "destination_kind": "harvester", + "description": "Stands in for NGWMN.", + }, + ) + assert response.status_code == 201, response.text + created = response.json() + + yield created + + with session_ctx() as session: + session.execute( + delete(PublicationConsent).where( + PublicationConsent.destination_id == created["id"] + ) + ) + session.execute(delete(Destination).where(Destination.id == created["id"])) + session.execute( + delete(AuthorizationAudit).where( + AuthorizationAudit.actor == ADMIN_PAYLOAD["sub"] + ) + ) + session.commit() + + +@pytest.fixture +def public_destination(): + response = client.post( + "/access/destination", + json={ + "slug": "test-public-web", + "name": "Test Public Web", + "destination_kind": "public web", + }, + ) + assert response.status_code == 201, response.text + created = response.json() + + yield created + + with session_ctx() as session: + session.execute( + delete(PublicationConsent).where( + PublicationConsent.destination_id == created["id"] + ) + ) + session.execute(delete(Destination).where(Destination.id == created["id"])) + session.execute( + delete(AuthorizationAudit).where( + AuthorizationAudit.actor == ADMIN_PAYLOAD["sub"] + ) + ) + session.commit() + + +@pytest.fixture +def grants(): + created = [] + + yield created + + with session_ctx() as session: + for grant_id in created: + session.execute( + delete(PermissionGrant).where(PermissionGrant.id == grant_id) + ) + session.execute( + delete(AuthorizationAudit).where( + AuthorizationAudit.actor == ADMIN_PAYLOAD["sub"] + ) + ) + session.commit() + + +def consent_to(thing_id, data_type, **overrides): + payload = { + "thing_id": thing_id, + "destination_slug": SLUG, + "data_type": data_type, + "starts_at": TODAY.isoformat(), + } + payload.update(overrides) + return client.post("/access/consent", json=payload) + + +def published(**params): + response = client.get(f"/access/destination/{SLUG}/thing", params=params) + assert response.status_code == 200, response.text + return response.json() + + +# ------ destinations ---------- + + +def test_register_a_destination(destination): + assert destination["slug"] == SLUG + assert destination["active"] is True + + +def test_a_duplicate_slug_is_a_conflict(destination): + response = client.post( + "/access/destination", + json={"slug": SLUG, "name": "Again", "destination_kind": "harvester"}, + ) + assert response.status_code == 409 + + +def test_an_unknown_destination_is_a_404(): + response = client.get("/access/destination/nobody-registered-this/thing") + assert response.status_code == 404 + + +# ------ the case the model exists for ---------- + + +def test_levels_yes_chemistry_no(destination, water_well_thing): + """The kitchen-table promise: per data type, on one well.""" + assert consent_to(water_well_thing.id, "water level").status_code == 201 + + entries = published() + assert len(entries) == 1 + assert entries[0]["thing_id"] == water_well_thing.id + assert entries[0]["data_types"] == ["water level"] + assert entries[0]["properties"]["name"] == water_well_thing.name + + assert published(data_type="water chemistry") == [] + + +# ------ field projection ---------- + + +def test_a_destination_receives_only_its_allowlisted_fields( + destination, water_well_thing +): + consent_to(water_well_thing.id, "water level") + properties = published()[0]["properties"] + + # Named for the harvester audience in core/field-allowlists.yml. + assert set(properties) == { + "id", + "name", + "thing_type", + "well_depth", + "hole_depth", + "well_casing_depth", + "well_completion_date", + } + + +def test_never_public_fields_reach_nobody(destination, water_well_thing): + consent_to(water_well_thing.id, "water level") + entry = published()[0] + + for column in ("created_by_id", "created_by_name", "nma_pk_welldata"): + assert column not in entry["properties"] + for column in ("nma_location_notes", "nma_coordinate_notes"): + assert column not in entry["location"] + + +def test_coordinates_are_rounded_for_the_audience( + destination, public_destination, water_well_thing +): + """The same well, two audiences, two precisions -- not published or hidden.""" + consent_to(water_well_thing.id, "water level") + client.post( + "/access/consent", + json={ + "thing_id": water_well_thing.id, + "destination_slug": public_destination["slug"], + "data_type": "water level", + "starts_at": TODAY.isoformat(), + }, + ) + + harvester_location = published()[0]["location"] + public_response = client.get( + f"/access/destination/{public_destination['slug']}/thing" + ) + public_location = public_response.json()[0]["location"] + + assert harvester_location["latitude"] == round(harvester_location["latitude"], 4) + assert public_location["latitude"] == round(public_location["latitude"], 2) + assert public_location["latitude"] != harvester_location["latitude"] + # Rounded, not withheld: the well still appears on the public map. + assert public_location["latitude"] is not None + + +def test_nothing_is_published_without_consent(destination, water_well_thing): + """Default deny. A registered destination starts with nothing.""" + assert published() == [] + + +def test_withdrawal_takes_effect_on_the_next_read(destination, water_well_thing): + consent_id = consent_to(water_well_thing.id, "water level").json()["id"] + assert len(published()) == 1 + + revocation = client.post(f"/access/consent/{consent_id}/revocation") + assert revocation.status_code == 201 + assert revocation.json()["revoked_by"] == ADMIN_PAYLOAD["sub"] + + assert published() == [] + + +def test_consent_cannot_be_withdrawn_twice(destination, water_well_thing): + consent_id = consent_to(water_well_thing.id, "water level").json()["id"] + client.post(f"/access/consent/{consent_id}/revocation") + + assert client.post(f"/access/consent/{consent_id}/revocation").status_code == 422 + + +def test_withdrawn_consent_is_kept_for_the_record(destination, water_well_thing): + consent_id = consent_to(water_well_thing.id, "water level").json()["id"] + client.post(f"/access/consent/{consent_id}/revocation") + + live = client.get("/access/consent", params={"thing_id": water_well_thing.id}) + assert live.json() == [] + + history = client.get( + "/access/consent", + params={"thing_id": water_well_thing.id, "include_revoked": True}, + ) + assert [row["id"] for row in history.json()] == [consent_id] + + +def test_expired_consent_stops_publishing(destination, water_well_thing): + yesterday = TODAY - timedelta(days=1) + consent_to( + water_well_thing.id, + "water level", + starts_at=(TODAY - timedelta(days=30)).isoformat(), + ends_at=yesterday.isoformat(), + ) + + assert published() == [] + assert len(published(on_date=yesterday.isoformat())) == 1 + + +def test_consent_to_an_unknown_destination_is_a_404(water_well_thing): + response = client.post( + "/access/consent", + json={ + "thing_id": water_well_thing.id, + "destination_slug": "not-registered", + "data_type": "water level", + "starts_at": TODAY.isoformat(), + }, + ) + assert response.status_code == 404 + + +# ------ grants ---------- + + +def make_grant(grants, **overrides): + payload = { + "principal_type": "user", + "principal_id": ADMIN_PAYLOAD["sub"], + "capability": "read", + "scope_type": "global", + "scope_id": None, + "data_type": "water level", + "starts_at": TODAY.isoformat(), + "reason": "test", + } + payload.update(overrides) + response = client.post("/access/grant", json=payload) + if response.status_code == 201: + grants.append(response.json()["id"]) + return response + + +def decision(**params): + response = client.get("/access/decision", params=params) + assert response.status_code == 200, response.text + return response.json() + + +def test_a_grant_answers_only_for_its_data_type(grants): + assert make_grant(grants).status_code == 201 + + assert decision(capability="read", data_type="water level")["allowed"] is True + assert decision(capability="read", data_type="water chemistry")["allowed"] is False + + +def test_no_grant_is_a_no(grants): + assert decision(capability="read", data_type="water level")["allowed"] is False + + +def test_a_grant_answers_only_for_its_capability(grants): + make_grant(grants) + + assert decision(capability="correct", data_type="water level")["allowed"] is False + + +def test_revoking_a_grant_is_effective_immediately(grants): + grant_id = make_grant(grants).json()["id"] + assert decision(capability="read", data_type="water level")["allowed"] is True + + assert client.post(f"/access/grant/{grant_id}/revocation").status_code == 201 + + assert decision(capability="read", data_type="water level")["allowed"] is False + + +def test_a_thing_scoped_grant_stops_at_that_thing(grants, water_well_thing): + make_grant(grants, scope_type="thing", scope_id=water_well_thing.id) + + covered = decision( + capability="read", data_type="water level", thing_id=water_well_thing.id + ) + other = decision( + capability="read", data_type="water level", thing_id=water_well_thing.id + 9999 + ) + assert covered["allowed"] is True + assert other["allowed"] is False + + +def test_a_global_grant_with_a_scope_id_is_rejected(grants): + """The rule lives in domain/access.py; the route surfaces it as a 422.""" + assert make_grant(grants, scope_id=7).status_code == 422 + + +def test_a_grant_naming_no_data_type_cannot_be_written(grants): + """No wildcards. Pydantic rejects it before the service is reached.""" + assert make_grant(grants, data_type=None).status_code == 422 + + +def test_listing_grants_with_no_filter_returns_everything(grants): + grant_id = make_grant(grants).json()["id"] + + everyone = client.get("/access/grant") + assert grant_id in [row["id"] for row in everyone.json()] + + +def test_listing_grants_filters_by_data_type(grants): + grant_id = make_grant(grants).json()["id"] + + match = client.get("/access/grant", params={"data_type": "water level"}) + assert grant_id in [row["id"] for row in match.json()] + + no_match = client.get("/access/grant", params={"data_type": "water chemistry"}) + assert grant_id not in [row["id"] for row in no_match.json()] + + +def test_listing_grants_hides_revoked_ones_by_default(grants): + grant_id = make_grant(grants).json()["id"] + client.post(f"/access/grant/{grant_id}/revocation") + + live = client.get("/access/grant", params={"principal_id": ADMIN_PAYLOAD["sub"]}) + assert live.json() == [] + + history = client.get( + "/access/grant", + params={"principal_id": ADMIN_PAYLOAD["sub"], "include_revoked": True}, + ) + assert [row["id"] for row in history.json()] == [grant_id] + + +# ------ audit ---------- + + +def audit_events(): + with session_ctx() as session: + return [ + row.event_type + for row in session.execute( + select(AuthorizationAudit) + .where(AuthorizationAudit.actor == ADMIN_PAYLOAD["sub"]) + .order_by(AuthorizationAudit.id) + ).scalars() + ] + + +def test_every_authorization_change_is_logged(destination, grants, water_well_thing): + consent_id = consent_to(water_well_thing.id, "water level").json()["id"] + client.post(f"/access/consent/{consent_id}/revocation") + grant_id = make_grant(grants).json()["id"] + client.post(f"/access/grant/{grant_id}/revocation") + + assert audit_events() == [ + "destination.registered", + "consent.recorded", + "consent.revoked", + "grant.created", + "grant.revoked", + ] + + +def test_the_log_records_who_and_what(destination, water_well_thing): + consent_to(water_well_thing.id, "water level") + + with session_ctx() as session: + entry = ( + session.execute( + select(AuthorizationAudit) + .where(AuthorizationAudit.event_type == "consent.recorded") + .order_by(AuthorizationAudit.id.desc()) + ) + .scalars() + .first() + ) + + assert entry.actor == ADMIN_PAYLOAD["sub"] + assert entry.subject_table == "publication_consent" + assert entry.detail["thing_id"] == water_well_thing.id + assert entry.detail["data_type"] == "water level" diff --git a/tests/test_access_seed.py b/tests/test_access_seed.py new file mode 100644 index 000000000..f88037a82 --- /dev/null +++ b/tests/test_access_seed.py @@ -0,0 +1,217 @@ +# =============================================================================== +# Copyright 2025 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""The day-one role baseline (ADR5, 5.2). + +An empty grant table denies everyone, including the people who run the system. +These cover the seeder that writes today's access down, and the property that +matters more than convenience: it never gives back what someone took away. +""" + +from datetime import date, datetime, timezone + +import pytest +from sqlalchemy import delete, select + +from core.dependencies import admin_function, viewer_function +from db.authorization_audit import AuthorizationAudit +from db.engine import session_ctx +from db.permission_grant import PermissionGrant +from main import app +from services.access_seed import ( + ROLE_BASELINE, + SEED_ACTOR, + planned_entries, + seed_role_grants, +) +from tests import client, override_authentication + +VIEWER_PAYLOAD = {"sub": "test-viewer", "groups": ["Viewer"]} + + +@pytest.fixture(autouse=True) +def seeded_grants(): + app.dependency_overrides[admin_function] = override_authentication( + default=VIEWER_PAYLOAD + ) + app.dependency_overrides[viewer_function] = override_authentication( + default=VIEWER_PAYLOAD + ) + + yield + + app.dependency_overrides = {} + with session_ctx() as session: + session.execute( + delete(PermissionGrant).where(PermissionGrant.granted_by == SEED_ACTOR) + ) + session.execute( + delete(AuthorizationAudit).where(AuthorizationAudit.actor == SEED_ACTOR) + ) + session.commit() + + +def seeded_rows(session): + return ( + session.execute( + select(PermissionGrant).where(PermissionGrant.granted_by == SEED_ACTOR) + ) + .scalars() + .all() + ) + + +# ------ the seeder ---------- + + +def test_seeding_writes_one_row_per_role_capability_and_data_type(): + with session_ctx() as session: + plan = seed_role_grants(session) + assert len(plan.created) == len(planned_entries()) + assert len(seeded_rows(session)) == len(planned_entries()) + + +def test_seeding_twice_creates_nothing_the_second_time(): + with session_ctx() as session: + seed_role_grants(session) + again = seed_role_grants(session) + + assert again.created == [] + assert len(again.skipped) == len(planned_entries()) + + +def test_a_preview_writes_nothing(): + with session_ctx() as session: + plan = seed_role_grants(session, apply=False) + assert plan.created + assert seeded_rows(session) == [] + + +def test_a_revoked_baseline_grant_is_not_resurrected(): + """Narrowing the baseline is the point; re-running must not undo it.""" + with session_ctx() as session: + seed_role_grants(session) + grant = ( + session.execute( + select(PermissionGrant).where( + PermissionGrant.granted_by == SEED_ACTOR, + PermissionGrant.principal_id == "Viewer", + PermissionGrant.data_type == "water chemistry", + ) + ) + .scalars() + .one() + ) + grant.revoked_at = datetime.now(timezone.utc) + grant.revoked_by = "someone-who-decided" + session.commit() + + plan = seed_role_grants(session) + + assert plan.created == [] + + +def test_every_seeded_grant_is_marked_as_history_not_judgement(): + with session_ctx() as session: + seed_role_grants(session) + rows = seeded_rows(session) + + assert all(row.granted_by == SEED_ACTOR for row in rows) + assert all("ADR5" in row.reason for row in rows) + assert all(row.scope_type == "global" and row.scope_id is None for row in rows) + + +def test_seeding_is_audited(): + with session_ctx() as session: + seed_role_grants(session) + events = ( + session.execute( + select(AuthorizationAudit).where(AuthorizationAudit.actor == SEED_ACTOR) + ) + .scalars() + .all() + ) + + assert len(events) == len(planned_entries()) + assert {event.event_type for event in events} == {"grant.created"} + + +def test_a_viewer_gets_read_and_nothing_else(): + """The tiers nest within a family; the baseline has to say so.""" + assert ROLE_BASELINE["Viewer"] == ("read",) + assert set(ROLE_BASELINE["Admin"]) > set(ROLE_BASELINE["Editor"]) + assert set(ROLE_BASELINE["Editor"]) > set(ROLE_BASELINE["Viewer"]) + + +def test_the_dark_workbench_group_is_not_seeded(): + """AMP.Staging gates a workbench still being validated. Seeding it would + grant exactly the access nobody intended.""" + assert "AMP.Staging" not in ROLE_BASELINE + assert not any(role.startswith("Lexicon") for role in ROLE_BASELINE) + + +# ------ what it is for ---------- + + +def test_decision_says_no_before_seeding(): + response = client.get( + "/access/decision", params={"capability": "read", "data_type": "water level"} + ) + assert response.json()["allowed"] is False + + +def test_decision_says_yes_to_a_role_holder_after_seeding(): + with session_ctx() as session: + seed_role_grants(session) + + allowed = client.get( + "/access/decision", params={"capability": "read", "data_type": "water level"} + ).json() + assert allowed["allowed"] is True + assert "role:Viewer" in allowed["principals"] + + +def test_a_viewer_still_cannot_correct_after_seeding(): + with session_ctx() as session: + seed_role_grants(session) + + response = client.get( + "/access/decision", + params={"capability": "correct", "data_type": "water level"}, + ) + assert response.json()["allowed"] is False + + +def test_the_baseline_is_global_so_it_reaches_any_thing(): + with session_ctx() as session: + seed_role_grants(session) + + response = client.get( + "/access/decision", + params={ + "capability": "read", + "data_type": "water level", + "thing_id": 999999, + }, + ) + assert response.json()["allowed"] is True + + +def test_seeded_grants_start_today_not_retroactively(): + with session_ctx() as session: + seed_role_grants(session) + rows = seeded_rows(session) + + assert all(row.starts_at == date.today() for row in rows) diff --git a/tests/test_domain_access.py b/tests/test_domain_access.py new file mode 100644 index 000000000..0362794e5 --- /dev/null +++ b/tests/test_domain_access.py @@ -0,0 +1,272 @@ +# =============================================================================== +# Copyright 2025 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""Access-control rules (ADR5). No database, no fixtures.""" + +from datetime import date, datetime, timezone + +import pytest + +from domain.access import ( + AccessRequest, + CAPABILITIES, + PRINCIPAL_TYPES, + SCOPE_TYPES, + BackwardsDateRange, + Consent, + Grant, + MissingDataType, + ScopeIdMismatch, + UnknownCapability, + UnknownPrincipalType, + UnknownScopeType, + any_consent_publishes, + any_grant_allows, + consent_covers, + grant_covers, + is_active, + scope_covers, + validate_grant, +) + +TODAY = date(2026, 8, 24) +STUDENT = ("user", "authentik-sub-1") +EDITOR_ROLE = ("role", "Editor") + + +def a_grant(**overrides): + fields = { + "principal_type": "user", + "principal_id": "authentik-sub-1", + "capability": "read", + "scope_type": "thing", + "scope_id": 7, + "data_type": "water level", + "starts_at": date(2026, 1, 1), + "ends_at": None, + "revoked_at": None, + } + fields.update(overrides) + return Grant(**fields) + + +def a_request(**overrides): + fields = { + "capability": "read", + "data_type": "water level", + "principals": (STUDENT,), + "thing_id": 7, + "group_ids": (), + } + fields.update(overrides) + return AccessRequest(**fields) + + +# ------ default deny ---------- + + +def test_no_grants_is_a_no(): + assert any_grant_allows([], a_request(), TODAY) is False + + +def test_no_principals_is_a_no(): + assert any_grant_allows([a_grant()], a_request(principals=()), TODAY) is False + + +def test_a_matching_grant_allows(): + assert any_grant_allows([a_grant()], a_request(), TODAY) is True + + +# ------ every axis has to match ---------- + + +@pytest.mark.parametrize( + "override", + [ + {"capability": "correct"}, + {"data_type": "water chemistry"}, + {"principal_id": "somebody-else"}, + {"scope_id": 8}, + ], +) +def test_one_axis_off_denies(override): + assert grant_covers(a_grant(**override), a_request(), TODAY) is False + + +def test_principal_type_is_part_of_identity(): + """A role named the same as a subject is not the same principal.""" + grant = a_grant(principal_type="role", principal_id="authentik-sub-1") + assert grant_covers(grant, a_request(), TODAY) is False + + +def test_a_role_grant_covers_a_caller_holding_that_role(): + grant = a_grant(principal_type="role", principal_id="Editor") + request = a_request(principals=(STUDENT, EDITOR_ROLE)) + assert grant_covers(grant, request, TODAY) is True + + +# ------ scope ---------- + + +def test_global_scope_reaches_everything(): + assert scope_covers("global", None, thing_id=999, group_ids=()) is True + + +def test_group_scope_reaches_a_member_thing(): + assert scope_covers("group", 3, thing_id=7, group_ids=(3, 4)) is True + + +def test_group_scope_stops_at_a_non_member(): + assert scope_covers("group", 3, thing_id=7, group_ids=(4,)) is False + + +def test_thing_scope_needs_a_thing(): + assert scope_covers("thing", 7, thing_id=None, group_ids=()) is False + + +def test_an_unknown_scope_type_denies_rather_than_raising(): + """A row written by newer code must not read as permission to older code.""" + assert scope_covers("watershed", 1, thing_id=7, group_ids=()) is False + + +# ------ time ---------- + + +def test_a_grant_is_dead_before_it_starts(): + assert is_active(date(2026, 9, 1), None, None, TODAY) is False + + +def test_a_grant_is_dead_after_it_ends(): + assert is_active(date(2026, 1, 1), date(2026, 8, 23), None, TODAY) is False + + +def test_a_grant_is_live_on_its_last_day(): + assert is_active(date(2026, 1, 1), TODAY, None, TODAY) is True + + +def test_revocation_beats_the_date_range(): + revoked = datetime(2026, 8, 20, tzinfo=timezone.utc) + assert is_active(date(2026, 1, 1), None, revoked, TODAY) is False + + +def test_expiry_is_checked_at_use_not_swept(): + """The same row answers differently on different days, with no job run.""" + semester = a_grant(starts_at=date(2026, 1, 1), ends_at=date(2026, 5, 15)) + assert grant_covers(semester, a_request(), date(2026, 5, 15)) is True + assert grant_covers(semester, a_request(), date(2026, 5, 16)) is False + + +# ------ validation, before a row is written ---------- + + +def test_a_grant_without_a_data_type_is_rejected(): + """The no-wildcard rule: NULL does not mean 'all'.""" + with pytest.raises(MissingDataType): + validate_grant("user", "read", "global", None, None, TODAY, None) + + +def test_a_global_grant_carries_no_scope_id(): + with pytest.raises(ScopeIdMismatch): + validate_grant("user", "read", "global", 7, "water level", TODAY, None) + + +def test_a_thing_grant_needs_a_scope_id(): + with pytest.raises(ScopeIdMismatch): + validate_grant("user", "read", "thing", None, "water level", TODAY, None) + + +def test_an_unknown_capability_is_rejected(): + with pytest.raises(UnknownCapability): + validate_grant("user", "delete", "global", None, "water level", TODAY, None) + + +def test_an_unknown_principal_type_is_rejected(): + with pytest.raises(UnknownPrincipalType): + validate_grant("robot", "read", "global", None, "water level", TODAY, None) + + +def test_an_unknown_scope_type_is_rejected(): + with pytest.raises(UnknownScopeType): + validate_grant("user", "read", "watershed", 1, "water level", TODAY, None) + + +def test_a_backwards_date_range_is_rejected(): + with pytest.raises(BackwardsDateRange): + validate_grant( + "user", "read", "global", None, "water level", TODAY, date(2026, 1, 1) + ) + + +def test_validation_errors_are_value_errors(): + """The importers and the routes both rely on this (ADR4).""" + with pytest.raises(ValueError): + validate_grant("user", "read", "global", None, None, TODAY, None) + + +# ------ the domain constants and the lexicon must agree ---------- + + +def test_the_domain_vocabularies_match_the_lexicon(): + """domain/access.py cannot import the lexicon without taking a database + dependency, so its constants are hand-copied. They drifted once: the + lexicon says `api key` and the constant said `api_key`, which made every + API-key grant fail validation with a 422. This is the guard.""" + from core.enums import Capability, GrantScopeType, PrincipalType + + assert {member.value for member in PrincipalType} == set(PRINCIPAL_TYPES) + assert {member.value for member in Capability} == set(CAPABILITIES) + assert {member.value for member in GrantScopeType} == set(SCOPE_TYPES) + + +def test_an_api_key_grant_can_actually_be_written(): + validate_grant("api key", "read", "global", None, "water level", TODAY, None) + + +# ------ publication consent ---------- + + +def a_consent(**overrides): + fields = { + "thing_id": 7, + "destination_id": 2, + "data_type": "water level", + "starts_at": date(2026, 1, 1), + "ends_at": None, + "revoked_at": None, + } + fields.update(overrides) + return Consent(**fields) + + +def test_consent_publishes_the_type_it_names(): + assert consent_covers(a_consent(), 7, 2, "water level", TODAY) is True + + +def test_consent_to_levels_is_not_consent_to_chemistry(): + """The case the whole design exists for.""" + assert consent_covers(a_consent(), 7, 2, "water chemistry", TODAY) is False + + +def test_consent_is_per_destination(): + assert consent_covers(a_consent(), 7, 3, "water level", TODAY) is False + + +def test_withdrawn_consent_stops_being_offered_immediately(): + withdrawn = a_consent(revoked_at=datetime(2026, 8, 24, tzinfo=timezone.utc)) + assert any_consent_publishes([withdrawn], 7, 2, "water level", TODAY) is False + + +def test_nothing_published_without_a_consent_row(): + assert any_consent_publishes([], 7, 2, "water level", TODAY) is False diff --git a/tests/test_domain_release.py b/tests/test_domain_release.py new file mode 100644 index 000000000..3165d6dac --- /dev/null +++ b/tests/test_domain_release.py @@ -0,0 +1,129 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""Embargo rules, exercised without a database.""" + +from datetime import date, timedelta + +import pytest + +from domain.release import ( + STATUS_EMBARGOED, + STATUS_PRIVATE, + STATUS_PUBLIC, + MissingReleaseDate, + UnscheduledReleaseDate, + due_for_release, + validate_release, +) + +TODAY = date(2026, 9, 1) +YESTERDAY = TODAY - timedelta(days=1) +TOMORROW = TODAY + timedelta(days=1) + + +# ------ when an embargo runs out ---------- + + +def test_an_embargo_lifts_on_its_release_date(): + """release_at is the day the record becomes public, not the last day it + is held. Off by one here publishes a day late, every time.""" + assert due_for_release(STATUS_EMBARGOED, TODAY, TODAY) is True + + +def test_an_embargo_is_still_holding_the_day_before(): + assert due_for_release(STATUS_EMBARGOED, TOMORROW, TODAY) is False + + +def test_a_missed_run_still_releases_afterwards(): + """The job does not have to run on the day. A date that passed while + nothing ran is still due the next time it does.""" + assert due_for_release(STATUS_EMBARGOED, YESTERDAY, TODAY) is True + + +def test_expiry_is_checked_at_use_not_swept(): + """The same row answers differently on different days, with no job run.""" + assert due_for_release(STATUS_EMBARGOED, TODAY, YESTERDAY) is False + assert due_for_release(STATUS_EMBARGOED, TODAY, TODAY) is True + + +# ------ what the release job refuses to touch ---------- + + +@pytest.mark.parametrize( + "status", [STATUS_PUBLIC, STATUS_PRIVATE, "draft", "archived", None] +) +def test_only_embargoed_records_are_released(status): + """A past date on any other level schedules nothing. Publishing a draft + because it happens to carry a date would be the job inventing a decision + nobody made.""" + assert due_for_release(status, YESTERDAY, TODAY) is False + + +def test_an_embargo_without_a_date_is_never_due(): + """validate_release makes this unwritable, but a row predating this + feature, or written by hand, must not crash the nightly job.""" + assert due_for_release(STATUS_EMBARGOED, None, TODAY) is False + + +# ------ validation, before a row is written ---------- + + +def test_an_embargo_names_its_release_date(): + with pytest.raises(MissingReleaseDate): + validate_release(STATUS_EMBARGOED, None) + + +def test_a_release_date_on_another_level_is_rejected(): + """It would schedule nothing, and read as a promise that something was.""" + with pytest.raises(UnscheduledReleaseDate): + validate_release("draft", TOMORROW) + + +def test_a_public_record_carries_no_release_date(): + with pytest.raises(UnscheduledReleaseDate): + validate_release(STATUS_PUBLIC, TOMORROW) + + +def test_an_embargo_with_a_date_is_valid(): + assert validate_release(STATUS_EMBARGOED, TOMORROW) is None + + +def test_a_record_with_neither_is_valid(): + assert validate_release("draft", None) is None + + +def test_a_release_date_in_the_past_is_accepted(): + """Backfilling an embargo that already expired is a legitimate thing to + do; the next run publishes it. Rejecting it would push the caller into + setting a fake future date.""" + assert validate_release(STATUS_EMBARGOED, YESTERDAY) is None + + +# ------ the domain constants and the lexicon must agree ---------- + + +def test_the_release_levels_exist_in_the_lexicon(): + """domain/release.py cannot import the lexicon without taking a database + dependency, so its constants are hand-copied -- the same drift that cost + domain/access.py every API-key grant. A rename in core/lexicon.json that + stranded the release job would otherwise be silent: nothing would ever be + due, and nothing would raise.""" + from core.enums import ReleaseStatus + + levels = {member.value for member in ReleaseStatus} + assert STATUS_EMBARGOED in levels + assert STATUS_PUBLIC in levels + assert STATUS_PRIVATE in levels diff --git a/tests/test_field_projection.py b/tests/test_field_projection.py new file mode 100644 index 000000000..1f9514c75 --- /dev/null +++ b/tests/test_field_projection.py @@ -0,0 +1,189 @@ +# =============================================================================== +# Copyright 2025 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""Field projection (ADR5, 3.5). + +The rule tests need no database. The loader tests import the models to know +what fields exist, but never connect. +""" + +import pytest + +from domain.field_projection import ( + EntityProjection, + NeverPublicFieldAllowed, + UnknownField, + UnknownTransform, + project, + round_to, + validate_projection, +) +from services.field_projection import ( + LOCATION, + THING, + _build_projection, + _configuration, + known_fields, + never_public_fields, + projection_for, +) + +RECORD = { + "id": 1, + "name": "Test Well", + "well_depth": 120.0, + "created_by_id": "authentik-sub-1", + "nma_pk_welldata": 4321, +} + + +class FakeDestination: + def __init__(self, slug, destination_kind): + self.slug = slug + self.destination_kind = destination_kind + + +# ------ the allowlist ---------- + + +def test_only_listed_fields_survive(): + projection = EntityProjection(fields=frozenset({"id", "name"})) + assert project(RECORD, projection) == {"id": 1, "name": "Test Well"} + + +def test_a_field_nobody_listed_is_absent_not_null(): + """Omission produces silence: the key is gone, not set to None.""" + projection = EntityProjection(fields=frozenset({"id"})) + assert "well_depth" not in project(RECORD, projection) + + +def test_no_projection_publishes_nothing(): + """Default deny. An audience with no rule receives an empty record.""" + assert project(RECORD, None) == {} + + +def test_a_new_field_is_invisible_until_someone_lists_it(): + projection = EntityProjection(fields=frozenset({"id", "name"})) + with_new_column = dict(RECORD, gate_code="1234") + assert "gate_code" not in project(with_new_column, projection) + + +# ------ never-public ---------- + + +def test_never_public_beats_an_allowlist_that_asked_for_it(): + """Belt and braces: enforced at load, and again at projection time.""" + projection = EntityProjection(fields=frozenset({"id", "created_by_id"})) + projected = project(RECORD, projection, never_public=frozenset({"created_by_id"})) + assert projected == {"id": 1} + + +def test_configuring_a_never_public_field_raises_at_load(): + with pytest.raises(NeverPublicFieldAllowed): + validate_projection( + entity=THING, + fields=["id", "created_by_id"], + transforms={}, + known_fields=known_fields(THING), + never_public=frozenset({"created_by_id"}), + ) + + +def test_provenance_columns_are_never_public(): + assert "created_by_id" in never_public_fields(THING) + assert "updated_by_name" in never_public_fields(THING) + + +def test_staff_written_location_notes_are_never_public(): + """Gate codes and candid landowner notes have landed in these columns.""" + assert "nma_location_notes" in never_public_fields(LOCATION) + assert "nma_coordinate_notes" in never_public_fields(LOCATION) + + +# ------ transformation ---------- + + +def test_a_coordinate_can_be_rounded_rather_than_dropped(): + projection = EntityProjection( + fields=frozenset({"latitude"}), transforms={"latitude": ("round", 2)} + ) + assert project({"latitude": 33.809712}, projection) == {"latitude": 33.81} + + +def test_rounding_leaves_a_missing_coordinate_missing(): + assert round_to(None, 2) is None + + +def test_an_untransformed_field_passes_through(): + projection = EntityProjection( + fields=frozenset({"latitude", "elevation"}), + transforms={"latitude": ("round", 2)}, + ) + projected = project({"latitude": 33.809712, "elevation": 2464.9}, projection) + assert projected["elevation"] == 2464.9 + + +# ------ configuration validation ---------- + + +def test_an_unknown_field_raises(): + """A typo in an allowlist silently withholds data, so it fails loudly.""" + with pytest.raises(UnknownField): + _build_projection(THING, ["id", "welll_depth"], {}) + + +def test_an_unknown_entity_raises(): + with pytest.raises(KeyError): + _build_projection("borehole", ["id"], {}) + + +def test_an_unknown_transform_raises(): + with pytest.raises(UnknownTransform): + _build_projection( + LOCATION, + {"fields": ["latitude"], "transforms": {"latitude": {"fuzz": 2}}}, + {}, + ) + + +def test_a_transform_on_an_unpublished_field_raises(): + with pytest.raises(UnknownField): + _build_projection( + LOCATION, {"fields": ["id"], "transforms": {"latitude": {"round": 2}}}, {} + ) + + +def test_the_shipped_configuration_is_valid(): + """Loading validates every audience; this asserts it stays that way.""" + assert _configuration()["audiences"] + + +# ------ audience lookup ---------- + + +def test_a_destination_inherits_the_rules_for_its_kind(): + projection = projection_for(FakeDestination("ngwmn", "harvester"), THING) + assert "name" in projection.fields + + +def test_an_unregistered_kind_receives_nothing(): + assert projection_for(FakeDestination("mystery", "carrier pigeon"), THING) is None + + +def test_the_public_web_gets_a_coarser_coordinate_than_a_harvester(): + public = projection_for(FakeDestination("web", "public web"), LOCATION) + harvester = projection_for(FakeDestination("ngwmn", "harvester"), LOCATION) + assert public.transforms["latitude"] == ("round", 2) + assert harvester.transforms["latitude"] == ("round", 4) diff --git a/tests/test_ogc_embargo.py b/tests/test_ogc_embargo.py new file mode 100644 index 000000000..62d63c192 --- /dev/null +++ b/tests/test_ogc_embargo.py @@ -0,0 +1,242 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""An embargoed record must not reach the public OGC relations. + +The whole feature is one claim -- "this measurement is not published until +that date" -- and this is where it is checked against the relations a +consumer actually reads, rather than against the rule in isolation. + +The four relations here are the public ones that read the observation chain +(migration b4c5d6e7f8a9). The chemistry collections are absent because they +read the legacy NMA_* tables, which carry no release columns at all; see +docs/data-embargo.md. +""" + +from datetime import date, timedelta + +import pytest +from sqlalchemy import delete, text + +from db.engine import session_ctx +from db.field import FieldActivity, FieldEvent +from db.location import Location, LocationThingAssociation +from db.observation import Observation +from db.sample import Sample +from db.thing import Thing +from services.release_schedule import lift_due_embargoes +from tests import get_parameter_id + +POINT_ID = "EMBARGO-TEST-1" +TOMORROW = date.today() + timedelta(days=1) + +# Every public relation that reads the observation chain, with the matviews +# flagged: those answer from their last refresh, so a test that changes data +# has to refresh them the way the nightly pg_cron job does. +WATER_LEVEL_RELATIONS = ( + ("ogc_water_well_summary", True), + ("ogc_water_elevation_wells", True), + ("ogc_depth_to_water_trend_wells", True), + ("ogc_latest_depth_to_water_wells", True), +) + + +def _refresh(session): + for relation, materialized in WATER_LEVEL_RELATIONS: + if materialized: + session.execute(text(f"REFRESH MATERIALIZED VIEW {relation}")) + session.commit() + + +def _relations_naming(session, thing_id) -> set: + """Which of the public relations currently publish this well.""" + present = set() + for relation, _materialized in WATER_LEVEL_RELATIONS: + found = session.execute( + text(f"SELECT 1 FROM {relation} WHERE id = :id LIMIT 1"), + {"id": thing_id}, + ).scalar() + if found: + present.add(relation) + return present + + +@pytest.fixture() +def well_with_one_water_level(): + """A public well whose only measurement is public, and its ids.""" + with session_ctx() as session: + location = Location( + point="POINT(-106.5 34.5)", + elevation=1600.0, + release_status="public", + ) + session.add(location) + session.flush() + + thing = Thing( + name=POINT_ID, + thing_type="water well", + release_status="public", + well_depth=100.0, + ) + session.add(thing) + session.flush() + + session.add( + LocationThingAssociation( + location_id=location.id, + thing_id=thing.id, + effective_start="2025-02-01T00:00:00Z", + ) + ) + + event = FieldEvent( + thing_id=thing.id, + event_date="2024-03-15T19:00:00Z", + release_status="public", + ) + session.add(event) + session.flush() + + activity = FieldActivity( + field_event_id=event.id, + activity_type="groundwater level", + release_status="public", + ) + session.add(activity) + session.flush() + + sample = Sample( + field_activity_id=activity.id, + sample_date="2024-03-15T19:00:00Z", + sample_name=f"{POINT_ID}-wl-1", + sample_matrix="water", + sample_method="Steel-tape measurement", + qc_type="Normal", + release_status="public", + ) + session.add(sample) + session.flush() + + observation = Observation( + sample_id=sample.id, + parameter_id=get_parameter_id("groundwater level", "Field Parameter"), + observation_datetime="2024-03-15T19:00:00Z", + value=50.0, + unit="ft", + measuring_point_height=2.5, + release_status="public", + ) + session.add(observation) + session.commit() + + ids = { + "thing": thing.id, + "location": location.id, + "observation": observation.id, + "sample": sample.id, + "activity": activity.id, + "event": event.id, + } + _refresh(session) + + yield ids + + session.execute(delete(Thing).where(Thing.id == ids["thing"])) + session.execute(delete(Location).where(Location.id == ids["location"])) + session.commit() + _refresh(session) + + +def test_a_public_measurement_reaches_every_water_level_relation( + well_with_one_water_level, +): + """The control. Without this, a test asserting absence proves nothing -- + the well could be missing for any of a dozen unrelated reasons.""" + with session_ctx() as session: + present = _relations_naming(session, well_with_one_water_level["thing"]) + assert present == {relation for relation, _ in WATER_LEVEL_RELATIONS} + + +@pytest.mark.parametrize( + "level, model", + [ + ("observation", Observation), + ("sample", Sample), + ("activity", FieldActivity), + ("event", FieldEvent), + ], +) +def test_an_embargo_anywhere_in_the_chain_withholds_the_measurement( + well_with_one_water_level, level, model +): + """Embargoing any link hides the measurement. A consumer cannot reach an + embargoed reading by way of a public parent.""" + with session_ctx() as session: + row = session.get(model, well_with_one_water_level[level]) + row.release_status = "embargoed" + row.release_at = TOMORROW + session.commit() + _refresh(session) + + assert _relations_naming(session, well_with_one_water_level["thing"]) == set() + + +def test_an_embargoed_well_is_absent_from_every_relation(well_with_one_water_level): + """Whole-site embargo needs no clause of its own: `release_status = + 'public'` already excludes it.""" + with session_ctx() as session: + thing = session.get(Thing, well_with_one_water_level["thing"]) + thing.release_status = "embargoed" + thing.release_at = TOMORROW + session.commit() + _refresh(session) + + assert _relations_naming(session, well_with_one_water_level["thing"]) == set() + + +def test_a_record_returns_when_its_embargo_is_lifted(well_with_one_water_level): + """End to end: embargo, hidden; the date arrives, the job runs, the + refresh follows, and the measurement is published.""" + with session_ctx() as session: + observation = session.get(Observation, well_with_one_water_level["observation"]) + observation.release_status = "embargoed" + observation.release_at = date.today() + session.commit() + _refresh(session) + assert _relations_naming(session, well_with_one_water_level["thing"]) == set() + + lift_due_embargoes(session) + _refresh(session) + + assert _relations_naming(session, well_with_one_water_level["thing"]) == { + relation for relation, _ in WATER_LEVEL_RELATIONS + } + + +def test_a_draft_measurement_is_still_published(well_with_one_water_level): + """The migration adds an embargo clause, not a release-policy change. + Observations sitting at `draft` reached these four relations before it and + still do -- tightening that is a separate decision with its own row + counts (see migration b4c5d6e7f8a9).""" + with session_ctx() as session: + observation = session.get(Observation, well_with_one_water_level["observation"]) + observation.release_status = "draft" + session.commit() + _refresh(session) + + assert _relations_naming(session, well_with_one_water_level["thing"]) == { + relation for relation, _ in WATER_LEVEL_RELATIONS + } diff --git a/tests/test_ogc_projection.py b/tests/test_ogc_projection.py new file mode 100644 index 000000000..42ed04289 --- /dev/null +++ b/tests/test_ogc_projection.py @@ -0,0 +1,177 @@ +# =============================================================================== +# Copyright 2025 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""The public OGC collections, under the projection chokepoint (ADR5, A.2). + +Two of these pin leaks that were live before the allowlist existed: +ogc_well_water_column published nma_pk_welldata, and +ogc_temp_depth_measurements published entered_by, a staff member's name. +""" + +import re +from pathlib import Path + +import pytest +from sqlalchemy import text + +from core.pygeoapi import THING_COLLECTIONS +from db.engine import session_ctx +from tests import client +from services.field_projection import ( + ogc_allowlist, + ogc_collection_tables, + ogc_never_public, +) + +PUBLIC_CONFIG = Path(__file__).parent.parent / "core" / "pygeoapi-config.yml" +# The EDR collections are wired in core/pygeoapi.py rather than the YAML. +EDR_TABLES = ("ogc_waterlevels", "ogc_water_chemistry") + +COLUMNS_QUERY = text(""" + select a.attname + from pg_attribute a + join pg_class c on c.oid = a.attrelid + join pg_namespace n on n.oid = c.relnamespace + where n.nspname = 'public' + and c.relname = :relation + and a.attnum > 0 + and not a.attisdropped + """) + + +def public_tables() -> list: + """Every table the public mount serves. + + Three sources, and missing the third is how eleven collections published + nma_pk_welldata unnoticed: the YAML config, the thing collections built in + core/pygeoapi.py, and the EDR collections wired there too. + """ + configured = set(re.findall(r"table: (ogc_[a-z_]+)", PUBLIC_CONFIG.read_text())) + things = { + "ogc_" + collection["id"] + for collection in THING_COLLECTIONS + if not collection.get("internal_only") + } + return sorted(configured | things | set(EDR_TABLES)) + + +def live_columns(relation: str) -> set: + with session_ctx() as session: + return set(session.execute(COLUMNS_QUERY, {"relation": relation}).scalars()) + + +# ------ every public collection is projected ---------- + + +@pytest.mark.parametrize("table", public_tables()) +def test_every_public_collection_has_an_allowlist(table): + """A collection with no entry publishes nothing. Fail here, not in prod.""" + assert table in ogc_collection_tables(), ( + f"{table} is served publicly but has no entry in " + "core/field-allowlists.yml, so it would publish no properties." + ) + + +@pytest.mark.parametrize("table", public_tables()) +def test_an_allowlist_never_names_a_never_public_column(table): + assert not ogc_allowlist(table) & ogc_never_public() + + +@pytest.mark.parametrize("table", public_tables()) +def test_an_allowlist_only_names_columns_the_view_has(table): + """A stale name would make pygeoapi select a column that is not there.""" + assert ogc_allowlist(table) <= live_columns(table) + + +@pytest.mark.parametrize("table", public_tables()) +def test_no_never_public_column_survives_into_a_published_collection(table): + """The guard that matters: whatever the view grew, it is not published.""" + published = ogc_allowlist(table) + assert not (live_columns(table) & ogc_never_public() & published) + + +# ------ the leaks this closed ---------- + + +@pytest.mark.parametrize( + "table", + [ + "ogc_well_water_column", + "ogc_water_wells", + "ogc_springs", + "ogc_meteorological_stations", + ], +) +def test_the_thing_layers_no_longer_publish_the_legacy_key(table): + """Eleven public collections carried nma_pk_welldata before this.""" + assert "nma_pk_welldata" in live_columns(table) + assert "nma_pk_welldata" not in ogc_allowlist(table) + + +def test_the_geothermal_layer_no_longer_publishes_who_typed_the_record(): + assert "entered_by" in live_columns("ogc_temp_depth_measurements") + assert "entered_by" not in ogc_allowlist("ogc_temp_depth_measurements") + + +# ------ what is and is not projected ---------- + + +def test_the_internal_mount_is_not_projected(): + """Authenticated staff see the whole view; per-role rules are not built.""" + assert ogc_allowlist("ogc_internal_locations") is None + + +def test_an_unlisted_public_collection_publishes_nothing(): + """Default deny, rather than falling back to everything.""" + assert ogc_allowlist("ogc_collection_nobody_configured") == frozenset() + + +# ------ through the mounted service ---------- + + +def _json(path): + response = client.get(path, headers={"Accept": "application/json"}) + assert response.status_code == 200, response.text + return response.text + + +@pytest.mark.parametrize( + "collection,hidden,published", + [ + ("well_water_column", "nma_pk_welldata", "well_depth"), + ("temp_depth_measurements", "entered_by", "depth"), + ], +) +def test_the_mount_publishes_the_allowlist_and_nothing_else( + collection, hidden, published +): + """Not just absent from features: absent from the schema and the + queryables too, so a filter cannot probe for it either.""" + for path in ( + f"/ogcapi/collections/{collection}/schema", + f"/ogcapi/collections/{collection}/queryables", + f"/ogcapi/collections/{collection}/items?limit=1", + ): + body = _json(path) + assert hidden not in body, f"{hidden} leaked through {path}" + + # The collection still works; this is projection, not breakage. + assert published in _json(f"/ogcapi/collections/{collection}/schema") + + +def test_release_status_is_still_published(): + """The filter column stays visible; consumers read it to know what they + have, and hiding it would be a behavior change nobody asked for.""" + assert "release_status" in ogc_allowlist("ogc_waterlevels") diff --git a/tests/test_release_schedule.py b/tests/test_release_schedule.py new file mode 100644 index 000000000..fdaed834f --- /dev/null +++ b/tests/test_release_schedule.py @@ -0,0 +1,230 @@ +# =============================================================================== +# Copyright 2026 ross +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# =============================================================================== +"""The scheduled release of embargoed records.""" + +from datetime import date, timedelta + +import pytest +from sqlalchemy import delete, select +from sqlalchemy.exc import DBAPIError + +from db.authorization_audit import RELEASE_LIFTED, AuthorizationAudit +from db.engine import session_ctx +from db.location import Location +from services.release_schedule import RELEASE_ACTOR, lift_due_embargoes + +TODAY = date.today() +YESTERDAY = TODAY - timedelta(days=1) +TOMORROW = TODAY + timedelta(days=1) + + +@pytest.fixture() +def embargoed_locations(): + """Three locations: due yesterday, due today, still held.""" + created = {} + with session_ctx() as session: + for label, release_at in ( + ("overdue", YESTERDAY), + ("due", TODAY), + ("held", TOMORROW), + ): + location = Location( + point="POINT(-106.0 34.0)", + elevation=1500.0, + release_status="embargoed", + release_at=release_at, + ) + session.add(location) + session.commit() + session.refresh(location) + created[label] = location.id + + yield created + + session.execute( + delete(AuthorizationAudit).where( + AuthorizationAudit.subject_table == Location.__tablename__, + AuthorizationAudit.subject_id.in_(created.values()), + ) + ) + session.execute(delete(Location).where(Location.id.in_(created.values()))) + session.commit() + + +def _status(session, location_id): + location = session.get(Location, location_id) + return location.release_status, location.release_at + + +def test_a_record_is_released_on_its_date(embargoed_locations): + with session_ctx() as session: + lift_due_embargoes(session) + assert _status(session, embargoed_locations["due"]) == ("public", None) + + +def test_a_record_whose_date_passed_unnoticed_is_still_released(embargoed_locations): + """A day the job did not run does not strand the record.""" + with session_ctx() as session: + lift_due_embargoes(session) + assert _status(session, embargoed_locations["overdue"]) == ("public", None) + + +def test_a_record_still_under_embargo_is_left_alone(embargoed_locations): + with session_ctx() as session: + lift_due_embargoes(session) + assert _status(session, embargoed_locations["held"]) == ( + "embargoed", + TOMORROW, + ) + + +def test_the_release_date_is_cleared_on_release(embargoed_locations): + """A public row carrying a release_at would advertise a schedule it has + already kept, and would fail validate_release if it were re-checked.""" + with session_ctx() as session: + lift_due_embargoes(session) + _, release_at = _status(session, embargoed_locations["due"]) + assert release_at is None + + +def test_a_preview_writes_nothing(embargoed_locations): + with session_ctx() as session: + plan = lift_due_embargoes(session, apply=False) + assert len(plan.lifted) == 2 + assert _status(session, embargoed_locations["due"]) == ("embargoed", TODAY) + assert _status(session, embargoed_locations["overdue"]) == ( + "embargoed", + YESTERDAY, + ) + + +def test_running_twice_releases_nothing_the_second_time(embargoed_locations): + with session_ctx() as session: + first = lift_due_embargoes(session) + second = lift_due_embargoes(session) + assert len(first.lifted) == 2 + assert second.lifted == [] + + +def test_every_release_lands_in_the_authorization_log(embargoed_locations): + """The question after an incident is who published this, and when.""" + with session_ctx() as session: + lift_due_embargoes(session) + rows = ( + session.execute( + select(AuthorizationAudit).where( + AuthorizationAudit.subject_table == Location.__tablename__, + AuthorizationAudit.subject_id.in_(embargoed_locations.values()), + ) + ) + .scalars() + .all() + ) + + assert len(rows) == 2 + assert {row.event_type for row in rows} == {RELEASE_LIFTED} + assert {row.actor for row in rows} == {RELEASE_ACTOR} + + by_subject = {row.subject_id: row.detail for row in rows} + # The date the embargo was set for is recorded next to the date it was + # actually lifted, so "was this released early" stays answerable. + assert by_subject[embargoed_locations["due"]]["release_at"] == TODAY.isoformat() + assert ( + by_subject[embargoed_locations["overdue"]]["release_at"] + == YESTERDAY.isoformat() + ) + assert by_subject[embargoed_locations["due"]]["released_on"] == TODAY.isoformat() + + +def test_an_earlier_run_date_releases_less(embargoed_locations): + """on_date is a parameter so the job can be reasoned about, and so a + backfill can be replayed as of a past date.""" + with session_ctx() as session: + plan = lift_due_embargoes(session, on_date=YESTERDAY, apply=False) + assert len(plan.lifted) == 1 + assert plan.lifted[0][1] == embargoed_locations["overdue"] + + +def test_a_record_at_another_level_is_left_where_it_is(): + """Only 'embargoed' is scheduled. The job walks past everything else -- + a private record is not published because the job happened to look at it. + + It carries no release_at: the CHECK constraint below makes that pairing + unwritable, which is a stronger guarantee than this test could assert. + """ + with session_ctx() as session: + location = Location( + point="POINT(-106.1 34.1)", + elevation=1500.0, + release_status="private", + ) + session.add(location) + session.commit() + session.refresh(location) + location_id = location.id + + try: + lift_due_embargoes(session) + assert _status(session, location_id) == ("private", None) + finally: + session.execute(delete(Location).where(Location.id == location_id)) + session.commit() + + +# ------ the table refuses what the schema layer cannot see ---------- + + +def test_the_table_rejects_an_embargo_with_no_date(): + """A PATCH body carrying only release_status is a fragment, so the pair is + guarded by a CHECK constraint rather than by pydantic. This is that + constraint, reached the way the CLI and the transfers reach it.""" + with session_ctx() as session: + session.add( + Location( + point="POINT(-106.2 34.2)", + elevation=1500.0, + release_status="embargoed", + release_at=None, + ) + ) + # pg8000 maps a CHECK violation (SQLSTATE 23514) to ProgrammingError + # rather than IntegrityError, so match on the parent and the name. + with pytest.raises(DBAPIError, match="location_embargo_needs_date"): + session.commit() + session.rollback() + + +def test_the_table_rejects_a_release_date_without_an_embargo(): + with session_ctx() as session: + session.add( + Location( + point="POINT(-106.3 34.3)", + elevation=1500.0, + release_status="draft", + release_at=TOMORROW, + ) + ) + with pytest.raises(DBAPIError, match="location_embargo_needs_date"): + session.commit() + session.rollback() + + +def test_the_released_row_satisfies_the_constraint(embargoed_locations): + """The job clears release_at as it publishes; if it did not, its own + UPDATE would violate the constraint it just satisfied.""" + with session_ctx() as session: + lift_due_embargoes(session) + assert _status(session, embargoed_locations["due"]) == ("public", None) diff --git a/tests/test_thing.py b/tests/test_thing.py index 8c8859c44..025cc0b07 100644 --- a/tests/test_thing.py +++ b/tests/test_thing.py @@ -927,6 +927,8 @@ def test_get_water_wells_includes_contact_summary( "id": contact.id, "created_at": contact.created_at.astimezone(timezone.utc).strftime(DT_FMT), "release_status": contact.release_status, + "data_maturity": contact.data_maturity, + "release_at": contact.release_at, "name": contact.name, "organization": contact.organization, "contact_type": contact.contact_type, diff --git a/transfers/metrics.py b/transfers/metrics.py index 456e9b484..0d306d73f 100644 --- a/transfers/metrics.py +++ b/transfers/metrics.py @@ -34,7 +34,7 @@ TransducerObservation, Group, Asset, - PermissionHistory, + FieldAccessConsent, ThingGeologicFormationAssociation, NMA_Stratigraphy, NMA_FieldParameters, @@ -171,7 +171,7 @@ def weather_data_metrics(self, *args, **kw) -> None: self._handle_metrics(NMA_WeatherData, name="WeatherData", *args, **kw) def permissions_metrics(self, *args, **kw) -> None: - self._handle_metrics(PermissionHistory, *args, **kw) + self._handle_metrics(FieldAccessConsent, *args, **kw) def stratigraphy_metrics(self, *args, **kw) -> None: self._handle_metrics(ThingGeologicFormationAssociation, *args, **kw) diff --git a/transfers/permissions_transfer.py b/transfers/permissions_transfer.py index 346e9f147..6da629b66 100644 --- a/transfers/permissions_transfer.py +++ b/transfers/permissions_transfer.py @@ -4,7 +4,7 @@ from pandas import isna from sqlalchemy.orm import Session -from db import Thing, PermissionHistory, Contact, ThingContactAssociation +from db import Thing, FieldAccessConsent, Contact, ThingContactAssociation from transfers.util import read_csv, logger, replace_nans, chunk_by_size """ @@ -18,7 +18,7 @@ def _make_permission( wdf, well, contact_id, nma_field, permission_type -) -> PermissionHistory | None: +) -> FieldAccessConsent | None: values = wdf.loc[wdf["PointID"] == well.name, nma_field].values if len(values) == 0: @@ -27,7 +27,7 @@ def _make_permission( return None permission_allowed = bool(values[0]) - permission = PermissionHistory( + permission = FieldAccessConsent( contact_id=contact_id, permission_type=permission_type, permission_allowed=permission_allowed, @@ -48,7 +48,7 @@ def transfer_permissions(session: Session) -> None: The transferred wells and contacts need to be transferred first - to access the auto-generated well IDs - to know who gave permission to which well since contact_id is required for - PermissionHistory + FieldAccessConsent """ wdf = read_csv("WellData", dtype={"OSEWelltagID": str}) wdf = replace_nans(wdf) diff --git a/transfers/transfer_results_builder.py b/transfers/transfer_results_builder.py index 42e7c49b2..edb41a818 100644 --- a/transfers/transfer_results_builder.py +++ b/transfers/transfer_results_builder.py @@ -7,7 +7,7 @@ import pandas as pd from sqlalchemy import select, func -from db import Deployment, PermissionHistory, Sensor, Thing, ThingContactAssociation +from db import Deployment, FieldAccessConsent, Sensor, Thing, ThingContactAssociation from db.engine import session_ctx from transfers.sensor_transfer import ( EQUIPMENT_TO_SENSOR_TYPE_MAP, @@ -210,14 +210,14 @@ def _permissions_destination_series(session) -> pd.Series: sql = ( select( Thing.name.label("point_id"), - PermissionHistory.permission_type.label("permission_type"), - PermissionHistory.permission_allowed.label("permission_allowed"), + FieldAccessConsent.permission_type.label("permission_type"), + FieldAccessConsent.permission_allowed.label("permission_allowed"), ) - .select_from(PermissionHistory) - .join(Thing, Thing.id == PermissionHistory.target_id) - .where(PermissionHistory.target_table == "thing") + .select_from(FieldAccessConsent) + .join(Thing, Thing.id == FieldAccessConsent.target_id) + .where(FieldAccessConsent.target_table == "thing") .where( - PermissionHistory.permission_type.in_( + FieldAccessConsent.permission_type.in_( ("Water Chemistry Sample", "Water Level Sample") ) ) @@ -370,10 +370,10 @@ def _build_permissions(self, spec: TransferComparisonSpec) -> TransferResult: destination_row_count = int( session.execute( select(func.count()) - .select_from(PermissionHistory) - .where(PermissionHistory.target_table == "thing") + .select_from(FieldAccessConsent) + .where(FieldAccessConsent.target_table == "thing") .where( - PermissionHistory.permission_type.in_( + FieldAccessConsent.permission_type.in_( ("Water Chemistry Sample", "Water Level Sample") ) ) @@ -402,7 +402,7 @@ def _build_permissions(self, spec: TransferComparisonSpec) -> TransferResult: transfer_name=spec.transfer_name, source_csv=spec.source_csv, source_key_column=spec.source_key_column, - destination_model="PermissionHistory", + destination_model="FieldAccessConsent", destination_key_column=spec.destination_key_column, source_row_count=source_row_count, agreed_transfer_row_count=agreed_transfer_row_count, diff --git a/transfers/transfer_results_specs.py b/transfers/transfer_results_specs.py index 5a23f40bc..2a0071e5e 100644 --- a/transfers/transfer_results_specs.py +++ b/transfers/transfer_results_specs.py @@ -28,7 +28,7 @@ NMA_view_NGWMN_WaterLevels, NMA_view_NGWMN_WellConstruction, Observation, - PermissionHistory, + FieldAccessConsent, Sensor, Thing, WellScreen, @@ -523,7 +523,7 @@ def _record_new_contact( PermissionsTransferResult, "WellData", "PointID|PermissionType|PermissionAllowed", - PermissionHistory, + FieldAccessConsent, "thing.name|permission_type|permission_allowed", option_field="transfer_permissions", ),