From 4c3ba71634b467c3aecee44b43efa0a94d95d484 Mon Sep 17 00:00:00 2001 From: jakeross Date: Sat, 22 Aug 2026 07:52:13 -0700 Subject: [PATCH] chore(ogc): hide four layers from the public catalog BDMS-977 (A16), BDMS-978 (A17) and BDMS-979 (A18) all take a layer off the public /ogcapi catalog without touching its data: * avg_tds_wells averages 1.9 observations per well, so the statistic is unreliable for half the catalog, and latest_depth_to_water_wells repeats what water_well_summary already publishes. * locations exposes 11,174 features, 11,173 of which duplicate coverage the 11 thing-type layers already give, at the cost of being one of the two largest layers served. * other_things is internal vocabulary: "Thing" is the data-model name for a monitoring point and "other" names no feature class. Five features. The first three are config entries, so they come out of core/pygeoapi-config.yml and stay in pygeoapi-config-internal.yml. other_things is generated from THING_COLLECTIONS, which feeds both mounts, so it gets an internal_only marker that _thing_collections_block honors and only the internal _write_config call opts into. All backing relations are retained. /ogcapi-internal still publishes all four to staff GIS clients, which is the internal use A18 asked us to check for before dropping ogc_other_things, so no migration here. The A1 and A13 scenario tables lose the rows a public client can no longer request; the SQL-level A1 scenarios still cover those views' release_status filter. A16/A17/A18 scenarios get step definitions and the @production tag so CI enforces them. Co-Authored-By: Claude Opus 5 --- README.md | 11 ++-- core/pygeoapi-config.yml | 69 -------------------- core/pygeoapi.py | 11 ++++ tests/features/ogc-cleanup-sprint1.feature | 54 +++++++--------- tests/features/steps/ogc-cleanup-sprint1.py | 72 +++++++++++++++++++++ tests/test_ogc.py | 24 +++---- tests/test_pygeoapi_mount.py | 25 +++++++ 7 files changed, 150 insertions(+), 116 deletions(-) diff --git a/README.md b/README.md index 47a178e95..5b5df58b8 100644 --- a/README.md +++ b/README.md @@ -39,22 +39,21 @@ hits to `/_ah/warmup`. curl http://localhost:8000/ogcapi curl http://localhost:8000/ogcapi/conformance curl http://localhost:8000/ogcapi/collections -curl http://localhost:8000/ogcapi/collections/locations +curl http://localhost:8000/ogcapi/collections/water_wells ``` ### Items (GeoJSON) ```bash -curl "http://localhost:8000/ogcapi/collections/locations/items?limit=10&offset=0" -curl "http://localhost:8000/ogcapi/collections/water_wells/items?limit=5" +curl "http://localhost:8000/ogcapi/collections/water_wells/items?limit=10&offset=0" curl "http://localhost:8000/ogcapi/collections/springs/items?limit=5" -curl "http://localhost:8000/ogcapi/collections/locations/items/123" +curl "http://localhost:8000/ogcapi/collections/water_wells/items/123" ``` ### BBOX + datetime filters ```bash -curl "http://localhost:8000/ogcapi/collections/locations/items?bbox=-107.9,33.8,-107.8,33.9" +curl "http://localhost:8000/ogcapi/collections/water_wells/items?bbox=-107.9,33.8,-107.8,33.9" curl "http://localhost:8000/ogcapi/collections/water_wells/items?datetime=2020-01-01/2024-01-01" ``` @@ -63,7 +62,7 @@ curl "http://localhost:8000/ogcapi/collections/water_wells/items?datetime=2020-0 Use `filter` + `filter-lang=cql2-text` with `WITHIN(...)`: ```bash -curl "http://localhost:8000/ogcapi/collections/locations/items?filter=WITHIN(geometry,POLYGON((-107.9 33.8,-107.8 33.8,-107.8 33.9,-107.9 33.9,-107.9 33.8)))&filter-lang=cql2-text" +curl "http://localhost:8000/ogcapi/collections/water_wells/items?filter=WITHIN(geometry,POLYGON((-107.9 33.8,-107.8 33.8,-107.8 33.9,-107.9 33.9,-107.9 33.8)))&filter-lang=cql2-text" ``` ### OpenAPI UI diff --git a/core/pygeoapi-config.yml b/core/pygeoapi-config.yml index fb6fc6c2c..c82fcea2a 100644 --- a/core/pygeoapi-config.yml +++ b/core/pygeoapi-config.yml @@ -49,75 +49,6 @@ metadata: # publishes "pointOfContact" as the service's hours of operation. resources: - locations: - type: collection - title: Locations - description: Geographic locations and site coordinates used by Ocotillo features. - keywords: [locations] - extents: - spatial: - bbox: [-109.05, 31.33, -103.00, 37.00] - crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 - providers: - - type: feature - name: PostgreSQL - data: - host: {postgres_host} - port: {postgres_port} - dbname: {postgres_db} - user: {postgres_user} - password: {postgres_password_env} - search_path: [public] - id_field: id - table: ogc_locations - geom_field: point - - latest_depth_to_water_wells: - type: collection - title: Latest Depth to Water (Water Wells) - description: Most recent depth-to-water below ground surface observation for each water well. - keywords: [water-wells, groundwater-level, depth-to-water-bgs, latest] - extents: - spatial: - bbox: [-109.05, 31.33, -103.00, 37.00] - crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 - providers: - - type: feature - name: PostgreSQL - data: - host: {postgres_host} - port: {postgres_port} - dbname: {postgres_db} - user: {postgres_user} - password: {postgres_password_env} - search_path: [public] - id_field: id - table: ogc_latest_depth_to_water_wells - geom_field: point - - avg_tds_wells: - type: collection - title: Average TDS (Water Wells) - description: Average total dissolved solids (TDS) from major chemistry results for each water well. - keywords: [water-wells, chemistry, tds, total-dissolved-solids, average] - extents: - spatial: - bbox: [-109.05, 31.33, -103.00, 37.00] - crs: http://www.opengis.net/def/crs/OGC/1.3/CRS84 - providers: - - type: feature - name: PostgreSQL - data: - host: {postgres_host} - port: {postgres_port} - dbname: {postgres_db} - user: {postgres_user} - password: {postgres_password_env} - search_path: [public] - id_field: id - table: ogc_avg_tds_wells - geom_field: point - latest_tds_wells: type: collection title: Latest TDS (Water Wells) diff --git a/core/pygeoapi.py b/core/pygeoapi.py index 017af4588..849f4f381 100644 --- a/core/pygeoapi.py +++ b/core/pygeoapi.py @@ -74,6 +74,11 @@ "Feature records that do not match another defined thing type." ), "keywords": ["other"], + # "Thing" is internal data-model vocabulary and "other" names no + # recognisable feature class, so this layer is not published on the + # public mount (BDMS-979). Staff GIS clients still reach it through + # /ogcapi-internal, and ogc_other_things is retained either way. + "internal_only": True, }, { "id": "outfalls_wastewater_return_flow", @@ -247,9 +252,12 @@ def _thing_collections_block( user: str, password_placeholder: str, table_prefix: str = "ogc_", + include_internal_only: bool = False, ) -> str: resources: dict[str, dict] = {} for collection in THING_COLLECTIONS: + if collection.get("internal_only") and not include_internal_only: + continue resources[collection["id"]] = { "type": "collection", "title": collection["title"], @@ -384,6 +392,7 @@ def _write_config( table_prefix: str = "ogc_", template_path: Path | None = None, include_edr: bool = False, + include_internal_only: bool = False, ) -> None: host, port, dbname, user, password_placeholder = _pygeoapi_db_settings() template = (template_path or _template_path()).read_text(encoding="utf-8") @@ -394,6 +403,7 @@ def _write_config( user=user, password_placeholder=password_placeholder, table_prefix=table_prefix, + include_internal_only=include_internal_only, ) if include_edr: # EDR collections (core/edr_provider.py), backed by @@ -564,6 +574,7 @@ def mount_pygeoapi_internal(app: FastAPI) -> None: table_prefix="ogc_internal_", template_path=_internal_template_path(), include_edr=True, + include_internal_only=True, ) _generate_openapi(config_path, openapi_path) _assert_server_settings_match(_pygeoapi_dir() / "pygeoapi-config.yml", config_path) diff --git a/tests/features/ogc-cleanup-sprint1.feature b/tests/features/ogc-cleanup-sprint1.feature index 6e3c1e71b..d747c8bef 100644 --- a/tests/features/ogc-cleanup-sprint1.feature +++ b/tests/features/ogc-cleanup-sprint1.feature @@ -51,7 +51,6 @@ Feature: OGC Feature Layer Cleanup — Sprint 1 | meteorological_stations | | diversions_surface_water | | lakes_ponds_reservoirs | - | other_things | | water_well_summary | | depth_to_water_trend_wells | | water_elevation_wells | @@ -59,15 +58,14 @@ Feature: OGC Feature Layer Cleanup — Sprint 1 | minor_chemistry_wells | | latest_tds_wells | | actively_monitored_wells | - | avg_tds_wells | - | latest_depth_to_water_wells | - | locations | | project_areas | Then each response contains only records where release_status is "public" And no response contains a record where release_status is "private" And no response contains a record where release_status is "draft" - # other_things above: A1 must apply the filter to its view, but A18 removes - # other_things from the catalog — run this scenario before A18 is applied + # other_things, avg_tds_wells, latest_depth_to_water_wells and locations + # are not listed above: A16/A17/A18 took them off the public catalog, so a + # public client can no longer request their items. A1's filter still + # applies to their views, which the SQL-level scenarios above cover. @backend @ogc-exposure @sprint-1 @high-priority @A1 @production Scenario: project_areas returns 56 rows after all records are updated to public @@ -237,10 +235,9 @@ Feature: OGC Feature Layer Cleanup — Sprint 1 | lakes_ponds_reservoirs | | soil_gas_sample_locations | | outfalls_wastewater_return_flow | - | other_things | Then each feature includes a last_observation_date property - # other_things above: included in Group A view template, but A18 removes it - # from the catalog — run this scenario before A18 is applied + # other_things is not listed: it is in the Group A view template, but A18 + # took it off the public catalog — it is only reachable on /ogcapi-internal. @backend @ogc-data-currency @sprint-1 @medium-priority @A13 Scenario: last_observation_date is NULL for things with no associated observations @@ -256,11 +253,10 @@ Feature: OGC Feature Layer Cleanup — Sprint 1 | lakes_ponds_reservoirs | | soil_gas_sample_locations | | outfalls_wastewater_return_flow | - | other_things | When a client requests those features Then each feature's last_observation_date property is null - # other_things above: included in Group A view template, but A18 removes it - # from the catalog — run this scenario before A18 is applied + # other_things is not listed: it is in the Group A view template, but A18 + # took it off the public catalog — it is only reachable on /ogcapi-internal. @backend @ogc-data-currency @sprint-1 @medium-priority @A13 Scenario: Consumers can filter Group A layers by last_observation_date @@ -276,30 +272,29 @@ Feature: OGC Feature Layer Cleanup — Sprint 1 | lakes_ponds_reservoirs | | soil_gas_sample_locations | | outfalls_wastewater_return_flow | - | other_things | When a client requests items from each of those layers with filter """ last_observation_date > '2021-01-01' """ Then only features with a last_observation_date of "2023-06-01" are returned from each layer - # other_things above: included in Group A view template, but A18 removes it - # from the catalog — run this scenario before A18 is applied + # other_things is not listed: it is in the Group A view template, but A18 + # took it off the public catalog — it is only reachable on /ogcapi-internal. # --------------------------------------------------------------------------- # A16 — Hide avg_tds_wells and latest_depth_to_water_wells from public catalog # --------------------------------------------------------------------------- - @backend @ogc-data-currency @sprint-1 @medium-priority @A16 + @backend @ogc-data-currency @sprint-1 @medium-priority @A16 @production Scenario: avg_tds_wells is absent from the public collections catalog When a client requests /ogcapi/collections Then the response does not include a collection with id avg_tds_wells - @backend @ogc-data-currency @sprint-1 @medium-priority @A16 + @backend @ogc-data-currency @sprint-1 @medium-priority @A16 @production Scenario: latest_depth_to_water_wells is absent from the public collections catalog When a client requests /ogcapi/collections Then the response does not include a collection with id latest_depth_to_water_wells - @backend @ogc-data-currency @sprint-1 @medium-priority @A16 + @backend @ogc-data-currency @sprint-1 @medium-priority @A16 @production Scenario: Backing matviews for hidden layers are retained in the database Given avg_tds_wells and latest_depth_to_water_wells have been removed from the service catalog When the database schema is inspected @@ -310,12 +305,12 @@ Feature: OGC Feature Layer Cleanup — Sprint 1 # A17 — Hide locations layer from the public catalog # --------------------------------------------------------------------------- - @backend @ogc-data-currency @sprint-1 @medium-priority @A17 + @backend @ogc-data-currency @sprint-1 @medium-priority @A17 @production Scenario: locations is absent from the public collections catalog When a client requests /ogcapi/collections Then the response does not include a collection with id locations - @backend @ogc-data-currency @sprint-1 @medium-priority @A17 + @backend @ogc-data-currency @sprint-1 @medium-priority @A17 @production Scenario: Underlying locations table is retained in the database after catalog removal Given the locations entry has been removed from the service configuration When the database schema is inspected @@ -325,22 +320,21 @@ Feature: OGC Feature Layer Cleanup — Sprint 1 # A18 — Remove other_things from the public catalog # --------------------------------------------------------------------------- - @backend @ogc-naming @sprint-1 @medium-priority @A18 + @backend @ogc-naming @sprint-1 @medium-priority @A18 @production Scenario: other_things is absent from the public collections catalog When a client requests /ogcapi/collections Then the response does not include a collection with id other_things - @backend @ogc-naming @sprint-1 @medium-priority @A18 - Scenario: other_things backing view is dropped when no internal usage exists - Given the other_things view has zero references in the application codebase - When the cleanup is applied - Then the other_things backing view does not exist in the database schema - - @backend @ogc-naming @sprint-1 @medium-priority @A18 - Scenario: other_things backing view is retained when internal usage exists + # The A18 review found internal usage: /ogcapi-internal still publishes the + # layer to staff GIS clients off ogc_internal_other_things, and the public + # ogc_other_things view is still built by the shared Group A view template. + # Both views are therefore retained. + @backend @ogc-naming @sprint-1 @medium-priority @A18 @production + Scenario: other_things backing views are retained because the internal mount uses them Given the other_things view has at least one reference in the application codebase - When the cleanup is applied + When the database schema is inspected Then the other_things backing view still exists in the database schema + And the internal other_things backing view still exists in the database schema # --------------------------------------------------------------------------- # A22 — Verify NULL measuring_point_height assumption for water level layers diff --git a/tests/features/steps/ogc-cleanup-sprint1.py b/tests/features/steps/ogc-cleanup-sprint1.py index 6d701dedd..8497cca13 100644 --- a/tests/features/steps/ogc-cleanup-sprint1.py +++ b/tests/features/steps/ogc-cleanup-sprint1.py @@ -857,6 +857,78 @@ def step_then_no_collection_id_prefixed(context, prefix): assert not offending, f"found collections with id prefixed {prefix!r}: {offending}" +# --------------------------------------------------------------------------- +# A16/A17/A18 -- Layers hidden from the public catalog, backing relations kept +# --------------------------------------------------------------------------- + + +@given( + "avg_tds_wells and latest_depth_to_water_wells have been removed from the " + "service catalog" +) +@given("the locations entry has been removed from the service configuration") +@given("the other_things view has at least one reference in the application codebase") +def step_given_layer_hidden_from_public_catalog(context): + # No-op marker: the catalog is core/pygeoapi-config.yml and + # core.pygeoapi.THING_COLLECTIONS, both artifacts under test rather than + # runtime state to arrange. Same treatment as the A1/A2/A11 givens. + pass + + +@then("the response does not include a collection with id {collection_id}") +def step_then_response_excludes_collection_id(context, collection_id): + payload = context.response.json() + ids = {collection["id"] for collection in payload["collections"]} + assert collection_id not in ids, ( + f"{collection_id} is still published on the public catalog; " + f"collections: {sorted(ids)}" + ) + + +@then("the materialized view for {layer_id} exists in the database schema") +def step_then_matview_for_layer_exists(context, layer_id): + relation = f"ogc_{layer_id}" + assert relation in context.schema_relations, ( + f"{relation} is missing -- A16 hides the layer from the catalog but " + "keeps its materialized view for internal use" + ) + + +@then("the locations table still exists") +def step_then_locations_table_still_exists(context): + # context.schema_relations covers views and materialized views only, so + # the base table needs its own lookup. + with session_ctx() as session: + relkind = session.execute( + text( + "SELECT c.relkind FROM pg_class c " + "JOIN pg_namespace n ON n.oid = c.relnamespace " + "WHERE n.nspname = 'public' AND c.relname = 'location'" + ) + ).scalar_one_or_none() + assert relkind == "r", ( + "the location table is missing -- A17 hides the layer from the " + f"catalog but keeps the underlying table (relkind={relkind!r})" + ) + + +def _assert_other_things_view_exists(context, relation): + assert relation in context.schema_relations, ( + f"{relation} is missing -- A18 hides other_things from the public " + "catalog but /ogcapi-internal still serves the layer" + ) + + +@then("the other_things backing view still exists in the database schema") +def step_then_other_things_view_still_exists(context): + _assert_other_things_view_exists(context, "ogc_other_things") + + +@then("the internal other_things backing view still exists in the database schema") +def step_then_internal_other_things_view_still_exists(context): + _assert_other_things_view_exists(context, "ogc_internal_other_things") + + # --------------------------------------------------------------------------- # A2 -- Replace OGC server metadata placeholders in pygeoapi-config.yml # --------------------------------------------------------------------------- diff --git a/tests/test_ogc.py b/tests/test_ogc.py index 5385d243d..8b49ac0a6 100644 --- a/tests/test_ogc.py +++ b/tests/test_ogc.py @@ -565,7 +565,6 @@ def test_ogc_collections(ogc_client): payload = response.json() ids = {collection["id"] for collection in payload["collections"]} assert { - "locations", "water_wells", "springs", "latest_tds_wells", @@ -577,6 +576,19 @@ def test_ogc_collections(ogc_client): "actively_monitored_wells", "project_areas", }.issubset(ids) + # Hidden from the public catalog: locations duplicates the thing-type + # layers (BDMS-978), avg_tds_wells averages ~1.9 observations per well + # and latest_depth_to_water_wells repeats water_well_summary + # (BDMS-977), and other_things is internal vocabulary (BDMS-979). The + # backing relations are retained and still served on /ogcapi-internal. + assert ids.isdisjoint( + { + "locations", + "avg_tds_wells", + "latest_depth_to_water_wells", + "other_things", + } + ) def test_ogc_new_collection_items_endpoints(ogc_client): @@ -605,16 +617,6 @@ def test_ogc_project_areas_items_expose_groups_with_project_areas(ogc_client, gr assert str(group.id) in ids -@pytest.mark.skip("PostGIS spatial operators not available in CI - see issue #449") -def test_ogc_locations_items_bbox(location): - bbox = "-107.95,33.80,-107.94,33.81" - response = ogc_client.get(f"/ogcapi/collections/locations/items?bbox={bbox}") - assert response.status_code == 200 - payload = response.json() - assert payload["type"] == "FeatureCollection" - assert payload["numberReturned"] >= 1 - - def test_ogc_wells_items_and_item(ogc_client, water_well_thing): response = ogc_client.get("/ogcapi/collections/water_wells/items?limit=20") assert response.status_code == 200 diff --git a/tests/test_pygeoapi_mount.py b/tests/test_pygeoapi_mount.py index e3457a75c..ee9b11df8 100644 --- a/tests/test_pygeoapi_mount.py +++ b/tests/test_pygeoapi_mount.py @@ -109,3 +109,28 @@ def test_loading_a_mount_restores_config_env_vars(): after = {key: os.environ.get(key) for key in pygeoapi._PYGEOAPI_ENV_KEYS} assert after == before + + +# Layers hidden from the public catalog but still served to staff GIS +# clients on /ogcapi-internal: locations duplicates the thing-type layers +# (BDMS-978), avg_tds_wells and latest_depth_to_water_wells are misleading +# or redundant (BDMS-977), other_things is internal vocabulary (BDMS-979). +INTERNAL_ONLY_COLLECTIONS = { + "locations", + "avg_tds_wells", + "latest_depth_to_water_wells", + "other_things", +} + + +def test_hidden_layers_are_internal_only(): + public_module, internal_module = _load_both() + + public_ids = set(public_module.api_.config["resources"]) + internal_ids = set(internal_module.api_.config["resources"]) + + assert public_ids.isdisjoint(INTERNAL_ONLY_COLLECTIONS) + assert INTERNAL_ONLY_COLLECTIONS.issubset(internal_ids) + # The thing-type layers that stay public are on both mounts; the two + # catalogs otherwise differ (the geothermal layers are public-only). + assert {"water_wells", "springs"}.issubset(public_ids & internal_ids)