diff --git a/.release-please-manifest.json b/.release-please-manifest.json index 5fdd88304..a7fd6a0f1 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "1.1.0" + ".": "1.1.5" } diff --git a/CHANGELOG.md b/CHANGELOG.md index 51bda31fa..0ccd274f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,47 @@ # Changelog +## [1.1.5](https://github.com/DataIntegrationGroup/OcotilloAPI/compare/v1.1.4...v1.1.5) (2026-07-07) + + +### Bug Fixes + +* **deploy:** raise instance class to F4_1G to stop OOM instance churn ([ec012d6](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/ec012d60ff8b985e89c8a2ca4167b4f24cae8540)) + +## [1.1.4](https://github.com/DataIntegrationGroup/OcotilloAPI/compare/v1.1.3...v1.1.4) (2026-07-07) + + +### Bug Fixes + +* **ci:** deploy on inline workflow_call to CD (Production) ([8371f64](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/8371f646b08363a578d339d99da8ebf5863d21c9)) +* **ci:** deploy on inline workflow_call to CD (Production) ([1a11ee9](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/1a11ee9e5237d33ce7b9ee7f093dad5e023436c6)) + +## [1.1.3](https://github.com/DataIntegrationGroup/OcotilloAPI/compare/v1.1.2...v1.1.3) (2026-07-06) + + +### Bug Fixes + +* correct shapefile DBF schema and clean up temp dir on failure ([f63d9ce](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/f63d9ce2d28ba190b63628f40c61edee09ac1a5e)) +* drop yield_per in get_thing_features (incompatible with unique) ([fa727c4](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/fa727c42898e5d8791f248ff4dc0b5f43c7b3da2)) +* stop per-request OOM on /geospatial export endpoint ([d3a1358](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/d3a1358f54b9b6d02a6187d0004ba44b2b365ed0)) +* stop per-request OOM on /geospatial export endpoint ([2ed12b6](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/2ed12b602a080d98e2025fc0e4edefdc341412ba)) +* stream get_thing_features with yield_per instead of buffering ([877fff8](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/877fff833755034e9c0f1b67ed3d6c74a92d4084)) + +## [1.1.2](https://github.com/DataIntegrationGroup/OcotilloAPI/compare/v1.1.1...v1.1.2) (2026-07-06) + + +### Bug Fixes + +* **deploy:** align hotfix migration head with production DB (unblock v1.1.x deploy) ([1ee8a4b](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/1ee8a4b893027b851450d6ed98c811aea283c365)) +* **deploy:** align hotfix migration head with production DB + fix release tag passthrough ([790377f](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/790377f53e57c9f7671d45efbcab617189f2af36)) + +## [1.1.1](https://github.com/DataIntegrationGroup/OcotilloAPI/compare/v1.1.0...v1.1.1) (2026-07-06) + + +### Bug Fixes + +* **deploy:** prevent App Engine request starvation under burst load ([fec7dbd](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/fec7dbd2e0f59d6ce1c3319cda9fded5c6051eb8)) +* **deploy:** prevent App Engine request starvation under burst load ([385c974](https://github.com/DataIntegrationGroup/OcotilloAPI/commit/385c9743185520b82034f0b1549d51c57781f9e9)) + ## [1.1.0](https://github.com/DataIntegrationGroup/OcotilloAPI/compare/v1.0.0...v1.1.0) (2026-06-08) diff --git a/api/geospatial.py b/api/geospatial.py index 082979f8a..2d649d334 100644 --- a/api/geospatial.py +++ b/api/geospatial.py @@ -14,16 +14,21 @@ # limitations under the License. # =============================================================================== import json +import os +import shutil +import tempfile from typing import Annotated, List from fastapi import APIRouter, Query, HTTPException from fastapi.responses import FileResponse from geoalchemy2.shape import to_shape from shapely.io import to_geojson -from starlette.responses import JSONResponse +from starlette.background import BackgroundTask +from starlette.responses import StreamingResponse from core.dependencies import session_dependency, viewer_dependency from db import Group +from db.engine import session_ctx from schemas.thing import FeatureCollectionResponse from services.geospatial_helper import create_shapefile, get_thing_features from services.query_helper import simple_get_by_id @@ -54,8 +59,7 @@ def get_geospatial( """ if format_ == "geojson": - content = get_feature_collection(session, thing_type, group) - return JSONResponse(content=content, media_type="application/geo+json") + return get_feature_collection(thing_type, group) else: return get_location_shapefile(session, thing_type, group) @@ -89,17 +93,17 @@ def get_project_area( def get_feature_collection( - session: session_dependency, thing_type: List[str] | None = None, group: Annotated[ str | int, Query(title="group", description="group", alias="group") ] = None, -) -> FeatureCollectionResponse: - """ - Endpoint to retrieve a GeoJSON FeatureCollection. +) -> StreamingResponse: """ + Retrieve a GeoJSON FeatureCollection. - things = get_thing_features(session, thing_type, group) + Streamed feature-by-feature so the entire result set is never buffered in + memory at once. + """ def make_feature_dict(thing, geometry, elevation, *other): geometry = json.loads(geometry) @@ -115,12 +119,18 @@ def make_feature_dict(thing, geometry, elevation, *other): "geometry": geometry, } - features = [make_feature_dict(*item) for item in things] + def generate(): + # The request-scoped session is closed before this response body + # streams, so open a dedicated session scoped to the stream. + with session_ctx() as stream_session: + yield '{"type": "FeatureCollection", "features": [' + first = True + for item in get_thing_features(stream_session, thing_type, group): + yield ("" if first else ",") + json.dumps(make_feature_dict(*item)) + first = False + yield "]}" - return { - "type": "FeatureCollection", - "features": features, - } + return StreamingResponse(generate(), media_type="application/geo+json") def get_location_shapefile( @@ -133,16 +143,32 @@ def get_location_shapefile( """ things = get_thing_features(session, thing_type, group) - create_shapefile(things, "things.shp") - # Return the shapefile as a zip (optional: zip the .shp, .shx, .dbf files) - import zipfile + # Write into a temp dir: the App Engine app directory is read-only, and /tmp + # is RAM-backed, so build here and clean up after the response is sent. + tmpdir = tempfile.mkdtemp() + try: + shp_path = os.path.join(tmpdir, "things.shp") + zip_path = os.path.join(tmpdir, "things.zip") + + create_shapefile(things, shp_path) + + import zipfile + + with zipfile.ZipFile(zip_path, "w") as zf: + for ext in ["shp", "shx", "dbf"]: + zf.write(os.path.join(tmpdir, f"things.{ext}"), arcname=f"things.{ext}") + except Exception: + # BackgroundTask only runs on a successful response, so clean up here to + # avoid leaking temp dirs when generation fails. + shutil.rmtree(tmpdir, ignore_errors=True) + raise - with zipfile.ZipFile("things.zip", "w") as zf: - for ext in ["shp", "shx", "dbf"]: - zf.write(f"things.{ext}") return FileResponse( - "things.zip", media_type="application/zip", filename="things.zip" + zip_path, + media_type="application/zip", + filename="things.zip", + background=BackgroundTask(shutil.rmtree, tmpdir, ignore_errors=True), ) diff --git a/pyproject.toml b/pyproject.toml index 11ff4d3f7..e4b6aba75 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "OcotilloAPI" -version = "1.1.0" +version = "1.1.5" description = "FastAPI backend and CLI for managing Ocotillo groundwater locations, wells, assets, and bulk water-level data transfers." readme = "README.md" requires-python = ">=3.13" diff --git a/services/geospatial_helper.py b/services/geospatial_helper.py index fc1118aa8..fb8777178 100644 --- a/services/geospatial_helper.py +++ b/services/geospatial_helper.py @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # =============================================================================== +from typing import Iterator + import shapefile from geoalchemy2.functions import ST_GeomFromText, ST_Within, ST_AsGeoJSON from geoalchemy2.shape import to_shape @@ -31,7 +33,7 @@ def get_thing_features( session, thing_type: list | str | None, group: str | int | None -) -> list: +) -> Iterator: # sql = ( # select(Thing, ST_AsGeoJSON(Location.point).label("geojson")) # .join(LocationThingAssociation, Thing.id == LocationThingAssociation.thing_id) @@ -87,15 +89,29 @@ def get_thing_features( else: sql = sql.where(Group.id == group) - # unique needs to be invoked to prevent duplicates from eager loading - return session.execute(sql).unique().all() + # Stream the result with yield_per so the whole table is never buffered in + # memory at once. Thing has no eager-loaded collections (all relationships + # are lazy), so unique() is unnecessary -- and unique() is incompatible with + # yield_per anyway. Dedup defensively by id with a bounded set of ints in + # case the joins ever produce duplicate rows. + seen = set() + result = session.execute(sql.execution_options(yield_per=1000)) + for row in result: + thing_id = row[0].id + if thing_id in seen: + continue + seen.add(thing_id) + yield row def create_shapefile(things: list, filename: str = "things.shp") -> None: # Create a point shapefile with shapefile.Writer(filename, shapeType=shapefile.POINT) as shp: - shp.field("id", "L") + # Field schema must match the values written in shp.record() below: + # id (numeric), name (char), elevation (numeric). + shp.field("id", "N") shp.field("name", "C") + shp.field("elevation", "N", decimal=3) for thing, point, elevation in things: # Assume loc.point is WKT or a Shapely geometry or GeoJSON diff --git a/uv.lock b/uv.lock index 791d52f03..9d74208ac 100644 --- a/uv.lock +++ b/uv.lock @@ -1479,7 +1479,7 @@ wheels = [ [[package]] name = "ocotilloapi" -version = "1.1.0" +version = "1.1.5" source = { editable = "." } dependencies = [ { name = "aiofiles" },