fix: stop per-request OOM on location export endpoints

The /location/feature_collection and /location/shapefile endpoints loaded
the entire SampleLocation table into memory on a single request, spiking
memory enough for App Engine to terminate the process.

- feature_collection: stream GeoJSON row-by-row via StreamingResponse +
  yield_per instead of building two full in-memory lists. Output shape
  unchanged.
- shapefile: write to a tempfile.mkdtemp() dir (App Engine app dir is
  read-only; /tmp is RAM-backed) with yield_per streaming and clean up
  the temp dir via BackgroundTask after send.
- Both endpoints changed from async def to def so blocking sync DB/file
  work runs in the threadpool instead of stalling the event loop.
- db: disable echo on the production async engine and add bounded pools
  (pool_size, max_overflow, pool_recycle, pool_pre_ping) on both engines.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

Author: jirhiker

api/base.py

@@ -13,6 +13,10 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ===============================================================================
+import json
+import os
+import shutil
+import tempfile
 from typing import List, Union
 
 from constants import SRID_WGS84
@@ -29,7 +33,11 @@
 from services.validation.well import validate_screens
 from sqlalchemy import select, func
 from sqlalchemy.orm import Session
-from starlette.responses import FileResponse
+from starlette.background import BackgroundTask
+from starlette.responses import FileResponse, StreamingResponse
+
+# stream DB rows in chunks instead of buffering the whole table in memory
+_STREAM_CHUNK = 1000
 
 from db import get_db_session, adder
 from db.base import (
@@ -196,7 +204,7 @@ def create_equipment(
 
 # ==== Get ============================================
 @router.get("/location/shapefile", summary="Get location as shapefile")
-async def get_location_shapefile(
+def get_location_shapefile(
     query: str = None, session: Session = Depends(get_db_session)
 ):
     """
@@ -206,50 +214,55 @@ async def get_location_shapefile(
     if query:
         sql = sql.where(make_query(SampleLocation, query))
 
-    result = session.execute(sql)
-    locations = result.scalars().all()
-    # create a shapefile from the locations
+    # Write into a temp dir: the App Engine app directory is read-only, and
+    # streaming rows keeps the whole table out of memory at once.
+    tmpdir = tempfile.mkdtemp()
+    shp_path = os.path.join(tmpdir, "locations.shp")
+    zip_path = os.path.join(tmpdir, "locations.zip")
+
+    locations = session.execute(sql).scalars().yield_per(_STREAM_CHUNK)
+    create_shapefile(locations, shp_path)
 
-    create_shapefile(locations, "locations.shp")
-    # Return the shapefile as a zip (optional: zip the .shp, .shx, .dbf files)
     import zipfile
 
-    with zipfile.ZipFile("locations.zip", "w") as zf:
+    with zipfile.ZipFile(zip_path, "w") as zf:
         for ext in ["shp", "shx", "dbf"]:
-            zf.write(f"locations.{ext}")
+            zf.write(os.path.join(tmpdir, f"locations.{ext}"), arcname=f"locations.{ext}")
+
     return FileResponse(
-        "locations.zip", media_type="application/zip", filename="locations.zip"
+        zip_path,
+        media_type="application/zip",
+        filename="locations.zip",
+        background=BackgroundTask(shutil.rmtree, tmpdir, ignore_errors=True),
     )
 
 
 @router.get("/location/feature_collection", summary="Get location feature collection")
-async def get_location_feature_collection(
+def get_location_feature_collection(
     query: str = None, session: Session = Depends(get_db_session)
 ):
     """
     Retrieve all sample locations as a GeoJSON FeatureCollection.
+
+    Streamed row-by-row so the entire table is never buffered in memory at once.
     """
     sql = select(
         SampleLocation, geofunc.ST_AsGeoJSON(SampleLocation.point).label("geojson")
     )
     if query:
         sql = sql.where(make_query(SampleLocation, query))
 
-    result = session.execute(sql)
-    locations = result.all()
-
-    features = []
-    for location, geojson in locations:
-        feature = {
-            "type": "Feature",
-            "geometry": geojson,
-        }
-        features.append(feature)
-
-    return {
-        "type": "FeatureCollection",
-        "features": features,
-    }
+    def generate():
+        yield '{"type": "FeatureCollection", "features": ['
+        first = True
+        result = session.execute(sql).yield_per(_STREAM_CHUNK)
+        for location, geojson in result:
+            feature = {"type": "Feature", "geometry": geojson}
+            yield ("" if first else ",") + json.dumps(feature)
+            first = False
+        yield "]}"
+
+    return StreamingResponse(generate(), media_type="application/json")
 
 
 @router.get(

db/__init__.py

@@ -77,8 +77,12 @@ def asyncify_connection():
 
     return create_async_engine(
         "postgresql+asyncpg://",
-        echo=True,
+        echo=False,
         creator=asyncify_connection,
+        pool_size=5,
+        max_overflow=2,
+        pool_recycle=1800,
+        pool_pre_ping=True,
     )
 
 
@@ -106,6 +110,10 @@ def getconn():
             "postgresql+pg8000://",
             creator=getconn,
             echo=False,
+            pool_size=5,
+            max_overflow=2,
+            pool_recycle=1800,
+            pool_pre_ping=True,
         )
         return engine