feat: add thing_id association to AssociatedData and SoilRockResults models

Author: jirhiker

alembic/versions/c2f4a9d0b1e2_create_nma_associated_data.py

@@ -37,6 +37,12 @@ def upgrade() -> None:
             sa.Column("Notes", sa.String(length=255), nullable=True),
             sa.Column("Formation", sa.String(length=15), nullable=True),
             sa.Column("OBJECTID", sa.Integer(), nullable=True, unique=True),
+            sa.Column(
+                "thing_id",
+                sa.Integer(),
+                sa.ForeignKey("thing.id", ondelete="CASCADE"),
+                nullable=True,
+            ),
             sa.UniqueConstraint("LocationId", name="AssociatedData$LocationId"),
         )
         op.create_index("AssociatedData$PointID", "NMA_AssociatedData", ["PointID"])

alembic/versions/f5a6b7c8d9e0_create_nma_soil_rock_results.py

@@ -32,6 +32,12 @@ def upgrade() -> None:
             sa.Column("d13C", sa.Float(), nullable=True),
             sa.Column("d18O", sa.Float(), nullable=True),
             sa.Column("Sampled by", sa.String(length=255), nullable=True),
+            sa.Column(
+                "thing_id",
+                sa.Integer(),
+                sa.ForeignKey("thing.id", ondelete="CASCADE"),
+                nullable=True,
+            ),
         )
         op.create_index(
             "Soil_Rock_Results$Point_ID", "NMA_Soil_Rock_Results", ["Point_ID"]

db/nma_legacy.py

@@ -296,22 +296,11 @@ class AssociatedData(Base):
     notes: Mapped[Optional[str]] = mapped_column("Notes", String(255))
     formation: Mapped[Optional[str]] = mapped_column("Formation", String(15))
     object_id: Mapped[Optional[int]] = mapped_column("OBJECTID", Integer, unique=True)
-
-    major_chemistries: Mapped[List["NMAMajorChemistry"]] = relationship(
-        "NMAMajorChemistry",
-        back_populates="chemistry_sample_info",
-        cascade="all, delete-orphan",
-        passive_deletes=True,
+    thing_id: Mapped[Optional[int]] = mapped_column(
+        Integer, ForeignKey("thing.id", ondelete="CASCADE")
     )
 
-    @validates("thing_id")
-    def validate_thing_id(self, key, value):
-        """Prevent orphan ChemistrySampleInfo - must have a parent Thing."""
-        if value is None:
-            raise ValueError(
-                "ChemistrySampleInfo requires a parent Thing (thing_id cannot be None)"
-            )
-        return value
+    thing: Mapped["Thing"] = relationship("Thing")
 
 
 class SurfaceWaterData(Base):
@@ -414,6 +403,11 @@ class SoilRockResults(Base):
     d13c: Mapped[Optional[float]] = mapped_column("d13C", Float)
     d18o: Mapped[Optional[float]] = mapped_column("d18O", Float)
     sampled_by: Mapped[Optional[str]] = mapped_column("Sampled by", String(255))
+    thing_id: Mapped[Optional[int]] = mapped_column(
+        Integer, ForeignKey("thing.id", ondelete="CASCADE")
+    )
+
+    thing: Mapped["Thing"] = relationship("Thing")
 
 
 class NMAMinorTraceChemistry(Base):

tests/test_associated_data_legacy.py

@@ -32,7 +32,7 @@
 from db.nma_legacy import AssociatedData
 
 
-def test_create_associated_data_all_fields():
+def test_create_associated_data_all_fields(water_well_thing):
     """Test creating an associated data record with all fields."""
     with session_ctx() as session:
         record = AssociatedData(
@@ -42,6 +42,7 @@ def test_create_associated_data_all_fields():
             notes="Legacy notes",
             formation="TEST",
             object_id=42,
+            thing_id=water_well_thing.id,
         )
         session.add(record)
         session.commit()
@@ -53,6 +54,7 @@ def test_create_associated_data_all_fields():
         assert record.notes == "Legacy notes"
         assert record.formation == "TEST"
         assert record.object_id == 42
+        assert record.thing_id == water_well_thing.id
 
         session.delete(record)
         session.commit()

tests/test_soil_rock_results_legacy.py

@@ -31,7 +31,7 @@
 from db.nma_legacy import SoilRockResults
 
 
-def test_create_soil_rock_results_all_fields():
+def test_create_soil_rock_results_all_fields(water_well_thing):
     """Test creating a soil/rock results record with all fields."""
     with session_ctx() as session:
         record = SoilRockResults(
@@ -41,6 +41,7 @@ def test_create_soil_rock_results_all_fields():
             d13c=-5.5,
             d18o=12.3,
             sampled_by="Tester",
+            thing_id=water_well_thing.id,
         )
         session.add(record)
         session.commit()
@@ -53,6 +54,7 @@ def test_create_soil_rock_results_all_fields():
         assert record.d13c == -5.5
         assert record.d18o == 12.3
         assert record.sampled_by == "Tester"
+        assert record.thing_id == water_well_thing.id
         session.delete(record)
         session.commit()
 

transfers/associated_data.py

@@ -23,7 +23,8 @@
 from sqlalchemy.dialects.postgresql import insert
 from sqlalchemy.orm import Session
 
-from db import AssociatedData
+from db import AssociatedData, Thing
+from db.engine import session_ctx
 from transfers.logger import logger
 from transfers.transferer import Transferer
 from transfers.util import replace_nans
@@ -37,6 +38,14 @@ class AssociatedDataTransferer(Transferer):
     def __init__(self, *args, batch_size: int = 1000, **kwargs):
         super().__init__(*args, **kwargs)
         self.batch_size = batch_size
+        self._thing_id_cache: dict[str, int] = {}
+        self._build_thing_id_cache()
+
+    def _build_thing_id_cache(self) -> None:
+        with session_ctx() as session:
+            things = session.query(Thing.name, Thing.id).all()
+            self._thing_id_cache = {name: thing_id for name, thing_id in things}
+        logger.info(f"Built Thing ID cache with {len(self._thing_id_cache)} entries")
 
     def _get_dfs(self) -> tuple[pd.DataFrame, pd.DataFrame]:
         df = self._read_csv(self.source_table)
@@ -83,6 +92,7 @@ def _row_dict(self, row: dict[str, Any]) -> dict[str, Any]:
             "Notes": row.get("Notes"),
             "Formation": row.get("Formation"),
             "OBJECTID": row.get("OBJECTID"),
+            "thing_id": self._thing_id_cache.get(row.get("PointID")),
         }
 
     def _dedupe_rows(

transfers/soil_rock_results.py

@@ -21,7 +21,8 @@
 import pandas as pd
 from sqlalchemy.orm import Session
 
-from db import SoilRockResults
+from db import SoilRockResults, Thing
+from db.engine import session_ctx
 from transfers.logger import logger
 from transfers.transferer import Transferer
 from transfers.util import replace_nans
@@ -35,6 +36,14 @@ class SoilRockResultsTransferer(Transferer):
     def __init__(self, *args, batch_size: int = 1000, **kwargs):
         super().__init__(*args, **kwargs)
         self.batch_size = batch_size
+        self._thing_id_cache: dict[str, int] = {}
+        self._build_thing_id_cache()
+
+    def _build_thing_id_cache(self) -> None:
+        with session_ctx() as session:
+            things = session.query(Thing.name, Thing.id).all()
+            self._thing_id_cache = {name: thing_id for name, thing_id in things}
+        logger.info(f"Built Thing ID cache with {len(self._thing_id_cache)} entries")
 
     def _get_dfs(self) -> tuple[pd.DataFrame, pd.DataFrame]:
         df = self._read_csv(self.source_table)
@@ -67,6 +76,7 @@ def _row_dict(self, row: dict[str, Any]) -> dict[str, Any]:
             "d13C": self._float_val(row.get("d13C")),
             "d18O": self._float_val(row.get("d18O")),
             "Sampled by": row.get("Sampled by"),
+            "thing_id": self._thing_id_cache.get(row.get("Point_ID")),
         }
 
     def _float_val(self, value: Any) -> Optional[float]: