feat: improve data coercion functions for sensor transfer processing

Author: jirhiker

tests/test_sensor_transfer.py

@@ -1,8 +1,7 @@
+from transfers.sensor_transfer import _coerce_wi_mic_gain, _coerce_wi_int
 import numpy as np
 import pandas as pd
 
-from transfers.sensor_transfer import _coerce_wi_mic_gain
-
 
 def test_coerce_wi_mic_gain_numeric():
     assert _coerce_wi_mic_gain(1) is True
@@ -22,3 +21,22 @@ def test_coerce_wi_mic_gain_handles_none_like():
     assert _coerce_wi_mic_gain("  ") is None
     assert _coerce_wi_mic_gain(pd.NA) is None
     assert _coerce_wi_mic_gain(np.nan) is None
+
+
+def test_coerce_wi_int_numeric():
+    assert _coerce_wi_int(1) == 1
+    assert _coerce_wi_int(1.9) == 1
+    assert _coerce_wi_int(0.0) == 0
+
+
+def test_coerce_wi_int_strings():
+    assert _coerce_wi_int("2") == 2
+    assert _coerce_wi_int(" 3.0 ") == 3
+    assert _coerce_wi_int("true") is None
+
+
+def test_coerce_wi_int_none_like():
+    assert _coerce_wi_int(None) is None
+    assert _coerce_wi_int("  ") is None
+    assert _coerce_wi_int(pd.NA) is None
+    assert _coerce_wi_int(np.nan) is None

transfers/sensor_transfer.py

@@ -48,24 +48,36 @@
 }
 
 
+def _coerce_wi_int(value):
+    if value is None or (isinstance(value, str) and not value.strip()):
+        return None
+    if isinstance(value, bool):
+        return int(value)
+    try:
+        if pd.isna(value):
+            return None
+    except TypeError:
+        pass
+    try:
+        return int(float(value))
+    except (TypeError, ValueError):
+        return None
+
+
 def _coerce_wi_mic_gain(value):
     if value is None or (isinstance(value, str) and not value.strip()):
         return None
     if isinstance(value, str):
         value = value.strip()
-    if pd.isna(value):
-        return None
-    if isinstance(value, bool):
-        return value
+    try:
+        if pd.isna(value):
+            return None
+    except TypeError:
+        pass
     try:
         return bool(int(float(value)))
-    except (ValueError, TypeError):
-        lowered = str(value).strip().lower()
-        if lowered in {"true", "t", "yes", "y"}:
-            return True
-        if lowered in {"false", "f", "no", "n"}:
-            return False
-    return None
+    except (TypeError, ValueError):
+        return None
 
 
 class SensorTransferer(ThingBasedTransferer):
@@ -238,12 +250,12 @@ def _group_step(self, session: Session, row: pd.Series, db_item: Base):
             hanging_cable_length=row.HangingCableLength,
             hanging_point_height=row.HangingPointHgt,
             hanging_point_description=row.HangingPointDescription,
-            nma_WI_Duration=row.WI_Duration,
-            nma_WI_EndFrequency=row.WI_EndFrequency,
-            nma_WI_Magnitude=row.WI_Magnitude,
+            nma_WI_Duration=_coerce_wi_int(row.WI_Duration),
+            nma_WI_EndFrequency=_coerce_wi_int(row.WI_EndFrequency),
+            nma_WI_Magnitude=_coerce_wi_int(row.WI_Magnitude),
             nma_WI_MicGain=_coerce_wi_mic_gain(row.WI_MicGain),
-            nma_WI_MinSoundDepth=row.WI_MinSoundDepth,
-            nma_WI_StartFrequency=row.WI_StartFrequency,
+            nma_WI_MinSoundDepth=_coerce_wi_int(row.WI_MinSoundDepth),
+            nma_WI_StartFrequency=_coerce_wi_int(row.WI_StartFrequency),
         )
         session.add(deployment)
         logger.info(