refactor: update well inventory CSV files with corrected UTM coordinates and improved data validation

Author: jirhiker

pyproject.toml

@@ -96,6 +96,7 @@ dependencies = [
   "typing-inspection==0.4.1",
   "tzdata==2025.2",
   "urllib3==2.5.0",
+  "utm>=0.8.1",
   "uvicorn==0.38.0",
   "yarl==1.20.1",
 ]

schemas/well_inventory.py

@@ -17,7 +17,9 @@
 from datetime import datetime
 from typing import Optional, Annotated, TypeAlias
 
-from pydantic import BaseModel, model_validator, BeforeValidator
+import phonenumbers
+import utm
+from pydantic import BaseModel, model_validator, BeforeValidator, validate_email
 
 from constants import STATE_CODES
 from core.enums import (
@@ -80,12 +82,34 @@ def state_validator(v):
     return v
 
 
+def phone_validator(phone_number_str):
+    phone_number_str = phone_number_str.strip()
+    if phone_number_str:
+        parsed_number = phonenumbers.parse(phone_number_str, "US")
+        if phonenumbers.is_valid_number(parsed_number):
+            formatted_number = phonenumbers.format_number(
+                parsed_number, phonenumbers.PhoneNumberFormat.E164
+            )
+            return formatted_number
+        else:
+            raise ValueError(f"Invalid phone number. {phone_number_str}")
+
+
+def email_validator_function(email_str):
+    if email_str:
+        try:
+            validate_email(email_str)
+            return email_str
+        except ValueError as e:
+            raise ValueError(f"Invalid email format. {email_str}") from e
+
+
 # Reusable type
 PhoneTypeField: TypeAlias = Annotated[
     Optional[PhoneType], BeforeValidator(blank_to_none)
 ]
 ContactTypeField: TypeAlias = Annotated[
-    Optional[ContactType], BeforeValidator(primary_default)
+    Optional[ContactType], BeforeValidator(blank_to_none)
 ]
 EmailTypeField: TypeAlias = Annotated[
     Optional[EmailType], BeforeValidator(blank_to_none)
@@ -102,6 +126,10 @@ def state_validator(v):
     Optional[str], BeforeValidator(postal_code_or_none)
 ]
 StateField: TypeAlias = Annotated[Optional[str], BeforeValidator(state_validator)]
+PhoneField: TypeAlias = Annotated[Optional[str], BeforeValidator(phone_validator)]
+EmailField: TypeAlias = Annotated[
+    Optional[str], BeforeValidator(email_validator_function)
+]
 
 
 # ============= EOF =============================================
@@ -126,14 +154,14 @@ class WellInventoryRow(BaseModel):
     contact_1_name: Optional[str] = None
     contact_1_organization: Optional[str] = None
     contact_1_role: ContactRoleField = None
-    contact_1_type: ContactTypeField = "Primary"
-    contact_1_phone_1: Optional[str] = None
+    contact_1_type: ContactTypeField = None
+    contact_1_phone_1: PhoneField = None
     contact_1_phone_1_type: PhoneTypeField = None
-    contact_1_phone_2: Optional[str] = None
+    contact_1_phone_2: PhoneField = None
     contact_1_phone_2_type: PhoneTypeField = None
-    contact_1_email_1: Optional[str] = None
+    contact_1_email_1: EmailField = None
     contact_1_email_1_type: EmailTypeField = None
-    contact_1_email_2: Optional[str] = None
+    contact_1_email_2: EmailField = None
     contact_1_email_2_type: EmailTypeField = None
     contact_1_address_1_line_1: Optional[str] = None
     contact_1_address_1_line_2: Optional[str] = None
@@ -151,14 +179,14 @@ class WellInventoryRow(BaseModel):
     contact_2_name: Optional[str] = None
     contact_2_organization: Optional[str] = None
     contact_2_role: ContactRoleField = None
-    contact_2_type: ContactTypeField = "Primary"
-    contact_2_phone_1: Optional[str] = None
+    contact_2_type: ContactTypeField = None
+    contact_2_phone_1: PhoneField = None
     contact_2_phone_1_type: PhoneTypeField = None
-    contact_2_phone_2: Optional[str] = None
+    contact_2_phone_2: PhoneField = None
     contact_2_phone_2_type: PhoneTypeField = None
-    contact_2_email_1: Optional[str] = None
+    contact_2_email_1: EmailField = None
     contact_2_email_1_type: EmailTypeField = None
-    contact_2_email_2: Optional[str] = None
+    contact_2_email_2: EmailField = None
     contact_2_email_2_type: EmailTypeField = None
     contact_2_address_1_line_1: Optional[str] = None
     contact_2_address_1_line_2: Optional[str] = None
@@ -220,6 +248,16 @@ class WellInventoryRow(BaseModel):
 
     @model_validator(mode="after")
     def validate_model(self):
+        # verify utm in NM
+        zone = int(self.utm_zone[:-1])
+        northern = self.utm_zone[-1] == "N"
+
+        lat, lon = utm.to_latlon(
+            self.utm_easting, self.utm_northing, zone, northern=northern
+        )
+        if not ((31.33 <= lat <= 37.00) and (-109.05 <= lon <= -103.00)):
+            raise ValueError("UTM coordinates are outside of the NM")
+
         required_attrs = ("line_1", "type", "state", "city", "postal_code")
         all_attrs = ("line_1", "line_2", "type", "state", "city", "postal_code")
         for jdx in (1, 2):
@@ -234,19 +272,30 @@ def validate_model(self):
                         raise ValueError("All contact address fields must be provided")
 
                 name = getattr(self, f"{key}_name")
-                if name and not getattr(self, f"{key}_role"):
-                    raise ValueError("Role must be provided if name is provided")
-
-                phone = getattr(self, f"{key}_phone_1")
-                phone_type = getattr(self, f"{key}_phone_1_type")
+                if name:
+                    if not getattr(self, f"{key}_role"):
+                        raise ValueError(
+                            f"{key}_role must be provided if name is provided"
+                        )
+                    if not getattr(self, f"{key}_type"):
+                        raise ValueError(
+                            f"{key}_type must be provided if name is provided"
+                        )
+
+                phone = getattr(self, f"{key}_phone_{idx}")
+                tag = f"{key}_phone_{idx}_type"
+                phone_type = getattr(self, f"{key}_phone_{idx}_type")
                 if phone and not phone_type:
                     raise ValueError(
-                        "Phone type must be provided if phone number is provided"
+                        f"{tag} must be provided if phone number is provided"
                     )
 
-                email = getattr(self, f"{key}_email_1")
-                email_type = getattr(self, f"{key}_email_1_type")
+                email = getattr(self, f"{key}_email_{idx}")
+                tag = f"{key}_email_{idx}_type"
+                email_type = getattr(self, tag)
                 if email and not email_type:
-                    raise ValueError("Email type must be provided if email is provided")
+                    raise ValueError(
+                        f"{tag} type must be provided if email is provided"
+                    )
 
         return self

tests/features/data/well-inventory-duplicate.csv

@@ -1,3 +1,3 @@
 project,measuring_point_height_ft,well_name_point_id,site_name,date_time,field_staff,contact_role,utm_easting,utm_northing,utm_zone,elevation_ft,elevation_method
-foo,10,WELL001,Site Alpha,2025-02-15T10:30:00-08:00,Jane Doe,Owner,345678.12,3987654.21,13,5120.5,LiDAR DEM
-foob,10,WELL001,Site Beta,2025-03-20T09:15:00-08:00,John Smith,Manager,346789.34,3987655.32,13,5130.7,LiDAR DEM
+foo,10,WELL001,Site Alpha,2025-02-15T10:30:00-08:00,Jane Doe,Owner,250000,4000000,13N,5120.5,LiDAR DEM
+foob,10,WELL001,Site Beta,2025-03-20T09:15:00-08:00,John Smith,Manager,250000,4000000,13N,5130.7,LiDAR DEM

tests/features/data/well-inventory-invalid-contact-type.csv

@@ -0,0 +1,3 @@
+project,well_name_point_id,site_name,date_time,field_staff,utm_easting,utm_northing,utm_zone,elevation_ft,elevation_method,measuring_point_height_ft,field_staff_2,field_staff_3,contact_1_name,contact_1_organization,contact_1_role,contact_1_type,contact_1_phone_1,contact_1_phone_1_type,contact_1_phone_2,contact_1_phone_2_type,contact_1_email_1,contact_1_email_1_type,contact_1_email_2,contact_1_email_2_type,contact_1_address_1_line_1,contact_1_address_1_line_2,contact_1_address_1_type,contact_1_address_1_state,contact_1_address_1_city,contact_1_address_1_postal_code,contact_1_address_2_line_1,contact_1_address_2_line_2,contact_1_address_2_type,contact_1_address_2_state,contact_1_address_2_city,contact_1_address_2_postal_code,contact_2_name,contact_2_organization,contact_2_role,contact_2_type,contact_2_phone_1,contact_2_phone_1_type,contact_2_phone_2,contact_2_phone_2_type,contact_2_email_1,contact_2_email_1_type,contact_2_email_2,contact_2_email_2_type,contact_2_address_1_line_1,contact_2_address_1_line_2,contact_2_address_1_type,contact_2_address_1_state,contact_2_address_1_city,contact_2_address_1_postal_code,contact_2_address_2_line_1,contact_2_address_2_line_2,contact_2_address_2_type,contact_2_address_2_state,contact_2_address_2_city,contact_2_address_2_postal_code,directions_to_site,specific_location_of_well,repeat_measurement_permission,sampling_permission,datalogger_installation_permission,public_availability_acknowledgement,result_communication_preference,contact_special_requests_notes,ose_well_record_id,date_drilled,completion_source,total_well_depth_ft,historic_depth_to_water_ft,depth_source,well_pump_type,well_pump_depth_ft,is_open,datalogger_possible,casing_diameter_ft,measuring_point_description,well_purpose,well_hole_status,monitoring_frequency,sampling_scenario_notes,well_measuring_notes,sample_possible
+Middle Rio Grande Groundwater Monitoring,MRG-001_MP1,Smith Farm Domestic Well,2025-02-15T10:30:00-07:00,A Lopez,250000,4000000,13N,5250,Survey-grade GPS,1.5,B Chen,,John Smith,NMBGMR,Owner,foo,505-555-0101,Primary,,,john.smith@example.com,Primary,,,123 County Rd 7,,Mailing,NM,Los Lunas,87031,,,,,,,Maria Garcia,NMBGMR,Principal Investigator,Secondary,505-555-0123,Home,,,maria.garcia@mrgcd.nm.gov,Work,,,1931 2nd St SW,Suite 200,Mailing,NM,Albuquerque,87102,,,,,,,Gate off County Rd 7 0.4 miles south of canal crossing,Domestic well in pump house east of residence,True,True,True,True,email,Call before visits during irrigation season,OSE-123456,2010-06-15,OSE well record,280,45,owner estimate,submersible,200,True,True,0.5,Top of steel casing inside pump house marked with orange paint,Domestic,active,Biannual,Sample only when pump has been off more than 12 hours,Measure before owner starts irrigation,True
+Middle Rio Grande Groundwater Monitoring,MRG-003_MP1,Old Orchard Well,2025-01-20T09:00:00-07:00,B Chen,250000,4000000,13N,5320,Global positioning system (GPS),1.8,,,Emily Davis,NMBGMR,Biologist,Primary,505-555-0303,Work,,,emily.davis@example.org,Work,,,78 Orchard Ln,,Mailing,NM,Los Lunas,87031,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,From Main St turn east on Orchard Ln well house at dead end,Abandoned irrigation well in small cinderblock building,False,False,False,True,phone,Owner prefers weekday visits,,1965-04-10,historic log scanned,350,60,historic log,vertical turbine inactive,280,False,False,0.75,Top of steel casing under removable hatch use fixed reference mark,Irrigation,abandoned,Annual,Sampling not permitted water level only when owner present,Well house can be locked coordinate ahead,False

tests/features/data/well-inventory-invalid-date-format.csv

@@ -0,0 +1,3 @@
+project,well_name_point_id,site_name,date_time,field_staff,utm_easting,utm_northing,utm_zone,elevation_ft,elevation_method,measuring_point_height_ft,field_staff_2,field_staff_3,contact_1_name,contact_1_organization,contact_1_role,contact_1_type,contact_1_phone_1,contact_1_phone_1_type,contact_1_phone_2,contact_1_phone_2_type,contact_1_email_1,contact_1_email_1_type,contact_1_email_2,contact_1_email_2_type,contact_1_address_1_line_1,contact_1_address_1_line_2,contact_1_address_1_type,contact_1_address_1_state,contact_1_address_1_city,contact_1_address_1_postal_code,contact_1_address_2_line_1,contact_1_address_2_line_2,contact_1_address_2_type,contact_1_address_2_state,contact_1_address_2_city,contact_1_address_2_postal_code,contact_2_name,contact_2_organization,contact_2_role,contact_2_type,contact_2_phone_1,contact_2_phone_1_type,contact_2_phone_2,contact_2_phone_2_type,contact_2_email_1,contact_2_email_1_type,contact_2_email_2,contact_2_email_2_type,contact_2_address_1_line_1,contact_2_address_1_line_2,contact_2_address_1_type,contact_2_address_1_state,contact_2_address_1_city,contact_2_address_1_postal_code,contact_2_address_2_line_1,contact_2_address_2_line_2,contact_2_address_2_type,contact_2_address_2_state,contact_2_address_2_city,contact_2_address_2_postal_code,directions_to_site,specific_location_of_well,repeat_measurement_permission,sampling_permission,datalogger_installation_permission,public_availability_acknowledgement,result_communication_preference,contact_special_requests_notes,ose_well_record_id,date_drilled,completion_source,total_well_depth_ft,historic_depth_to_water_ft,depth_source,well_pump_type,well_pump_depth_ft,is_open,datalogger_possible,casing_diameter_ft,measuring_point_description,well_purpose,well_hole_status,monitoring_frequency,sampling_scenario_notes,well_measuring_notes,sample_possible
+Middle Rio Grande Groundwater Monitoring,MRG-001_MP1,Smith Farm Domestic Well,25-02-15T10:30:00-07:00,A Lopez,250000,4000000,13N,5250,Survey-grade GPS,1.5,B Chen,,John Smith,NMBGMR,Owner,Primary,505-555-0101,Primary,,,john.smith@example.com,Primary,,,123 County Rd 7,,Mailing,NM,Los Lunas,87031,,,,,,,Maria Garcia,NMBGMR,Principal Investigator,Secondary,505-555-0123,Home,,,maria.garcia@mrgcd.nm.gov,Work,,,1931 2nd St SW,Suite 200,Mailing,NM,Albuquerque,87102,,,,,,,Gate off County Rd 7 0.4 miles south of canal crossing,Domestic well in pump house east of residence,True,True,True,True,email,Call before visits during irrigation season,OSE-123456,2010-06-15,OSE well record,280,45,owner estimate,submersible,200,True,True,0.5,Top of steel casing inside pump house marked with orange paint,Domestic,active,Biannual,Sample only when pump has been off more than 12 hours,Measure before owner starts irrigation,True
+Middle Rio Grande Groundwater Monitoring,MRG-003_MP1,Old Orchard Well,2025-01-20T09:00:00-07:00,B Chen,250000,4000000,13N,5320,Global positioning system (GPS),1.8,,,Emily Davis,NMBGMR,Biologist,Primary,505-555-0303,Work,,,emily.davis@example.org,Work,,,78 Orchard Ln,,Mailing,NM,Los Lunas,87031,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,From Main St turn east on Orchard Ln well house at dead end,Abandoned irrigation well in small cinderblock building,False,False,False,True,phone,Owner prefers weekday visits,,1965-04-10,historic log scanned,350,60,historic log,vertical turbine inactive,280,False,False,0.75,Top of steel casing under removable hatch use fixed reference mark,Irrigation,abandoned,Annual,Sampling not permitted water level only when owner present,Well house can be locked coordinate ahead,False

tests/features/data/well-inventory-invalid-date.csv

@@ -1,5 +1,5 @@
 well_name_point_id,site_name,date_time,field_staff,contact_role,utm_easting,utm_northing,utm_zone,elevation_ft,elevation_method
-WELL005,Site Alpha,2025-02-30T10:30:00-08:00,Jane Doe,Owner,345678.12,3987654.21,13,5120.5,GPS
-WELL006,Site Beta,2025-13-20T09:15:00-08:00,John Smith,Manager,346789.34,3987655.32,13,5130.7,Survey
-WELL007,Site Gamma,not-a-date,Emily Clark,Supervisor,347890.45,3987657.54,13,5150.3,Survey
-WELL008,Site Delta,2025-04-10 11:00:00,Michael Lee,Technician,348901.56,3987658.65,13,5160.4,GPS
+WELL005,Site Alpha,2025-02-30T10:30:00-08:00,Jane Doe,Owner,250000,4000000,13N,5120.5,GPS
+WELL006,Site Beta,2025-13-20T09:15:00-08:00,John Smith,Manager,250000,4000000,13N,5130.7,Survey
+WELL007,Site Gamma,not-a-date,Emily Clark,Supervisor,250000,4000000,13N,5150.3,Survey
+WELL008,Site Delta,2025-04-10 11:00:00,Michael Lee,Technician,250000,4000000,13N,5160.4,GPS

tests/features/data/well-inventory-invalid-email.csv

@@ -0,0 +1,3 @@
+project,well_name_point_id,site_name,date_time,field_staff,utm_easting,utm_northing,utm_zone,elevation_ft,elevation_method,measuring_point_height_ft,field_staff_2,field_staff_3,contact_1_name,contact_1_organization,contact_1_role,contact_1_type,contact_1_phone_1,contact_1_phone_1_type,contact_1_phone_2,contact_1_phone_2_type,contact_1_email_1,contact_1_email_1_type,contact_1_email_2,contact_1_email_2_type,contact_1_address_1_line_1,contact_1_address_1_line_2,contact_1_address_1_type,contact_1_address_1_state,contact_1_address_1_city,contact_1_address_1_postal_code,contact_1_address_2_line_1,contact_1_address_2_line_2,contact_1_address_2_type,contact_1_address_2_state,contact_1_address_2_city,contact_1_address_2_postal_code,contact_2_name,contact_2_organization,contact_2_role,contact_2_type,contact_2_phone_1,contact_2_phone_1_type,contact_2_phone_2,contact_2_phone_2_type,contact_2_email_1,contact_2_email_1_type,contact_2_email_2,contact_2_email_2_type,contact_2_address_1_line_1,contact_2_address_1_line_2,contact_2_address_1_type,contact_2_address_1_state,contact_2_address_1_city,contact_2_address_1_postal_code,contact_2_address_2_line_1,contact_2_address_2_line_2,contact_2_address_2_type,contact_2_address_2_state,contact_2_address_2_city,contact_2_address_2_postal_code,directions_to_site,specific_location_of_well,repeat_measurement_permission,sampling_permission,datalogger_installation_permission,public_availability_acknowledgement,result_communication_preference,contact_special_requests_notes,ose_well_record_id,date_drilled,completion_source,total_well_depth_ft,historic_depth_to_water_ft,depth_source,well_pump_type,well_pump_depth_ft,is_open,datalogger_possible,casing_diameter_ft,measuring_point_description,well_purpose,well_hole_status,monitoring_frequency,sampling_scenario_notes,well_measuring_notes,sample_possible
+Middle Rio Grande Groundwater Monitoring,MRG-001_MP1,Smith Farm Domestic Well,2025-02-15T10:30:00-07:00,A Lopez,250000,4000000,13N,5250,Survey-grade GPS,1.5,B Chen,,John Smith,NMBGMR,Owner,Primary,505-555-0101,Primary,,,john.smithexample.com,Primary,,,123 County Rd 7,,Mailing,NM,Los Lunas,87031,,,,,,,Maria Garcia,NMBGMR,Principal Investigator,Secondary,505-555-0123,Home,,,maria.garcia@mrgcd.nm.gov,Work,,,1931 2nd St SW,Suite 200,Mailing,NM,Albuquerque,87102,,,,,,,Gate off County Rd 7 0.4 miles south of canal crossing,Domestic well in pump house east of residence,True,True,True,True,email,Call before visits during irrigation season,OSE-123456,2010-06-15,OSE well record,280,45,owner estimate,submersible,200,True,True,0.5,Top of steel casing inside pump house marked with orange paint,Domestic,active,Biannual,Sample only when pump has been off more than 12 hours,Measure before owner starts irrigation,True
+Middle Rio Grande Groundwater Monitoring,MRG-003_MP1,Old Orchard Well,2025-01-20T09:00:00-07:00,B Chen,250000,4000000,13N,5320,Global positioning system (GPS),1.8,,,Emily Davis,NMBGMR,Biologist,Primary,505-555-0303,Work,,,emily.davis@example.org,Work,,,78 Orchard Ln,,Mailing,NM,Los Lunas,87031,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,From Main St turn east on Orchard Ln well house at dead end,Abandoned irrigation well in small cinderblock building,False,False,False,True,phone,Owner prefers weekday visits,,1965-04-10,historic log scanned,350,60,historic log,vertical turbine inactive,280,False,False,0.75,Top of steel casing under removable hatch use fixed reference mark,Irrigation,abandoned,Annual,Sampling not permitted water level only when owner present,Well house can be locked coordinate ahead,False

tests/features/data/well-inventory-invalid-lexicon.csv

@@ -1,6 +1,5 @@
 project,well_name_point_id,site_name,date_time,field_staff,utm_easting,utm_northing,utm_zone,elevation_ft,elevation_method,measuring_point_height_ft,contact_role,contact_type
-ProjectA,WELL001,Site1,2025-02-15T10:30:00-08:00,John Doe,345678,3987654,13,5000,Survey,2.5,INVALID_ROLE,owner
-ProjectB,WELL002,Site2,2025-02-16T11:00:00-08:00,Jane Smith,345679,3987655,13,5100,Survey,2.7,manager,INVALID_TYPE
-ProjectC,WELL003,Site3,2025-02-17T09:45:00-08:00,Jim Beam,345680,3987656,13,5200,INVALID_METHOD,2.6,manager,owner
-ProjectD,WELL004,Site4,2025-02-18T08:20:00-08:00,Jack Daniels,345681,3987657,13,5300,Survey,2.8,INVALID_ROLE,INVALID_TYPE
-
+ProjectA,WELL001,Site1,2025-02-15T10:30:00-08:00,John Doe,250000,4000000,13N,5000,Survey,2.5,INVALID_ROLE,owner
+ProjectB,WELL002,Site2,2025-02-16T11:00:00-08:00,Jane Smith,250000,4000000,13N,5100,Survey,2.7,manager,INVALID_TYPE
+ProjectC,WELL003,Site3,2025-02-17T09:45:00-08:00,Jim Beam,250000,4000000,13N,5200,INVALID_METHOD,2.6,manager,owner
+ProjectD,WELL004,Site4,2025-02-18T08:20:00-08:00,Jack Daniels,250000,4000000,13N,5300,Survey,2.8,INVALID_ROLE,INVALID_TYPE