fix: adjust regex check for timezone offsets, allowing : to be included (#181)

Author: stevenhsd

src/dve/core_engine/backends/implementations/spark/spark_helpers.py

@@ -112,7 +112,7 @@ def __post_init__(self):
     "%H": "HH",
     "%M": "mm",
     "%S": "ss",
-    "%z": "XX",
+    "%z": "XXX",
     "%Z": "z",
 }
 """A mapping of python to java datetime component formats. Not exhaustive but will hopefully support

src/dve/core_engine/backends/utilities.py

@@ -40,7 +40,7 @@
     "H": r"[0-9]{2}",
     "M": r"[0-9]{2}",
     "S": r"[0-9]{2}",
-    "z": r"(\+|\-)?[0-9]+(\.[0-9]*)?",
+    "z": r"[+-][0-9]+(:[0-9]{2})?(\.[0-9]*)?",
     "Z": r"[A-Z]{0,3}",
 }
 
@@ -54,7 +54,6 @@ def datetime_format_to_regex(format_str: str) -> str:
     Helper function to convert python datetime formats to regexp string checks for casting
     purposes.
     """
-
     tokens = list(format_str.replace("%", ""))
 
     for idx, tkn in enumerate(tokens):

tests/test_core_engine/test_backends/test_implementations/test_duckdb/test_duckdb_helpers.py

@@ -26,7 +26,7 @@
 )
 from dve.core_engine.backends.utilities import datetime_format_to_regex
 from dve.core_engine.constants import RECORD_INDEX_COLUMN_NAME
-from dve.metadata_parser.domain_types import formatteddatetime
+from dve.metadata_parser.domain_types import conformatteddate, formatteddatetime
 
 @pytest.fixture
 def casting_test_table(temp_ddb_conn):
@@ -225,8 +225,8 @@ def test_duckdb_rel_to_dictionaries(temp_ddb_conn: DuckDBPyConnection,
                           ("date_test", datetime.date,"TRY_CAST(CASE WHEN REGEXP_FULL_MATCH(TRIM(\"date_test\"), '^[0-9]{4}\-[0-9]{2}\-[0-9]{2}$') THEN TRY_STRPTIME(TRIM(\"date_test\"), '%Y-%m-%d') ELSE NULL END as DATE)"),
                           ("timestamp_test", datetime.datetime, "TRY_CAST(CASE WHEN REGEXP_FULL_MATCH(TRIM(\"timestamp_test\"), '^[0-9]{4}\-[0-9]{2}\-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}$') THEN TRY_STRPTIME(TRIM(\"timestamp_test\"), '%Y-%m-%dT%H:%M:%S') ELSE NULL END as TIMESTAMP)"),
                           ("list_int_field", list[int], "TRY_CAST(LIST_TRANSFORM(\"list_int_field\", x -> TRIM(\"x\")) as BIGINT[])"),
-                          ("basic_model", BasicModel, "TRY_CAST(STRUCT_PACK(\"str_field\":= TRIM(\"basic_model\".str_field),\"date_field\":= TRY_CAST(CASE WHEN REGEXP_FULL_MATCH(TRIM(\"basic_model\".date_field), '^[0-9]{4}\-[0-9]{2}\-[0-9]{2}$') THEN TRY_STRPTIME(TRIM(\"basic_model\".date_field), '%Y-%m-%d') ELSE NULL END as DATE),\"timestamp_field\":= TRY_CAST(CASE WHEN REGEXP_FULL_MATCH(TRIM(\"basic_model\".timestamp_field), '^[0-9]{2}/[0-9]{2}/[0-9]{4} [0-9]{2}:[0-9]{2}:[0-9]{2}(\+|\-)?[0-9]+(\.[0-9]*)?$') THEN TRY_STRPTIME(TRIM(\"basic_model\".timestamp_field), '%d/%m/%Y %H:%M:%S%z') ELSE NULL END as TIMESTAMP)) as STRUCT(str_field VARCHAR, date_field DATE, timestamp_field TIMESTAMP))"),
-                          ("another_model", AnotherModel, "TRY_CAST(STRUCT_PACK(\"unique_id\":= TRIM(\"another_model\".unique_id),\"basic_models\":= LIST_TRANSFORM(\"another_model\".basic_models, x -> STRUCT_PACK(\"str_field\":= TRIM(\"x\".str_field),\"date_field\":= TRY_CAST(CASE WHEN REGEXP_FULL_MATCH(TRIM(\"x\".date_field), '^[0-9]{4}\-[0-9]{2}\-[0-9]{2}$') THEN TRY_STRPTIME(TRIM(\"x\".date_field), '%Y-%m-%d') ELSE NULL END as DATE),\"timestamp_field\":= TRY_CAST(CASE WHEN REGEXP_FULL_MATCH(TRIM(\"x\".timestamp_field), '^[0-9]{2}/[0-9]{2}/[0-9]{4} [0-9]{2}:[0-9]{2}:[0-9]{2}(\+|\-)?[0-9]+(\.[0-9]*)?$') THEN TRY_STRPTIME(TRIM(\"x\".timestamp_field), '%d/%m/%Y %H:%M:%S%z') ELSE NULL END as TIMESTAMP)))) as STRUCT(unique_id BIGINT, basic_models STRUCT(str_field VARCHAR, date_field DATE, timestamp_field TIMESTAMP)[]))")])
+                          ("basic_model", BasicModel, "TRY_CAST(STRUCT_PACK(\"str_field\":= TRIM(\"basic_model\".str_field),\"date_field\":= TRY_CAST(CASE WHEN REGEXP_FULL_MATCH(TRIM(\"basic_model\".date_field), '^[0-9]{4}\-[0-9]{2}\-[0-9]{2}$') THEN TRY_STRPTIME(TRIM(\"basic_model\".date_field), '%Y-%m-%d') ELSE NULL END as DATE),\"timestamp_field\":= TRY_CAST(CASE WHEN REGEXP_FULL_MATCH(TRIM(\"basic_model\".timestamp_field), '^[0-9]{2}/[0-9]{2}/[0-9]{4} [0-9]{2}:[0-9]{2}:[0-9]{2}[+-][0-9]+(:[0-9]{2})?(\.[0-9]*)?$') THEN TRY_STRPTIME(TRIM(\"basic_model\".timestamp_field), '%d/%m/%Y %H:%M:%S%z') ELSE NULL END as TIMESTAMP)) as STRUCT(str_field VARCHAR, date_field DATE, timestamp_field TIMESTAMP))"),
+                          ("another_model", AnotherModel, "TRY_CAST(STRUCT_PACK(\"unique_id\":= TRIM(\"another_model\".unique_id),\"basic_models\":= LIST_TRANSFORM(\"another_model\".basic_models, x -> STRUCT_PACK(\"str_field\":= TRIM(\"x\".str_field),\"date_field\":= TRY_CAST(CASE WHEN REGEXP_FULL_MATCH(TRIM(\"x\".date_field), '^[0-9]{4}\-[0-9]{2}\-[0-9]{2}$') THEN TRY_STRPTIME(TRIM(\"x\".date_field), '%Y-%m-%d') ELSE NULL END as DATE),\"timestamp_field\":= TRY_CAST(CASE WHEN REGEXP_FULL_MATCH(TRIM(\"x\".timestamp_field), '^[0-9]{2}/[0-9]{2}/[0-9]{4} [0-9]{2}:[0-9]{2}:[0-9]{2}[+-][0-9]+(:[0-9]{2})?(\.[0-9]*)?$') THEN TRY_STRPTIME(TRIM(\"x\".timestamp_field), '%d/%m/%Y %H:%M:%S%z') ELSE NULL END as TIMESTAMP)))) as STRUCT(unique_id BIGINT, basic_models STRUCT(str_field VARCHAR, date_field DATE, timestamp_field TIMESTAMP)[]))")])
 def test_get_duckdb_cast_statement_from_annotation(field_name, field_type, cast_statement):
     assert get_duckdb_cast_statement_from_annotation(field_name, field_type) == cast_statement
 
@@ -283,6 +283,25 @@ def test_relation_is_empty(temp_ddb_conn: DuckDBPyConnection):
 @pytest.mark.parametrize("date_format, expected_regex", [("%Y-%m-%d", r"^[0-9]{4}\-[0-9]{2}\-[0-9]{2}$"),
                                                           ("%Y-%m-%dT%H:%M:%S", r"^[0-9]{4}\-[0-9]{2}\-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}$"),
                                                           ("%m/%d/%Y", r"^[0-9]{2}/[0-9]{2}/[0-9]{4}$"),
-                                                          ("%Y/%m/%d %H:%M:%S%z", r"^[0-9]{4}/[0-9]{2}/[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}(\+|\-)?[0-9]+(\.[0-9]*)?$")])
+                                                          ("%Y/%m/%d %H:%M:%S%z", r"^[0-9]{4}/[0-9]{2}/[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}[+-][0-9]+(:[0-9]{2})?(\.[0-9]*)?$")])
+                                                          
 def test_datetime_format_to_regex(date_format, expected_regex):
-    assert datetime_format_to_regex(date_format) == expected_regex
+    assert datetime_format_to_regex(date_format) == expected_regex
+
+
+@pytest.mark.parametrize("datetime_str, datetime_format, expected_value", [("2020-06-11", "%Y-%m-%d", datetime.date(2020,6,11)),
+                                                                           ("15/11/2018", "%d/%m/%Y", datetime.date(2018,11,15)),
+                                                                           ("27/14/2027", "%d/%m/%Y", None),
+                                                                           ("2014-05-27", "%d/%m/%y", None),
+                                                                           ("2020-03-15T13:24:54+0000", "%Y-%m-%dT%H:%M:%S%z", datetime.datetime(2020, 3, 15, 13, 24, 54)),
+                                                                           ("2020-03-15T13:24:54+00:00", "%Y-%m-%dT%H:%M:%S%z", datetime.datetime(2020, 3, 15, 13, 24, 54))])
+def test_datetime_casting(temp_ddb_conn: DuckDBPyConnection, datetime_str, datetime_format, expected_value):
+    _, con = temp_ddb_conn
+    if isinstance(expected_value, datetime.datetime):
+        type_ = formatteddatetime(date_format=datetime_format)
+    else:
+        type_ = conformatteddate(date_format=datetime_format)
+    cast_statement = get_duckdb_cast_statement_from_annotation("test", type_)
+    rel = con.sql(f"SELECT '{datetime_str}' as test")
+    check = rel.project(cast_statement).fetchone()[0]
+    assert check == expected_value

tests/test_core_engine/test_backends/test_implementations/test_spark/test_spark_helpers.py

@@ -33,7 +33,7 @@
     df_is_empty
 )
 from dve.core_engine.constants import RECORD_INDEX_COLUMN_NAME
-from dve.metadata_parser.domain_types import conformatteddate
+from dve.metadata_parser.domain_types import conformatteddate, formatteddatetime
 
 from .....fixtures import spark  # pylint: disable=unused-import
 
@@ -380,6 +380,22 @@ def test_df_is_empty(spark):
 
     df5 = df4.union(spark.createDataFrame([(4, None, 8)], schema=_schema_w_ri))
     assert not df_is_empty(df5)
+
+@pytest.mark.parametrize("datetime_str, datetime_format, expected_value", [("2020-06-11", "%Y-%m-%d", dt.date(2020,6,11)),
+                                                                           ("15/11/2018", "%d/%m/%Y", dt.date(2018,11,15)),
+                                                                           ("27/14/2027", "%d/%m/%Y", None),
+                                                                           ("2014-05-27", "%d/%m/%y", None),
+                                                                           ("2020-03-15T13:24:54+00:00", "%Y-%m-%dT%H:%M:%S%z", dt.datetime(2020, 3, 15, 13, 24, 54))])
+def test_datetime_casting(spark, datetime_str, datetime_format, expected_value):
+
+    if isinstance(expected_value, dt.datetime):
+        type_ = formatteddatetime(date_format=datetime_format)
+    else:
+        type_ = conformatteddate(date_format=datetime_format)
+    cast_statement = get_spark_cast_statement_from_annotation("test", type_)
+    df = spark.createDataFrame([(datetime_str,)], schema=StructType([StructField("test", StringType())]))
+    check = df.select(cast_statement.alias("test")).first().test
+    assert check == expected_value