From f9f97c3d738e37c5267d27eb2cec640657e41999 Mon Sep 17 00:00:00 2001 From: Kevin Costner <120246174+kevincostner17@users.noreply.github.com> Date: Mon, 21 Sep 2026 01:26:56 +0530 Subject: [PATCH] fix(dtypes): dayfirst=True no longer corrupts ISO-8601 dates fd.clean(df, dayfirst=True) on ["2021-01-05", "2021-02-11"] silently returned 2021-05-01 and 2021-11-02 -- month and day swapped, with no warning, no report entry and no coercion record. dayfirst exists to resolve ambiguous short dates like 05/12/2021. An ISO-8601 date is YYYY-MM-DD by definition and has no ambiguity to resolve, so this was not a defensible reading of the input. Cause: pandas 2 infers one format for a whole column from its first value, and under dayfirst=True reads an ISO date as %Y-%d-%m. _parse_datetime now passes format="mixed" in that case, so each value is read by its own shape. pandas 1.x infers per value and was never affected -- verified directly on 1.5.3 -- and has no format="mixed", hence the version guard. What made it easy to miss is that the corruption was data-dependent. It was silent only while every day was <= 12: a day >= 13 made the guessed format fail on that value, dropped the parse share below datetime_threshold, and triggered the mixed-format retry that produced the correct reading. The same column therefore read correctly or incorrectly depending on values it happened to contain, or on an unrelated threshold. A user validating on a sample containing a day >= 13 would see correct output and ship. Only the top-level dayfirst=True kwarg was affected. dayfirst="auto" and both semantic_context routes were already correct, which is why this was never caught. Verified the tests fail without the fix rather than assuming: reverting src gives 3 failed / 5 passed. With it, 8 pass on py3.12/pandas 2.3.3 and on py3.9/pandas 1.5.3. dayfirst still resolves genuinely ambiguous slash dates in both directions. --- CHANGELOG.md | 28 +++++++++++ src/freshdata/steps/dtypes.py | 11 ++++- tests/test_dayfirst_iso_dates.py | 81 ++++++++++++++++++++++++++++++++ 3 files changed, 119 insertions(+), 1 deletion(-) create mode 100644 tests/test_dayfirst_iso_dates.py diff --git a/CHANGELOG.md b/CHANGELOG.md index d62ad8b..fd3fcb9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,34 @@ adheres to [Semantic Versioning](https://semver.org/). ## [Unreleased] +### Fixed + +- `dayfirst=True` no longer reinterprets unambiguous ISO-8601 dates. + `fd.clean(df, dayfirst=True)` on `["2021-01-05", "2021-02-11"]` silently + returned `2021-05-01`, `2021-11-02` — month and day swapped, with no warning + and no coercion record. `dayfirst` resolves *ambiguous* short dates such as + `05/12/2021`; an ISO-8601 date is `YYYY-MM-DD` by definition and has no + ambiguity to resolve. Cause: pandas 2 infers one format for a whole column + from its first value and, under `dayfirst=True`, reads an ISO date as + `%Y-%d-%m`; `_parse_datetime` now passes `format="mixed"` in that case so + each value is read by its own shape. pandas 1.x infers per value and was + never affected, and has no `format="mixed"`, so the change is guarded on the + pandas major version. + + The corruption was data-dependent, which is what made it easy to miss: it was + silent only while every day was `<= 12`, because a day `>= 13` made the + guessed format fail, dropped the parse share below `datetime_threshold`, and + triggered the mixed-format retry that produced the correct reading. The same + column therefore read correctly or incorrectly depending on values it + happened to contain, or on an unrelated threshold. + + **Compatibility impact:** output changes for ISO-8601 columns cleaned with + `dayfirst=True` on pandas 2 — from a wrong reading to the correct one. + `dayfirst` behaviour on genuinely ambiguous slash dates is unchanged, as are + the `dayfirst="auto"` and `semantic_context` routes, which were never + affected. + + ### Documentation - `StreamingCleanConfig.window_size` was documented as sizing "rolling diff --git a/src/freshdata/steps/dtypes.py b/src/freshdata/steps/dtypes.py index 263d3ec..7a84c05 100644 --- a/src/freshdata/steps/dtypes.py +++ b/src/freshdata/steps/dtypes.py @@ -381,7 +381,16 @@ def _parse_datetime( s: pd.Series, mixed_formats: bool, dayfirst: bool = False ) -> pd.Series | None: kwargs: dict = {"errors": "coerce", "dayfirst": dayfirst} - if mixed_formats: + if mixed_formats or (dayfirst and PANDAS_MAJOR >= 2): + # ``dayfirst`` exists to resolve *ambiguous* short dates like + # ``05/12/2021``. An ISO-8601 date is not ambiguous. But pandas 2's + # format inference infers a single format for the whole column from + # the first value, and with ``dayfirst=True`` it reads ``2021-01-05`` + # as ``%Y-%d-%m`` -- silently returning 2021-05-01. ``format="mixed"`` + # parses each value by its own apparent format, which fixes ISO input + # while leaving genuinely ambiguous slash dates to ``dayfirst``. + # pandas 1.x infers per value already and is unaffected; it also has + # no ``format="mixed"``, hence the version guard. kwargs["format"] = "mixed" with warnings.catch_warnings(): warnings.simplefilter("ignore") # format-inference chatter; report covers it diff --git a/tests/test_dayfirst_iso_dates.py b/tests/test_dayfirst_iso_dates.py new file mode 100644 index 0000000..f068df9 --- /dev/null +++ b/tests/test_dayfirst_iso_dates.py @@ -0,0 +1,81 @@ +"""``dayfirst=True`` must not reinterpret unambiguous ISO-8601 dates. + +``dayfirst`` exists to resolve *ambiguous* short dates such as ``05/12/2021``. +An ISO-8601 date is ``YYYY-MM-DD`` by definition, so there is nothing for it to +resolve. Before this fix ``fd.clean(df, dayfirst=True)`` silently returned +``2021-05-01`` for the input ``2021-01-05``. + +The cause was pandas 2's format inference, which picks ONE format for a whole +column from its first value and, with ``dayfirst=True``, reads an ISO date as +``%Y-%d-%m``. pandas 1.x infers per value and was never affected. + +The corruption was data-dependent, which is what made it dangerous: it is +silent only while every day is <= 12. A day >= 13 makes the guessed format fail +on that value, dropping the parse share below ``datetime_threshold`` so the +mixed-format retry replaces the column with the correct reading. The same +column therefore read correctly or incorrectly depending on values it happened +to contain, or on an unrelated threshold. +""" + +from __future__ import annotations + +import pandas as pd +import pytest + +import freshdata as fd + +ISO = ["2021-01-05", "2021-02-11", "2021-03-09"] + + +def _dates(frame, column="v"): + return [str(v.date()) for v in frame[column]] + + +def test_iso_dates_are_not_reinterpreted_when_dayfirst_is_true(): + """The regression. Before the fix this returned 2021-05-01 and friends.""" + out = fd.clean(pd.DataFrame({"v": ISO}), dayfirst=True, verbose=False) + assert _dates(out) == ISO + + +@pytest.mark.parametrize("dayfirst", [True, False, "auto", None]) +def test_iso_dates_read_the_same_whatever_dayfirst_says(dayfirst): + """``dayfirst`` is not a question ISO-8601 input can answer differently.""" + kwargs = {} if dayfirst is None else {"dayfirst": dayfirst} + out = fd.clean(pd.DataFrame({"v": ISO}), verbose=False, **kwargs) + assert _dates(out) == ISO + + +def test_dayfirst_still_does_its_actual_job_on_ambiguous_slash_dates(): + """The fix must not disarm ``dayfirst`` where it is genuinely needed.""" + ambiguous = ["05/12/2021", "06/11/2021"] + day_first = fd.clean(pd.DataFrame({"v": ambiguous}), dayfirst=True, verbose=False) + assert _dates(day_first) == ["2021-12-05", "2021-11-06"] + + month_first = fd.clean(pd.DataFrame({"v": ambiguous}), dayfirst=False, verbose=False) + assert _dates(month_first) == ["2021-05-12", "2021-06-11"] + + +def test_the_reading_no_longer_depends_on_whether_a_day_exceeds_twelve(): + """The data-dependence that made the corruption hard to notice. + + Nineteen consecutive January dates. Before the fix these read correctly at + the default threshold but became 2021-01-01, 2021-02-01, 2021-03-01 ... at + ``datetime_threshold=0.5``, losing seven values to ``NaT``. + """ + dates = [f"2021-01-{d:02d}" for d in range(1, 20)] + + default = fd.clean(pd.DataFrame({"v": dates}), dayfirst=True, verbose=False) + lowered = fd.clean( + pd.DataFrame({"v": dates}), dayfirst=True, datetime_threshold=0.5, verbose=False + ) + + assert _dates(default) == dates + assert _dates(lowered) == dates + assert not default["v"].isna().any() + assert not lowered["v"].isna().any() + + +def test_a_column_mixing_iso_and_ambiguous_forms_reads_each_by_its_own_shape(): + mixed = ["2021-01-05", "06/11/2021", "2021-03-09"] + out = fd.clean(pd.DataFrame({"v": mixed}), dayfirst=True, verbose=False) + assert _dates(out) == ["2021-01-05", "2021-11-06", "2021-03-09"]