diff --git a/src/freshdata/_csv_io.py b/src/freshdata/_csv_io.py index f77f8801..1704cc5b 100644 --- a/src/freshdata/_csv_io.py +++ b/src/freshdata/_csv_io.py @@ -15,6 +15,7 @@ import pandas as pd +from ._numeric import safe_to_numeric from .steps.dtypes import _has_leading_zero_ids #: Rows read by the pre-scan. Zero padding that first appears after this many rows @@ -63,6 +64,6 @@ def leading_zero_dtypes( values = sample.iloc[:, position].dropna() if values.empty or not _has_leading_zero_ids(values): continue - if pd.to_numeric(values, errors="coerce").notna().all(): + if safe_to_numeric(values, errors="coerce").notna().all(): padded[column] = str return padded diff --git a/src/freshdata/_numeric.py b/src/freshdata/_numeric.py new file mode 100644 index 00000000..4831dfc3 --- /dev/null +++ b/src/freshdata/_numeric.py @@ -0,0 +1,220 @@ +"""Crash-safe :func:`pandas.to_numeric`. + +pandas < 3 reads the exponent digits of a numeric string into a C int with no +overflow check, and it does so *before* rejecting trailing text +(pandas-dev/pandas#62617, #63089, #63167; fixed in pandas 3.0 by +pandas-dev/pandas#62741). A cell that merely starts with such a token -- e.g. +the hash-like value ``"81e3104049863b72"`` -- can segfault ``to_numeric`` and +take the whole process down, whatever ``errors=`` says. + +:func:`safe_to_numeric` keeps those cells away from the parser. Every +``to_numeric`` call on data that may hold text must go through it +(``tests/test_numeric.py`` enforces this for ``src/freshdata``). +""" + +from __future__ import annotations + +import re +from typing import Any + +import numpy as np +import pandas as pd +from pandas.api.extensions import ExtensionArray + +# The leading scientific-notation token of a cell, as pandas' C float parser +# (precise_xstrtod) reads it. Match the prefix, not the whole cell: trailing +# text is only rejected after the exponent has been accumulated. +_SCIENTIFIC_PREFIX = re.compile(r"[+-]?(?:\d+(?:\.\d*)?|\.\d+)[eE]([+-]?\d+)") +# precise_xstrtod accumulates the exponent digits in a C int (n = n * 10 + d) +# and adds n to a mantissa adjustment, with no overflow check. Leading zeros +# keep n at 0, so only significant digits matter: nine (below 10**9) cannot +# overflow even after the adjustment, ten might. Every shorter exponent -- +# including subnormal and out-of-range values such as "4.9e-324" or "1e400" -- +# is left to pandas, which parses or rejects it safely. +_MIN_UNSAFE_EXPONENT = 10**9 +# Column-level screen: an unsafe cell contains an exponent marker followed by +# at least ten digits. A false positive (e.g. leading zeros) merely routes the +# column to the exact per-cell check. +_RISKY_EXPONENT = re.compile(r"[eE][+-]?\d{10,}") +# dtype kinds pandas converts without its string parser: bool, integer, +# unsigned, float, complex, datetime and timedelta. +_PARSER_FREE_KINDS = frozenset("biufcmM") +# Stand-in for masked cells under errors="ignore": text pandas cannot parse. +_UNPARSEABLE = "x" + + +def _has_unsafe_scientific_exponent(value: object) -> bool: + """True when *value* starts with scientific notation whose exponent can + overflow the C int in pandas' parser (ten or more significant digits).""" + if isinstance(value, bytes): # pandas parses bytes cells with the same C code + value = value.decode("latin-1") + if not isinstance(value, str): + return False + match = _SCIENTIFIC_PREFIX.match(value.lstrip()) + if match is None: + return False + try: + return abs(int(match.group(1))) >= _MIN_UNSAFE_EXPONENT + except ValueError: # e.g. more digits than int() accepts from a string + return True + + +def _text_blob(cells: np.ndarray) -> str: + """All text cells joined into one string, using C-level joins when possible.""" + try: + return "\x1f".join(cells) + except TypeError: # missing values or non-text cells + pass + try: + return "\x1f".join(cells[pd.notna(cells)]) + except TypeError: # non-text objects (numbers, bytes, lists) + pass + return "\x1f".join( + v.decode("latin-1") if isinstance(v, bytes) else v + for v in cells + if isinstance(v, (str, bytes)) + ) + + +def _unsafe_cells(cells: np.ndarray) -> np.ndarray | None: + """Positional mask of unsafe cells, or ``None`` when there are none.""" + blob = _text_blob(cells) + if ("e" not in blob and "E" not in blob) or _RISKY_EXPONENT.search(blob) is None: + return None + unsafe = np.fromiter( + map(_has_unsafe_scientific_exponent, cells), dtype=bool, count=len(cells) + ) + return unsafe if unsafe.any() else None + + +def _parsed_cells(values: Any) -> np.ndarray | None: + """The 1-D cells pandas would run through its string parser, else ``None``.""" + if isinstance(values, (pd.Series, pd.Index, np.ndarray, ExtensionArray)): + if values.ndim != 1 or values.dtype.kind in _PARSER_FREE_KINDS: + return None + if isinstance(values.dtype, pd.CategoricalDtype) and _unsafe_cells( + np.asarray(values.dtype.categories, dtype=object) + ) is None: + return None + return values if isinstance(values, np.ndarray) else values.to_numpy() + if isinstance(values, (list, tuple)): + cells = np.array(values, dtype=object) + return cells if cells.ndim == 1 else None + if isinstance(values, (str, bytes)): + return np.array([values], dtype=object) + return None + + +def _categories(values: Any) -> Any: + """The ``.cat``-style accessor of a categorical Series or Categorical.""" + return values.cat if isinstance(values, pd.Series) else values + + +def _is_categorical(values: Any) -> bool: + return isinstance(getattr(values, "dtype", None), pd.CategoricalDtype) + + +def _substitute(values: Any, unsafe: np.ndarray, fill: object) -> Any: + """A copy of *values* with the unsafe cells replaced by *fill* + (``None`` means missing), keeping the container pandas dispatches on.""" + if isinstance(values, pd.Index): + return values.where(~unsafe) if fill is None else values.where(~unsafe, fill) + if isinstance(values, (pd.Series, ExtensionArray)) and fill is None: + return ( + values.mask(unsafe) + if isinstance(values, pd.Series) + else pd.Series(values, copy=False).mask(unsafe).array + ) + positions = np.flatnonzero(unsafe) + if isinstance(values, (list, tuple)): + out = np.array(values, dtype=object) + elif isinstance(values, np.ndarray): + # str/bytes arrays cannot hold None; pandas parses them as object anyway. + out = values.astype(object) if fill is None else values.copy() + elif isinstance(values, (pd.Series, ExtensionArray)): + if _is_categorical(values) and fill not in values.dtype.categories: + values = _categories(values).add_categories([fill]) + out = values.copy() + else: + return fill # scalar + if isinstance(out, pd.Series): + out.iloc[positions] = fill + else: + out[positions] = fill + return out + + +def _head(values: Any, stop: int) -> Any: + """The cells before position *stop*, in the same container.""" + if isinstance(values, pd.Series): + return values.iloc[:stop] + if isinstance(values, (str, bytes)): + return np.array([], dtype=object) + return values[:stop] + + +def _restore(result: Any, values: Any, cells: np.ndarray, unsafe: np.ndarray) -> Any: + """Put the original cells back into an ``errors="ignore"`` result.""" + if isinstance(values, (str, bytes)): + return values + positions = np.flatnonzero(unsafe) + if isinstance(result, pd.Index): + arr = result.to_numpy(dtype=object, copy=True) + arr[positions] = cells[positions] + return pd.Index(arr, dtype=result.dtype, name=result.name) + out = result.copy() + if isinstance(out, pd.Series): + out.iloc[positions] = cells[positions] + else: + out[positions] = cells[positions] + if ( + _is_categorical(out) + and _is_categorical(values) + and _UNPARSEABLE not in values.dtype.categories + ): + out = _categories(out).remove_categories([_UNPARSEABLE]) + return out + + +def safe_to_numeric(values: Any, **kwargs: Any) -> Any: + """:func:`pandas.to_numeric` that cannot crash on exponent-overflow text. + + Accepts everything ``pd.to_numeric`` does (scalar, list, tuple, 1-D array, + ``Index`` or ``Series``) and forwards every keyword unchanged + (``errors=``, ``downcast=``, ``dtype_backend=``). Input with no unsafe cell + -- including every numeric, boolean or datetime dtype -- goes to pandas + untouched, so the result is exactly pandas' result (index, name, dtype). + + A cell that starts with scientific notation whose exponent has ten or more + significant digits (enough to overflow pandas' C int) is treated as + unparseable text: it becomes missing with ``errors="coerce"``, raises + pandas' ``Unable to parse string`` ``ValueError`` with ``errors="raise"``, + and is returned as-is with ``errors="ignore"``. pandas rejects almost all + such cells too; the exception is a whole-cell negative exponent small + enough not to overflow, which pandas would underflow to 0.0. Shorter + exponents, including subnormal and out-of-range ones, are left to pandas. + + The check is cheap: text columns are screened as one joined string, and + only a column that contains an exponent marker followed by ten digits + pays for the per-cell check. + """ + cells = _parsed_cells(values) + unsafe = None if cells is None else _unsafe_cells(cells) + if cells is None or unsafe is None: + return pd.to_numeric(values, **kwargs) + errors = kwargs.get("errors", "raise") + if errors == "coerce": + return pd.to_numeric(_substitute(values, unsafe, None), **kwargs) + if errors == "raise": + first = int(np.argmax(unsafe)) + # An earlier unparseable cell raises first, exactly as pandas would. + pd.to_numeric(_head(values, first), **kwargs) + cell = cells[first] + text = cell.decode("latin-1") if isinstance(cell, bytes) else cell + raise ValueError(f'Unable to parse string "{text}" at position {first}') + # errors="ignore" (pandas itself rejects any other value). + result = pd.to_numeric(_substitute(values, unsafe, _UNPARSEABLE), **kwargs) + return _restore(result, values, cells, unsafe) + + +__all__ = ["safe_to_numeric"] diff --git a/src/freshdata/context/validate.py b/src/freshdata/context/validate.py index b7016d2b..03811435 100644 --- a/src/freshdata/context/validate.py +++ b/src/freshdata/context/validate.py @@ -167,7 +167,9 @@ def _check_allowed_values(series: pd.Series, c: ColumnConstraint) -> QualityFind def _check_range(series: pd.Series, c: ColumnConstraint) -> QualityFinding | None: import pandas as pd # noqa: PLC0415 - keep the context package import-light - numeric = pd.to_numeric(series, errors="coerce") + from .._numeric import safe_to_numeric # noqa: PLC0415 + + numeric = safe_to_numeric(series, errors="coerce") lo, hi = c.params.get("lo"), c.params.get("hi") mask = pd.Series(False, index=series.index) if lo is not None: diff --git a/src/freshdata/domains/_common.py b/src/freshdata/domains/_common.py index 88cfc360..459b1792 100644 --- a/src/freshdata/domains/_common.py +++ b/src/freshdata/domains/_common.py @@ -18,6 +18,7 @@ import pandas as pd +from .._numeric import safe_to_numeric from .base import ColumnMapping, Rule __all__ = [ @@ -197,14 +198,14 @@ def check_numeric(df: pd.DataFrame, mapping: ColumnMapping, rule: Rule) -> list[ """Flag present values that are not numeric.""" col = mapping.actual(rule.fields[0]) series = df[col] - bad = series.notna() & pd.to_numeric(series, errors="coerce").isna() + bad = series.notna() & safe_to_numeric(series, errors="coerce").isna() return df.index[bad].tolist() def check_nonneg(df: pd.DataFrame, mapping: ColumnMapping, rule: Rule) -> list[Any]: """Flag present numeric values that are negative (non-numeric is a numeric rule's job).""" col = mapping.actual(rule.fields[0]) - numeric = pd.to_numeric(df[col], errors="coerce") + numeric = safe_to_numeric(df[col], errors="coerce") return df.index[numeric.notna() & (numeric < 0)].tolist() @@ -213,7 +214,7 @@ def check_nonneg_number(df: pd.DataFrame, mapping: ColumnMapping, rule: Rule) -> col = mapping.actual(rule.fields[0]) series = df[col] present = series.notna() - numeric = pd.to_numeric(series, errors="coerce") + numeric = safe_to_numeric(series, errors="coerce") bad = (present & numeric.isna()) | (numeric.notna() & (numeric < 0)) return df.index[bad].tolist() @@ -223,7 +224,7 @@ def check_positive(df: pd.DataFrame, mapping: ColumnMapping, rule: Rule) -> list col = mapping.actual(rule.fields[0]) series = df[col] present = series.notna() - numeric = pd.to_numeric(series, errors="coerce") + numeric = safe_to_numeric(series, errors="coerce") bad = (present & numeric.isna()) | (numeric.notna() & (numeric <= 0)) return df.index[bad].tolist() @@ -233,7 +234,7 @@ def check_positive_integer(df: pd.DataFrame, mapping: ColumnMapping, rule: Rule) col = mapping.actual(rule.fields[0]) series = df[col] present = series.notna() - numeric = pd.to_numeric(series, errors="coerce") + numeric = safe_to_numeric(series, errors="coerce") is_pos_int = numeric.notna() & (numeric > 0) & (numeric == numeric.round()) return df.index[present & ~is_pos_int].tolist() diff --git a/src/freshdata/domains/agriculture/validator.py b/src/freshdata/domains/agriculture/validator.py index 93bab115..8b4dd011 100644 --- a/src/freshdata/domains/agriculture/validator.py +++ b/src/freshdata/domains/agriculture/validator.py @@ -17,6 +17,7 @@ import pandas as pd +from ..._numeric import safe_to_numeric from .._common import ( check_both_present, check_iso_date, @@ -113,7 +114,7 @@ def _check_season_year( ) -> list[Any]: series = df[mapping.actual("season_year")] present = series.notna() - numeric = pd.to_numeric(series, errors="coerce") + numeric = safe_to_numeric(series, errors="coerce") max_year = pd.Timestamp.now().year + _MAX_YEAR_OFFSET valid = ( numeric.notna() @@ -127,7 +128,7 @@ def _check_date_year_matches_season( self, df: pd.DataFrame, mapping: ColumnMapping, rule: Rule ) -> list[Any]: parsed = to_datetime_safe(df[mapping.actual("operation_date")]) - season = pd.to_numeric(df[mapping.actual("season_year")], errors="coerce") + season = safe_to_numeric(df[mapping.actual("season_year")], errors="coerce") both = parsed.notna() & season.notna() bad = both & (parsed.dt.year != season) return df.index[bad].tolist() diff --git a/src/freshdata/domains/base.py b/src/freshdata/domains/base.py index 44abcf69..27747a82 100644 --- a/src/freshdata/domains/base.py +++ b/src/freshdata/domains/base.py @@ -22,6 +22,8 @@ import pandas as pd +from .._numeric import safe_to_numeric + #: Validation layers, executed in this strict order. LAYERS: tuple[str, ...] = ("schema", "format", "reference", "business", "semantic") #: Finding severities, in increasing order of seriousness. @@ -578,7 +580,7 @@ def _check_enum(self, df: pd.DataFrame, mapping: ColumnMapping, rule: Rule) -> l def _check_range(self, df: pd.DataFrame, mapping: ColumnMapping, rule: Rule) -> list[Any]: col = mapping.actual(rule.fields[0]) - numeric = pd.to_numeric(df[col], errors="coerce") + numeric = safe_to_numeric(df[col], errors="coerce") present = df[col].notna() low = rule.params.get("min") high = rule.params.get("max") diff --git a/src/freshdata/domains/education/validator.py b/src/freshdata/domains/education/validator.py index 62a1f8ad..01b71c7d 100644 --- a/src/freshdata/domains/education/validator.py +++ b/src/freshdata/domains/education/validator.py @@ -16,6 +16,7 @@ import pandas as pd +from ..._numeric import safe_to_numeric from .._common import ( check_both_present, check_ge_date, @@ -121,7 +122,7 @@ def _check_school_year( ) -> list[Any]: series = df[mapping.actual("school_year")] present = series.notna() - numeric = pd.to_numeric(series, errors="coerce") + numeric = safe_to_numeric(series, errors="coerce") max_year = pd.Timestamp.now().year + _MAX_YEAR_OFFSET valid = ( numeric.notna() @@ -135,7 +136,7 @@ def _check_enrollment_in_year( self, df: pd.DataFrame, mapping: ColumnMapping, rule: Rule ) -> list[Any]: enroll = to_datetime_safe(df[mapping.actual("enrollment_date")]) - year = pd.to_numeric(df[mapping.actual("school_year")], errors="coerce") + year = safe_to_numeric(df[mapping.actual("school_year")], errors="coerce") rows: list[Any] = [] for idx in df.index[enroll.notna() & year.notna()]: school_year = int(year.at[idx]) diff --git a/src/freshdata/domains/energy/validator.py b/src/freshdata/domains/energy/validator.py index 2f331d38..4a5f2f09 100644 --- a/src/freshdata/domains/energy/validator.py +++ b/src/freshdata/domains/energy/validator.py @@ -20,6 +20,7 @@ import pandas as pd +from ..._numeric import safe_to_numeric from .._common import check_iso_datetime, check_not_future, check_numeric from ..base import ColumnMapping, ConfigDrivenValidator, Rule, RuleResult @@ -96,7 +97,7 @@ def _check_function_code(self, df: pd.DataFrame, mapping: ColumnMapping, if col is None: return [] allowed = set(_ref("modbus_function_codes")["codes"]) - codes = pd.to_numeric(df[col], errors="coerce") + codes = safe_to_numeric(df[col], errors="coerce") present = df[col].notna() return list(df.index[present & ~codes.isin(allowed)]) @@ -118,7 +119,7 @@ def _check_function_object(self, df: pd.DataFrame, mapping: ColumnMapping, if fc_col is None or obj_col is None: return [] klass = {int(k): v for k, v in _ref("modbus_function_codes")["register_class"].items()} - codes = pd.to_numeric(df[fc_col], errors="coerce") + codes = safe_to_numeric(df[fc_col], errors="coerce") declared = df[obj_col].astype("string").str.strip().str.casefold() bad: list[Any] = [] for idx in df.index: diff --git a/src/freshdata/domains/finance/validator.py b/src/freshdata/domains/finance/validator.py index 89cfb07e..36edad29 100644 --- a/src/freshdata/domains/finance/validator.py +++ b/src/freshdata/domains/finance/validator.py @@ -20,6 +20,7 @@ import pandas as pd +from ..._numeric import safe_to_numeric from .._common import check_iso_datetime, check_not_future from ..base import ColumnMapping, ConfigDrivenValidator, Rule, RuleResult from ..reference import load_reference @@ -40,7 +41,7 @@ def _iso4217() -> dict[str, Any]: def _to_numeric(series: pd.Series) -> pd.Series: - return pd.to_numeric(series, errors="coerce") + return safe_to_numeric(series, errors="coerce") def _parse_iso(value: Any) -> pd.Timestamp | None: diff --git a/src/freshdata/domains/media/validator.py b/src/freshdata/domains/media/validator.py index 70a5d42d..07cf885c 100644 --- a/src/freshdata/domains/media/validator.py +++ b/src/freshdata/domains/media/validator.py @@ -19,6 +19,7 @@ import pandas as pd +from ..._numeric import safe_to_numeric from .._common import ( check_fhir_date, check_iso_date, @@ -341,7 +342,7 @@ def _check_positive_int_fields( if col is None: continue series = df[col] - numeric = pd.to_numeric(series, errors="coerce") + numeric = safe_to_numeric(series, errors="coerce") is_pos_int = numeric.notna() & (numeric > 0) & (numeric == numeric.round()) rows.update(df.index[series.notna() & ~is_pos_int].tolist()) return sorted(rows, key=_sort_key) @@ -366,7 +367,7 @@ def _check_single_track_count( if release_type_col is None: return [] is_single = df[release_type_col].astype("string").str.casefold() == "single" - track_count = pd.to_numeric(df[mapping.actual("track_count")], errors="coerce") + track_count = safe_to_numeric(df[mapping.actual("track_count")], errors="coerce") bad = ( is_single.fillna(False) & track_count.notna() diff --git a/src/freshdata/domains/retail/validator.py b/src/freshdata/domains/retail/validator.py index 673a21d2..1df22d8f 100644 --- a/src/freshdata/domains/retail/validator.py +++ b/src/freshdata/domains/retail/validator.py @@ -19,6 +19,7 @@ import numpy as np import pandas as pd +from ..._numeric import safe_to_numeric from ..base import ColumnMapping, ConfigDrivenValidator, Rule, RuleResult _PACK_DIR = Path(__file__).resolve().parent @@ -171,7 +172,7 @@ def _check_nonnull_maxlen( def _check_content_uom( self, df: pd.DataFrame, mapping: ColumnMapping, rule: Rule ) -> list[Any]: - content = pd.to_numeric(df[mapping.actual("net_content")], errors="coerce") + content = safe_to_numeric(df[mapping.actual("net_content")], errors="coerce") uom = df[mapping.actual("net_content_uom")] has_uom = uom.notna() & (uom.astype("string").str.strip() != "") has_pos_content = content.notna() & (content > 0) diff --git a/src/freshdata/domains/transport/validator.py b/src/freshdata/domains/transport/validator.py index efa06535..aba15114 100644 --- a/src/freshdata/domains/transport/validator.py +++ b/src/freshdata/domains/transport/validator.py @@ -16,6 +16,7 @@ import pandas as pd +from ..._numeric import safe_to_numeric from ..base import ColumnMapping, ConfigDrivenValidator, DomainError, Rule, RuleResult, SkipCheck _PACK_DIR = Path(__file__).resolve().parent @@ -131,7 +132,7 @@ def _check_route_type(self, df: pd.DataFrame, mapping: ColumnMapping, rule: Rule col = mapping.actual("route_type") series = df[col] present = series.notna() - numeric = pd.to_numeric(series, errors="coerce") + numeric = safe_to_numeric(series, errors="coerce") allowed = {int(v) for v in rule.params.get("values", ())} bad = present & ~numeric.isin(allowed) return df.index[bad].tolist() @@ -169,7 +170,7 @@ def _check_stop_sequence_unique( each value (in file order) is kept; later ones are flagged. """ trip = df[mapping.actual("trip_id")] - seq = pd.to_numeric(df[mapping.actual("stop_sequence")], errors="coerce") + seq = safe_to_numeric(df[mapping.actual("stop_sequence")], errors="coerce") work = pd.DataFrame({"_trip": trip, "_seq": seq}, index=df.index) work = work[work["_trip"].notna() & work["_seq"].notna()] repeated = work.duplicated(subset=["_trip", "_seq"], keep="first") diff --git a/src/freshdata/fieldcheck.py b/src/freshdata/fieldcheck.py index d8592146..2370972b 100644 --- a/src/freshdata/fieldcheck.py +++ b/src/freshdata/fieldcheck.py @@ -29,6 +29,7 @@ class to an action. The default policy is non-destructive: nothing is deleted, import pandas as pd +from ._numeric import safe_to_numeric from ._util import mask_sensitive_value from .findings import QualityFinding from .semantic.experts import is_plain_number, looks_like_date_value, parse_currency @@ -588,7 +589,7 @@ def _suspect_rows(series: pd.Series, spec: FieldSpec) -> pd.Index: checkable = ~missing if spec.semantic_type in _NUMERIC_TYPES: - parsed = pd.to_numeric(strs.str.replace(",", "", regex=False), errors="coerce") + parsed = safe_to_numeric(strs.str.replace(",", "", regex=False), errors="coerce") fine = parsed.notna() lo, hi = _num_bound(spec.min_value), _num_bound(spec.max_value) if lo is not None: @@ -688,7 +689,7 @@ def _column_consensus(series: pd.Series) -> tuple[str, float] | None: def _iqr_outliers(series: pd.Series, k: float = 3.0) -> pd.Series: """Boolean mask of extreme numeric values (Tukey fences, conservative k).""" - nums = pd.to_numeric(series, errors="coerce") + nums = safe_to_numeric(series, errors="coerce") valid = nums.dropna() if len(valid) < 8: return pd.Series(False, index=series.index) diff --git a/src/freshdata/imputation/missforest.py b/src/freshdata/imputation/missforest.py index bfa0968d..ff9d9330 100644 --- a/src/freshdata/imputation/missforest.py +++ b/src/freshdata/imputation/missforest.py @@ -20,6 +20,7 @@ is_numeric_dtype, ) +from .._numeric import safe_to_numeric from .._util import add_column, exact_int_stat, exceeds_float64_exact, safe_median from ..config import CleanConfig from ..engine.context import ColumnContext @@ -138,7 +139,7 @@ def _fit_predict_column( if plan.model_type == "regressor": model = RandomForestRegressor(**self._forest_kwargs(iteration)) - y_train = pd.to_numeric(df.loc[observed, plan.column], errors="coerce") + y_train = safe_to_numeric(df.loc[observed, plan.column], errors="coerce") else: model = RandomForestClassifier(**self._forest_kwargs(iteration)) y_train = self._classification_target(df.loc[observed, plan.column], plan) @@ -287,8 +288,8 @@ def _convergence_delta( prev = previous[plan.column].loc[plan.missing_mask] cur = work[plan.column].loc[plan.missing_mask] if plan.model_type == "regressor": - prev_num = pd.to_numeric(prev, errors="coerce") - cur_num = pd.to_numeric(cur, errors="coerce") + prev_num = safe_to_numeric(prev, errors="coerce") + cur_num = safe_to_numeric(cur, errors="coerce") # na_value: nullable (masked) columns with missing cells refuse a # plain float64 conversion on pandas < 2. values = cur_num.to_numpy(dtype="float64", na_value=np.nan) diff --git a/src/freshdata/learning/extract.py b/src/freshdata/learning/extract.py index f6bdbf06..0be33d18 100644 --- a/src/freshdata/learning/extract.py +++ b/src/freshdata/learning/extract.py @@ -26,6 +26,7 @@ import pandas as pd +from .._numeric import safe_to_numeric from ..context.types import ColumnConstraint, Provenance from ..memory import CleaningMemory, learn_cleaning_memory from .privacy import mask_value @@ -159,8 +160,8 @@ def _known_imputation_strategy(messy_col: pd.Series, clean_value: object) -> str non_null = messy_col.dropna() if non_null.empty: return None - numeric = pd.to_numeric(non_null, errors="coerce").dropna() - target = pd.to_numeric(pd.Series([clean_value]), errors="coerce").iloc[0] + numeric = safe_to_numeric(non_null, errors="coerce").dropna() + target = safe_to_numeric(pd.Series([clean_value]), errors="coerce").iloc[0] if len(numeric) >= max(2, int(0.5 * len(non_null))) and pd.notna(target): for name, stat in ( ("median", float(numeric.median())), diff --git a/src/freshdata/semantic/apply.py b/src/freshdata/semantic/apply.py index b2e1e1a3..c999c191 100644 --- a/src/freshdata/semantic/apply.py +++ b/src/freshdata/semantic/apply.py @@ -10,6 +10,7 @@ import pandas as pd +from .._numeric import safe_to_numeric from .._util import mask_sensitive_value from ..config import CleanConfig from ..report import CleanReport @@ -44,7 +45,7 @@ def _maybe_downcast(series: pd.Series, *, allow_numeric: bool = True) -> pd.Seri if not allow_numeric: return series try: - return pd.to_numeric(series) + return safe_to_numeric(series) except (ValueError, TypeError): return series diff --git a/src/freshdata/semantic/consistency.py b/src/freshdata/semantic/consistency.py index 33d0cf3f..af4b42af 100644 --- a/src/freshdata/semantic/consistency.py +++ b/src/freshdata/semantic/consistency.py @@ -22,6 +22,7 @@ import numpy as np import pandas as pd +from .._numeric import safe_to_numeric from ..config import CleanConfig from ..report import CleanReport from .types import SemanticContext @@ -283,7 +284,7 @@ def _check_fahrenheit_in_celsius(df: pd.DataFrame, report: CleanReport) -> None: for col in df.columns: if not _TEMP_NAME.search(str(col)): continue - numeric = pd.to_numeric(df[col], errors="coerce") + numeric = safe_to_numeric(df[col], errors="coerce") present = np.flatnonzero(numeric.notna().to_numpy(dtype=bool)) if len(present) < 8: continue @@ -358,7 +359,7 @@ def _check_negative_amounts(df: pd.DataFrame, report: CleanReport) -> None: for col in df.columns: if not _MONEY_NAME.search(str(col)): continue - numeric = pd.to_numeric(df[col], errors="coerce") + numeric = safe_to_numeric(df[col], errors="coerce") nonnull = numeric.dropna() n = len(nonnull) if n < 8: diff --git a/src/freshdata/steps/dtypes.py b/src/freshdata/steps/dtypes.py index 61482ad5..c2121858 100644 --- a/src/freshdata/steps/dtypes.py +++ b/src/freshdata/steps/dtypes.py @@ -23,6 +23,7 @@ import pandas as pd from pandas.api.types import infer_dtype, is_datetime64_any_dtype +from .._numeric import _has_unsafe_scientific_exponent, safe_to_numeric # noqa: F401 from .._util import ( PANDAS_MAJOR, as_string_view, @@ -44,22 +45,6 @@ # digit. These are almost always identifiers (ZIP, phone, padded keys) where # coercion to int silently destroys the padding, so we keep them as text. _LEADING_ZERO = re.compile(r"^\s*[+-]?0\d") -# The leading scientific-notation token of a cell, as pandas' C float parser -# (precise_xstrtod) reads it. pandas < 3 accumulates the exponent digits in a -# C int without an overflow check, and it does so *before* rejecting trailing -# text, so a hash-like token such as "81e3104049863b72" overflows and can -# segfault ``to_numeric(errors="coerce")`` (pandas-dev/pandas#62617, fixed in -# pandas 3.0 by pandas-dev/pandas#62741). Match the prefix, not the whole cell. -_SCIENTIFIC_PREFIX = re.compile( - r"[+-]?(?:\d+(?:\.\d*)?|\.\d+)[eE]([+-]?\d+)" -) -_MAX_SAFE_EXPONENT = 308 -# Column-level screen for the guard in _to_numeric_or_none: any string that -# pandas could parse as scientific notation with an exponent past the float -# range must contain an exponent marker followed by at least three digits -# (309 is the smallest unsafe magnitude). A false positive merely routes the -# column to the exact per-cell check. -_RISKY_EXPONENT = re.compile(r"[eE][+-]?\d{3,}") def _number_format( @@ -164,48 +149,14 @@ def _try_boolean(s: pd.Series, nonnull: pd.Series) -> pd.Series | None: def _to_numeric_or_none(values: pd.Series) -> pd.Series | None: """``to_numeric`` that tolerates non-scalar cells (lists raise even with ``errors="coerce"``).""" - # pandas < 3 can segfault while parsing scientific notation whose exponent - # overflows a C int -- including when the token merely *starts* with one - # (see _SCIENTIFIC_PREFIX). Mask every cell whose leading exponent is - # outside the finite float range before handing the series to pandas. - # The result is unchanged: pandas coerces all such cells to NaN anyway - # (out-of-range exponent or trailing text), so they stay missing. - if pd.api.types.is_object_dtype(values.dtype) or pd.api.types.is_string_dtype( - values.dtype - ): - # Screen the whole column as one joined blob first: a single C-level - # join plus one regex scan, no per-cell Python work in the common - # (safe) case. The join raises TypeError when non-string, non-missing - # objects are present — treat such columns as risky and let the exact - # per-cell predicate decide. - try: - blob = "\x1f".join(values.dropna().to_numpy()) - except TypeError: - blob = None - if blob is None or _RISKY_EXPONENT.search(blob) is not None: - unsafe = values.map(_has_unsafe_scientific_exponent) - if bool(unsafe.any()): - values = values.mask(unsafe) + # safe_to_numeric masks cells whose leading exponent can overflow the C int + # in pandas < 3's parser; every other cell is parsed exactly as pandas does. try: - return pd.to_numeric(values, errors="coerce") + return safe_to_numeric(values, errors="coerce") except (TypeError, ValueError): return None -def _has_unsafe_scientific_exponent(value: object) -> bool: - if isinstance(value, bytes): # pandas parses bytes cells with the same C code - value = value.decode("latin-1") - if not isinstance(value, str): - return False - match = _SCIENTIFIC_PREFIX.match(value.lstrip()) - if match is None: - return False - try: - return abs(int(match.group(1))) > _MAX_SAFE_EXPONENT - except ValueError: - return True - - def _rescue_formatted( s: pd.Series, parsed: pd.Series, formatted_re: re.Pattern, cleanup: Callable[[pd.Series], pd.Series], diff --git a/src/freshdata/streaming/_timeseries.py b/src/freshdata/streaming/_timeseries.py index 05c8d10d..f9f9ca6a 100644 --- a/src/freshdata/streaming/_timeseries.py +++ b/src/freshdata/streaming/_timeseries.py @@ -26,6 +26,7 @@ import numpy as np import pandas as pd +from .._numeric import safe_to_numeric from .._util import mask_sensitive_value, safe_median from ..config import CleanConfig from ..engine.context import infer_role @@ -159,7 +160,7 @@ def parse_timestamps(values: pd.Series, unit: str | None = None def _as_float(s: pd.Series) -> pd.Series: """Score a column as float64; non-numeric cells (e.g. a stray string) become NaN.""" - numeric = pd.to_numeric(s, errors="coerce") + numeric = safe_to_numeric(s, errors="coerce") return pd.Series(numeric.to_numpy(dtype="float64", na_value=np.nan), index=s.index) diff --git a/tests/test_dtypes.py b/tests/test_dtypes.py index 7040c6da..fd58db85 100644 --- a/tests/test_dtypes.py +++ b/tests/test_dtypes.py @@ -219,21 +219,33 @@ def test_unsafe_scientific_exponents_are_quarantined_before_pandas_parse(): def test_unsafe_exponent_guard_handles_mixed_and_boundary_payloads(): """The vectorized candidate scan must match the per-value guard exactly: - non-strings pass through, E308 stays parseable, E309 and an unparseable - exponent are masked, and safe exponents survive.""" + non-strings pass through, every exponent that cannot overflow pandas' C int + (in range, subnormal or out of range) parses exactly as pandas parses it, + and only a ten-digit exponent is masked.""" values = pd.Series( - ["1E308", "1e309", "2.5e-309", "1e+10", b"1e999", 7, None, "1" + "0" * 40] + ["1E308", "1e309", "2.5e-309", "1e+10", b"1e999", 7, None, "1" + "0" * 40, + "1e1000000000x"] ) parsed = _to_numeric_or_none(values) assert parsed is not None + # Exponents of at most three digits: safe to hand raw pandas in-process. + expected = pd.to_numeric(values.iloc[:-1], errors="coerce") + pd.testing.assert_series_equal(parsed.iloc[:-1], expected) assert parsed.iloc[0] == 1e308 - assert pd.isna(parsed.iloc[1]) # exponent 309 > 308: masked pre-parse - assert pd.isna(parsed.iloc[2]) # -309 out of range: masked pre-parse - assert parsed.iloc[3] == 1e10 - assert pd.isna(parsed.iloc[4]) # bytes are not a string: pandas coerces to NaN - assert parsed.iloc[5] == 7 - assert pd.isna(parsed.iloc[6]) + assert parsed.iloc[2] > 0 # subnormal: kept, not masked assert parsed.iloc[7] == 1e40 + assert pd.isna(parsed.iloc[-1]) # ten exponent digits: masked pre-parse + + +def test_dtype_inference_keeps_subnormal_and_underflow_values(): + values = pd.Series(["4.9e-324", "1e-320", "1e-310", "5e-400", "2.2e-308", "1.5"]) + parsed = _to_numeric_or_none(values) + assert parsed is not None + pd.testing.assert_series_equal(parsed, pd.to_numeric(values, errors="coerce")) + assert parsed.notna().all() + s = clean1(values.tolist(), drop_duplicates=False) + assert s.dtype == "float64" + assert s.notna().all() def test_unsafe_exponent_guard_handles_stringless_object_columns(): @@ -272,7 +284,12 @@ def test_unsafe_exponent_guard_handles_nullable_string_dtype(): def _overflows_c_int_exponent(token: str) -> bool: - """Mirror pandas' parser: up to 17 exponent digits read into a C int.""" + """Mirror pandas' parser: up to 17 exponent digits read into a C int. + + This mirrors ``precise_xstrtod`` as reported in pandas-dev/pandas#62617. + If pandas ever reads more digits, a token this filter lets through could + still crash a raw ``pd.to_numeric`` call, which is why parity baselines + run in a child interpreter.""" match = _C_INT_EXPONENT.match(token) return match is not None and int(match.group(1)) > 2**31 - 1 @@ -282,7 +299,11 @@ def test_exponent_overflow_prefix_is_flagged(token): assert _has_unsafe_scientific_exponent(token) -@pytest.mark.parametrize("token", ["1e308", "1e308abc", "12e3", "a1e3104049863", "e999", "1e"]) +@pytest.mark.parametrize( + "token", + ["1e308", "1e308abc", "12e3", "a1e3104049863", "e999", "1e", + "1e309", "4.9e-324", "1e999999999", "-1e-999999999", "1e0000000001"], +) def test_in_range_or_non_leading_exponents_are_not_flagged(token): assert not _has_unsafe_scientific_exponent(token) @@ -321,38 +342,53 @@ def test_exponent_overflow_tokens_never_reach_pandas_parser(): def test_prefix_exponent_guard_matches_pandas_on_every_safe_token(): - """Parity: masking a token that only *starts* with an out-of-range exponent - changes nothing, because pandas coerces it to NaN anyway. Compared on every - token pandas can parse without overflowing (the rest would crash it).""" + """Parity: the guard changes nothing on any token pandas can parse without + overflowing its exponent accumulator. In-range, subnormal and out-of-range + values parse as pandas parses them, and the masked ten-digit exponents here + are ones pandas rejects anyway (the rest would crash it). The + raw pandas baseline runs in a child interpreter, so a token that still + crashes pandas fails this test instead of killing the whole run.""" rng = np.random.default_rng(20260915) hexdigits = np.array(list("0123456789abcdef")) tokens = ["".join(rng.choice(hexdigits, 16)) for _ in range(4000)] tokens += ["1e880f3f8de2590b", "1e309abc", "2.5e-309x", "7e123456789z", "1e308x"] tokens += ["1e308", " 1e5 ", "12e3", "e999", "1e", "1e+", "3.5", "abc", None] - whole_cell = re.compile(r"[+-]?(?:\d+(?:\.\d*)?|\.\d+)[eE]([+-]?\d+)") - - def masked_before_this_change(token: str) -> bool: - # Whole-cell out-of-range exponents were already masked (pandas 3 - # parses them to inf); this test covers only the prefix extension. - match = whole_cell.fullmatch(token.strip()) - return match is not None and abs(int(match.group(1))) > 308 - - safe = [ - t - for t in tokens - if t is None - or not (_overflows_c_int_exponent(t) or masked_before_this_change(t)) - ] - newly_masked = [ - t for t in safe if t is not None and _has_unsafe_scientific_exponent(t) - ] - assert len(newly_masked) >= 5 # the change under test is exercised - for dtype in (object, "string"): - values = pd.Series(safe, dtype=dtype) - expected = pd.to_numeric(values, errors="coerce") - parsed = _to_numeric_or_none(values) - assert parsed is not None - pd.testing.assert_series_equal(parsed, expected) + # Subnormal, underflow and out-of-range values: parsed exactly as pandas. + tokens += ["4.9e-324", "1e-320", "1e-310", "5e-400", "1e309", "1e400", "-1e400"] + tokens += ["2.2e-308", "7e123456789", "1e999999999", "1e0000000001"] + # Ten exponent digits (masked) that pandas rejects anyway. + tokens += ["1e1000000000abc", "5e2000000000x", "4e1000000000", "2.5e-1999999999 tail"] + safe = [t for t in tokens if t is None or not _overflows_c_int_exponent(t)] + masked = [t for t in safe if t is not None and _has_unsafe_scientific_exponent(t)] + assert len(masked) >= 4 # the guard is exercised + code = textwrap.dedent( + """ + import ast + import sys + + import pandas as pd + from freshdata.steps.dtypes import _to_numeric_or_none + + safe = ast.literal_eval(sys.stdin.read()) + for dtype in (object, "string"): + values = pd.Series(safe, dtype=dtype) + expected = pd.to_numeric(values, errors="coerce") + parsed = _to_numeric_or_none(values) + assert parsed is not None + pd.testing.assert_series_equal(parsed, expected) + print("ok") + """ + ) + proc = subprocess.run( + [sys.executable, "-X", "faulthandler", "-c", code], + input=repr(safe), + capture_output=True, + text=True, + timeout=300, + check=False, + ) + assert proc.returncode == 0, (proc.returncode, proc.stderr[-2000:]) + assert proc.stdout.strip().endswith("ok") def test_relative_date_words_blocked_regardless_of_case_and_whitespace(): diff --git a/tests/test_numeric.py b/tests/test_numeric.py new file mode 100644 index 00000000..b2becfe2 --- /dev/null +++ b/tests/test_numeric.py @@ -0,0 +1,431 @@ +"""safe_to_numeric: pandas parity, crash safety and call-site coverage. + +pandas < 3 can segfault in ``to_numeric`` on a cell that starts with scientific +notation whose exponent overflows a C int (pandas-dev/pandas#62617). Tests +that hand such a token to a *raw* ``pd.to_numeric`` run in a child +interpreter; in-process tests only use exponents far below the C-int limit, or +a tripwire that stops before pandas parses anything. +""" + +from __future__ import annotations + +import ast +import json +import re +import subprocess +import sys +import textwrap +import warnings +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest + +import freshdata +from freshdata._numeric import _has_unsafe_scientific_exponent, safe_to_numeric + +PANDAS_MAJOR = int(pd.__version__.split(".")[0]) + +# Each of the first four segfaulted pd.to_numeric on Linux x86_64 / pandas 2.3.3. +_CRASH_TOKENS = [ + "81e3104049863b72", + "4e492493924924", + "1e3104049863", + "1e2147483648", + " -7.5E+99999999999xyz", + ".5e-3104049863 tail", +] +# Mirrors precise_xstrtod as reported in pandas-dev/pandas#62617: up to 17 +# exponent digits accumulated in a C int. +_C_INT_EXPONENT = re.compile(r"\s*[+-]?(?:\d+(?:\.\d*)?|\.\d+)[eE][+-]?(\d{1,17})") + + +def _overflows_c_int_exponent(value: object) -> bool: + if isinstance(value, bytes): + value = value.decode("latin-1") + match = _C_INT_EXPONENT.match(value) if isinstance(value, str) else None + return match is not None and int(match.group(1)) > 2**31 - 1 + + +def _run_child(code: str, stdin: str | None = None) -> str: + proc = subprocess.run( + [sys.executable, "-X", "faulthandler", "-c", textwrap.dedent(code)], + input=stdin, + capture_output=True, + text=True, + timeout=300, + check=False, + ) + assert proc.returncode == 0, (proc.returncode, proc.stderr[-2000:]) + return proc.stdout + + +# -- parity with pandas -------------------------------------------------------- + + +def _outcome(fn, values, kwargs): + with warnings.catch_warnings(): + warnings.simplefilter("ignore", FutureWarning) # errors="ignore" on pandas >= 2.2 + try: + return "ok", fn(values, **kwargs) + except (TypeError, ValueError) as exc: + return "err", (type(exc), str(exc)) + + +def _assert_same_outcome(values, **kwargs): + kind, expected = _outcome(pd.to_numeric, values, kwargs) + got_kind, got = _outcome(safe_to_numeric, values, kwargs) + assert (got_kind, type(got)) == (kind, type(expected)), (kwargs, expected, got) + if kind == "err": + assert got == expected, (kwargs, expected, got) + elif isinstance(expected, pd.Series): + pd.testing.assert_series_equal(got, expected) + elif isinstance(expected, pd.Index): + pd.testing.assert_index_equal(got, expected, exact=True) + elif isinstance(expected, np.ndarray): # Series comparison handles pd.NA cells + assert got.dtype == expected.dtype + pd.testing.assert_series_equal(pd.Series(got), pd.Series(expected)) + elif isinstance(expected, pd.api.extensions.ExtensionArray): + pd.testing.assert_extension_array_equal(got, expected) + else: # scalar + both_missing = pd.api.types.is_scalar(got) and pd.isna(got) and pd.isna(expected) + assert both_missing or got == expected, (kwargs, expected, got) + + +_MODES = [ + {"errors": "coerce"}, + {"errors": "raise"}, + {"errors": "ignore"}, + {}, + {"errors": "coerce", "downcast": "integer"}, + {"errors": "coerce", "downcast": "float"}, + {"errors": "raise", "downcast": "unsigned"}, +] +if PANDAS_MAJOR >= 2: + _MODES.append({"errors": "coerce", "dtype_backend": "numpy_nullable"}) + + +def _edge_inputs(): + # The guarded exponents here have ten digits (so the guard masks them) but + # stay far below INT_MAX, the limit of pandas' exponent accumulator + # (pandas-dev/pandas#62617), and are positive, so pandas rejects them too. + return { + "object_mixed": pd.Series( + ["1", "2.5", "1e1000000000abc", None, np.nan, " 7 ", 4, 5.5], + dtype=object, name="v", index=list("abcdefgh"), + ), + "object_clean_ints": pd.Series(["1", "2", "3"], name="n", index=[9, 8, 7]), + "object_bad_before_unsafe": pd.Series(["abc", "1e1000000000"]), + "object_list_cell_before_unsafe": pd.Series(["1", ["x"], "1e1000000000"], dtype=object), + "string_dtype": pd.Series(["1", "1e1000000000", None, "2e3"], dtype="string"), + "string_dtype_clean": pd.Series(["1", None, "2e3"], dtype="string", name="s"), + "bytes_cells": pd.Series([b"12", "3", b"1e1000000000"], dtype=object), + "bytes_clean": pd.Series([b"12", "3"], dtype=object), + "categorical": pd.Series(["1", "1e1000000000x", "2"], dtype="category"), + "float": pd.Series([1.5, np.nan, 3.0], name="f"), + "nullable_int": pd.Series([1, None, 300], dtype="Int64"), + "all_missing": pd.Series([None, np.nan], dtype=object), + "empty": pd.Series([], dtype=object), + "list": ["1", "1e1000000000x", None], + "tuple": ("4", "5e+1000000000"), + "ndarray_object": np.array(["1", "9e1000000000", None], dtype=object), + "ndarray_unicode": np.array(["1", "9e1000000000"]), + "ndarray_float": np.array([1.0, 2.0]), + "index": pd.Index(["1", "2e1000000000", "3"], name="i"), + "string_array": pd.array(["1", "7e1000000000", None], dtype="string"), + "scalar_unsafe": "1e1000000000", + "scalar_unsafe_bytes": b"1e1000000000", + "scalar_clean": "12", + "scalar_none": None, + "scalar_float": 2.5, + } + + +@pytest.mark.parametrize("mode", _MODES, ids=lambda m: ",".join(f"{k}={v}" for k, v in m.items())) +@pytest.mark.parametrize("name", list(_edge_inputs())) +def test_matches_pandas_on_edge_inputs(name, mode): + _assert_same_outcome(_edge_inputs()[name], **mode) + + +@pytest.mark.parametrize( + ("token", "unsafe"), + [ + ("1e999999999", False), # nine exponent digits: left to pandas + ("-1e-999999999", False), + ("1e0000000001", False), # leading zeros keep pandas' accumulator at 0 + ("4.9e-324", False), + ("1e400", False), + ("1e1000000000", True), # ten digits: may overflow after the mantissa adjustment + ("1e-1000000000", True), + ("1e2147483648", True), + ("1e" + "9" * 5000, True), # beyond int()'s string limit on 3.11+: ValueError path + (b"1e1000000000", True), + ], +) +def test_unsafe_exponent_bound_is_ten_significant_digits(token, unsafe): + assert _has_unsafe_scientific_exponent(token) is unsafe + + +def test_ten_digit_negative_exponent_is_unparseable(): + # The one intended difference from pandas: it would underflow this cell to + # 0.0, but a ten-digit exponent is not proven safe, so it stays text. + parsed = safe_to_numeric(pd.Series(["1e-1000000000", "2"]), errors="coerce") + assert pd.isna(parsed.iloc[0]) and parsed.iloc[1] == 2 + + +def test_rejects_what_pandas_rejects(): + for kwargs in ({"errors": "bogus"}, {"downcast": "bogus"}): + _assert_same_outcome(pd.Series(["1", "1e1000000000"]), **kwargs) + _assert_same_outcome(np.array([["1", "1e1000000000"]], dtype=object)) # 2-D + + +def test_matches_pandas_on_random_hex_tokens(): + """4,000 seeded hash-like tokens plus boundary cases, compared in every + mode. The raw pandas baseline runs in a child interpreter: the tokens that + overflow pandas' exponent parser are filtered out here, and if that filter + ever misses one, the child crashes and this test fails instead of the run.""" + rng = np.random.default_rng(20260915) + hexdigits = np.array(list("0123456789abcdef")) + tokens = ["".join(rng.choice(hexdigits, 16)) for _ in range(4000)] + tokens += ["1e880f3f8de2590b", "1e1000000000abc", "2.5e-309x", "7e123456789z", "1e308x"] + tokens += ["1e308", " 1e5 ", "12e3", "e999", "1e", "1e+", "3.5", "-0", "abc", None] + # Valid subnormal, underflow and out-of-range values parse exactly as pandas. + tokens += ["4.9e-324", "1e-320", "1e-310", "5e-400", "1e309", "1e400", "-1e400"] + tokens += ["2.2e-308", "7e123456789", "1e999999999", "-1e-999999999", "1e0000000001"] + # Guarded (ten exponent digits, below INT_MAX) and rejected by pandas as well. + tokens += ["5e2000000000x", "3.5e-1999999999 tail", "4e1000000000", "7e+1000000000"] + safe = [t for t in tokens if not _overflows_c_int_exponent(t)] + assert sum(1 for t in safe if _has_unsafe_scientific_exponent(t)) >= 5 + assert len(safe) < len(tokens) # crash tokens were generated and held back + out = _run_child( + """ + import json + import sys + import warnings + + import numpy as np + import pandas as pd + from freshdata._numeric import safe_to_numeric + + warnings.simplefilter("ignore", FutureWarning) + tokens = json.load(sys.stdin) + numeric = [t for t in tokens if t is not None and t.strip().lstrip("-").isdigit()] + + def outcome(fn, values, **kw): + try: + return "ok", fn(values, **kw) + except (TypeError, ValueError) as exc: + return "err", (type(exc), str(exc)) + + def same(values, **kw): + kind, expected = outcome(pd.to_numeric, values, **kw) + got_kind, got = outcome(safe_to_numeric, values, **kw) + assert got_kind == kind, (kw, expected, got) + if kind == "err": + assert got == expected, (kw, expected, got) + elif isinstance(expected, pd.Series): + pd.testing.assert_series_equal(got, expected) + elif isinstance(expected, pd.Index): + pd.testing.assert_index_equal(got, expected, exact=True) + elif isinstance(expected, np.ndarray): + assert got.dtype == expected.dtype + pd.testing.assert_series_equal(pd.Series(got), pd.Series(expected)) + else: + assert type(got) is type(expected) + assert (pd.isna(got) and pd.isna(expected)) or got == expected + + index = pd.RangeIndex(10, 10 + len(tokens)) + for dtype in (object, "string"): + values = pd.Series(tokens, dtype=dtype, name="token", index=index) + for errors in ("coerce", "raise", "ignore"): + same(values, errors=errors) + same(values, errors="coerce", downcast="float") + same(pd.Series(numeric, dtype=dtype), downcast="integer") + same(pd.Index(tokens, name="token"), errors="coerce") + same(list(tokens), errors="coerce") + same(np.array(tokens, dtype=object), errors="ignore") + for token in tokens[:400] + tokens[-10:]: + for errors in ("coerce", "raise", "ignore"): + same(token, errors=errors) + print("ok") + """, + stdin=json.dumps(safe), + ) + assert out.strip().endswith("ok") + + +# -- crash tokens never reach the parser --------------------------------------- + + +@pytest.fixture +def tripwire(monkeypatch): + """Fail (instead of crashing) if an overflowing exponent reaches pandas.""" + real = pd.to_numeric + + def guarded(arg, *args, **kwargs): + cells = [arg] if np.ndim(arg) == 0 else list(np.asarray(arg, dtype=object).ravel()) + leaked = [cell for cell in cells if _overflows_c_int_exponent(cell)] + assert not leaked, f"overflowing exponent reached pandas: {leaked!r}" + return real(arg, *args, **kwargs) + + monkeypatch.setattr(pd, "to_numeric", guarded) + + +@pytest.mark.parametrize( + "container", + [ + lambda v: pd.Series(v, dtype=object, name="t"), + lambda v: pd.Series(v, dtype="string"), + lambda v: pd.Series(v, dtype="category"), + list, + tuple, + lambda v: np.array(v, dtype=object), + lambda v: pd.Index(v, dtype=object), + lambda v: pd.array(v, dtype="string"), + ], + ids=["object", "string", "category", "list", "tuple", "ndarray", "index", "string_array"], +) +def test_crash_tokens_are_kept_from_pandas(tripwire, container): + values = container(["1", *_CRASH_TOKENS, "3"]) + parsed = np.asarray(safe_to_numeric(values, errors="coerce"), dtype=float) + assert parsed[0] == 1 and parsed[-1] == 3 + assert np.isnan(parsed[1:-1]).all() + + message = r'Unable to parse string "81e3104049863b72" at position 1' + with pytest.raises(ValueError, match=message): + safe_to_numeric(values) + + with warnings.catch_warnings(): + warnings.simplefilter("ignore", FutureWarning) + ignored = safe_to_numeric(values, errors="ignore") + assert list(np.asarray(ignored, dtype=object)) == ["1", *_CRASH_TOKENS, "3"] + + +@pytest.mark.parametrize("token", [*_CRASH_TOKENS, b"81e3104049863b72"]) +def test_crash_token_scalars_are_kept_from_pandas(tripwire, token): + assert np.isnan(safe_to_numeric(token, errors="coerce")) + with pytest.raises(ValueError, match="Unable to parse string"): + safe_to_numeric(token) + with warnings.catch_warnings(): + warnings.simplefilter("ignore", FutureWarning) + assert safe_to_numeric(token, errors="ignore") is token + + +def test_migrated_public_apis_survive_crash_tokens(): + """Runs in a child interpreter: on pandas < 3 an unguarded call site can + kill the process with SIGSEGV. The tripwire makes a bypass fail on every + platform, including those where the overflow happens not to crash.""" + out = _run_child( + f""" + import os + import re + import sys + + import numpy as np + import pandas as pd + + _real = pd.to_numeric + _exp = re.compile({_C_INT_EXPONENT.pattern!r}) + + def _tripwire(arg, *args, **kwargs): + cells = [arg] if np.ndim(arg) == 0 else np.asarray(arg, dtype=object).ravel() + for cell in cells: + text = cell.decode("latin-1") if isinstance(cell, bytes) else cell + match = _exp.match(text) if isinstance(text, str) else None + if match and int(match.group(1)) > 2**31 - 1: + sys.stderr.write(f"unguarded to_numeric reached {{cell!r}}\\n") + sys.stderr.flush() + os._exit(97) + return _real(arg, *args, **kwargs) + + pd.to_numeric = _tripwire + + import freshdata as fd + from freshdata.domains import run_domain + + tokens = {_CRASH_TOKENS!r} + column = (tokens + ["12.5", "7", "3.25"]) * 3 + + report = fd.validate_fields( + pd.DataFrame({{"amount": column}}), {{"amount": "currency_amount"}} + ) + assert report is not None + + frame = pd.DataFrame({{ + "transaction_id": range(len(column)), + "debit": column, + "credit": column, + "amount": column, + }}) + _, outcome = run_domain(frame, "finance") + assert outcome is not None + + out = fd.clean(pd.DataFrame({{"token": tokens, "n": range(len(tokens))}})) + assert out["token"].notna().all() + print("ok") + """ + ) + assert out.strip().endswith("ok") + + +# -- no call site bypasses the guard ------------------------------------------- + +_PACKAGE = Path(freshdata.__file__).resolve().parent + +# Left for a follow-up PR. +_DEFERRED = frozenset({ + "enterprise/contracts.py", + "enterprise/privacy.py", + "enterprise/privacy_policy.py", + "enterprise/cleaner.py", + "enterprise/interface.py", + "learning/privacy.py", + "experimental/ai_copilot.py", + "_util.py", + "parsers/base.py", + "execution/_config.py", + "execution/backends/_duckdb.py", +}) + +# Direct calls whose argument is provably numeric: pandas never runs its +# string parser on them. Counts must match exactly, so a new call in the same +# file has to be reviewed (and normally routed through safe_to_numeric). +_NUMERIC_ONLY = { + # guarded by is_numeric_dtype(s) and not is_bool_dtype(s) + "streaming/_state.py": 1, + "streaming/_drift.py": 1, + "imputation/missforest.py": 1, + # is_integer_dtype / float64 branches of _downcast_numeric + "steps/memory.py": 2, +} + + +def _to_numeric_references(path: Path) -> int: + tree = ast.parse(path.read_text(encoding="utf-8")) + count = 0 + for node in ast.walk(tree): + name = getattr(node, "attr", None) or getattr(node, "id", None) + if isinstance(node, (ast.Attribute, ast.Name)) and name == "to_numeric": + count += 1 + elif isinstance(node, ast.ImportFrom): + count += sum(alias.name == "to_numeric" for alias in node.names) + return count + + +def test_every_to_numeric_call_goes_through_safe_to_numeric(): + found = {} + for path in sorted(_PACKAGE.rglob("*.py")): + rel = path.relative_to(_PACKAGE).as_posix() + if rel == "_numeric.py" or rel in _DEFERRED: + continue + count = _to_numeric_references(path) + if count: + found[rel] = count + unguarded = {rel: n for rel, n in found.items() if _NUMERIC_ONLY.get(rel) != n} + assert not unguarded, ( + "pd.to_numeric can segfault on pandas < 3 (pandas-dev/pandas#62617); " + f"use freshdata._numeric.safe_to_numeric instead: {unguarded}" + ) + stale = {rel for rel in _NUMERIC_ONLY if rel not in found} + assert not stale, f"update _NUMERIC_ONLY: {stale}"