diff --git a/tests/test_dtypes_boolean_mutants.py b/tests/test_dtypes_boolean_mutants.py new file mode 100644 index 0000000..267b8c9 --- /dev/null +++ b/tests/test_dtypes_boolean_mutants.py @@ -0,0 +1,294 @@ +"""Type inference: the flags that decide when a column is *silently* retyped. + +Every test here pins one decision that ``steps/dtypes.py`` makes with a +boolean literal or a subset/identity comparison — the switches that say +"this text is a boolean vocabulary", "this cell matches the formatted-number +pattern", "this column contains separator noise", "this separator is a +literal, not a regex". They are the cheap-looking constants whose flip turns +a conservative *leave it as text* into a silent conversion (or the reverse), +which is exactly the class of change no assertion in ``test_dtypes.py`` +notices. + +Companion to ``test_dtypes.py``: that file covers the happy paths and the +threshold/quarantine behaviour; this one covers the boundaries those tests +step around. +""" + +import datetime as dt + +import pandas as pd +import pytest + +import freshdata as fd +from freshdata._util import sample_series +from freshdata.config import CleanConfig +from freshdata.steps.dtypes import ( + _BOOL_WORDS, + _finalize_numeric, + _number_format, + _rescue_formatted, + _to_numeric_or_none, + suggest_conversion, +) + + +def clean1(values, **options): + """Clean a single-column frame and return the resulting column.""" + return fd.clean(pd.DataFrame({"v": values}), **options)["v"] + + +def is_string(dtype) -> bool: + return pd.api.types.is_object_dtype(dtype) or isinstance(dtype, pd.StringDtype) + + +# ── boolean vocabulary: the subset test is a *subset-or-equal* test ────────── + + +def test_column_holding_the_entire_boolean_vocabulary_is_still_boolean(): + """A column whose values are *exactly* the true/false vocabulary — all + eight spellings, nothing else — is the most boolean column there is. The + detection asks ``values <= _BOOL_WORDS``; a strict ``<`` would reject the + one column that uses the whole vocabulary while accepting every subset of + it, so the conversion would depend on which spellings happen to appear. + """ + words = ["true", "t", "yes", "y", "false", "f", "no", "n"] + assert {w.casefold() for w in words} == _BOOL_WORDS # the equality case + + s = clean1(words) + assert s.dtype == bool + assert s.tolist() == [True, True, True, True, False, False, False, False] + + +def test_full_vocabulary_with_mixed_case_and_a_missing_cell_still_boolean(): + """The equality case again, through ``casefold`` and with a missing cell + so the nullable ``boolean`` dtype (not ``bool``) is the result.""" + words = ["TRUE", "T", "Yes", "Y", "False", "f", "NO", "n", None] + s = clean1(words, drop_empty_rows=False) + assert s.dtype == "boolean" + assert s.tolist()[:8] == [True, True, True, True, False, False, False, False] + assert pd.isna(s.iloc[8]) + + +# ── the decimal separator is replaced literally, never as a regex ──────────── + + +@pytest.mark.parametrize("separator", ["|", "^"]) +def test_decimal_separator_is_replaced_literally_not_as_a_regex(separator): + """``cleanup`` rewrites the locale decimal separator to ``"."`` with + ``regex=False``. The separator is caller-supplied and may be a regex + metacharacter: ``"|"`` as a pattern matches the empty string at every + position and ``"^"`` matches the start, so a regex replace would sprinkle + dots through the value instead of swapping the separator. + """ + _, _, cleanup = _number_format(CleanConfig(decimal=separator, thousands=",")) + values = [f"1{separator}5", f"2{separator}25", f"3{separator}75", f"10{separator}0"] + assert cleanup(pd.Series(values, dtype=object)).tolist() == [ + "1.5", "2.25", "3.75", "10.0", + ] + + s = clean1(values, decimal=separator) + assert s.dtype == "float64" + assert s.tolist() == [1.5, 2.25, 3.75, 10.0] + + +def test_comma_decimal_locale_is_unaffected_by_the_literal_replace(): + """The ordinary European locale keeps working: ``","`` is not a + metacharacter, so it is the metacharacter separators above that pin the + ``regex=False`` flag, not this case.""" + s = clean1(["1.234,56", "2.000,00", "3.500,75"], decimal=",", thousands=".") + assert s.dtype == "float64" + assert s.tolist() == [1234.56, 2000.0, 3500.75] + + +# ── a cell that is not readable text is neither "noise" nor a "match" ──────── + + +def test_unreadable_cells_do_not_count_as_separator_noise(): + """The formatted-number second pass is only attempted when the *sample* + actually contains a currency/separator character (``na=False``: a cell + with no text view is not evidence of noise). + + ``b"\\xff"`` is a non-UTF-8 BLOB, so the text view of the column has a + missing cell (see ``_text_view``). Counting that cell as noise starts the + formatted-number pass on a column that has no separators at all, and + ``"+ 12"`` — which plain ``to_numeric`` rejects — is then rewritten to a + number, so the BLOB is quarantined as an unparseable casualty. The column + must stay text instead. + """ + values = [b"\xff", "+ 12", "+ 34", "+ 56"] + target, converted, n_coerced = suggest_conversion( + pd.Series(values, dtype=object), CleanConfig(numeric_threshold=0.7) + ) + assert (target, converted, n_coerced) == ("none", None, 0) + + out, report = fd.clean( + pd.DataFrame({"v": values}), numeric_threshold=0.7, return_report=True, + ) + assert is_string(out["v"].dtype) + assert out["v"].tolist() == values # the BLOB is neither decoded nor nulled + assert "v" not in report.coerced_cells + + +def test_unreadable_cell_is_not_a_formatted_number_straggler(): + """``_rescue_formatted`` re-parses the values a plain ``to_numeric`` + nulled, but only those that match the formatted-number pattern. A + non-UTF-8 cell has no text view at all (``_text_view`` yields a missing + cell for it), and a missing cell is not a match — so for a column whose + only casualties are unreadable/word values there is nothing to rescue and + the parse is handed back untouched, not rewritten. + """ + formatted_re, _, cleanup = _number_format(CleanConfig()) + values = ["1", "2", "abc", b"\xff"] + + s = pd.Series(values, dtype=object) + parsed = _to_numeric_or_none(s) + # Untouched means untouched: nothing matched, so the same object comes + # back rather than a rewritten copy. + assert _rescue_formatted(s, parsed, formatted_re, cleanup) is parsed + + # Second, independent check that no write happens: with repeated index + # labels a write through ``.loc`` cannot succeed, so a rescue attempt on + # the unreadable cell would raise instead of returning the parse. + dup = pd.Series(values, index=[0, 0, 1, 1], dtype=object) + dup_parsed = _to_numeric_or_none(dup) + out = _rescue_formatted(dup, dup_parsed, formatted_re, cleanup) + assert out.tolist()[:2] == [1.0, 2.0] + assert out.isna().tolist() == [False, False, True, True] + + # Control: a real formatted straggler in the same shape *is* rescued. + with_straggler = pd.Series(["1", "2", "$1,234.56", b"\xff"], dtype=object) + rescued = _rescue_formatted( + with_straggler, _to_numeric_or_none(with_straggler), formatted_re, cleanup, + ) + assert rescued.tolist()[:3] == [1.0, 2.0, 1234.56] + + +def test_missing_cells_are_not_formatted_number_matches(): + """A missing cell has no text, so it cannot match the formatted-number + pattern (``fillna(False)``). If it did, a column where *no* value matches + would still enter the formatted-number pass, and a sample-based rejection + would be silently overturned by the presence of a missing value. + + The column below is deliberately one the cheap sample pre-screen rejects: + ``sample_size=1`` draws the single unparseable value, so the full-column + parse is never attempted. ``"1e5"`` parses as a number but never matches + the formatted-number pattern (no exponent in it), so the only "match" a + mutated fill could produce is the missing cell. + """ + s = pd.Series(["1e5"] * 19 + ["$x", None], dtype="string") + nonnull = s.dropna() + + def seed_sampling(value): + return next( + r for r in range(50) if sample_series(nonnull, 1, r).tolist() == [value] + ) + + # Sample the unparseable value: the pre-screen declines the column, and + # the missing cell must not overturn that by posing as a match. + declined = CleanConfig(sample_size=1, random_state=seed_sampling("$x")) + assert suggest_conversion(s, declined) == ("none", None, 0) + + # Control, same column: sample a parseable value and the column converts, + # so it is the sample — not the missing cell — that decides. + accepted = CleanConfig(sample_size=1, random_state=seed_sampling("1e5")) + target, converted, _ = suggest_conversion(s, accepted) + assert target == "numeric" + assert converted.tolist()[0] == 100000 + + +# ── defects found while writing the tests above: pinned, not fixed ────────── + + +def test_duplicate_index_labels_currently_break_the_formatted_number_rescue(): + """Pinned defect — current behaviour, deliberately NOT fixed here. + + ``_rescue_formatted`` writes the rescued values back with + ``parsed.loc[rescued.index] = rescued.to_numpy()``. When the frame's index + repeats a label, ``.loc`` expands that label to *every* row carrying it, so + the assignment length stops matching the values and the whole clean raises + ``ValueError`` instead of converting the column. The identical frame with a + unique index converts, which is what makes this a defect rather than a + documented limitation. + """ + values = [str(i) for i in range(19)] + ["$1,234.56"] + repeated = pd.DataFrame({"v": values}, index=list(range(10)) * 2) + with pytest.raises(ValueError, match="list-like indexer"): + fd.clean(repeated, verbose=False) + + s = clean1(values) # control: same values, unique index + assert s.dtype == "float64" + assert s.tolist()[-1] == 1234.56 + + +def test_complex_value_beside_text_currently_crashes_the_numeric_finalizer(): + """Pinned defect — current behaviour, deliberately NOT fixed here. + + ``_finalize_numeric``'s integrality check is ``nonnull % 1 == 0``, and + ``complex`` has no ``%``, so a complex value that reaches the finalizer + raises ``TypeError`` instead of the column being left alone. A column of + complex values *alone* is declined earlier (``infer_dtype`` reports + ``"complex"``, which is not a text-ish kind); one complex value beside text + makes the column ``"mixed"``, which is how it gets there. + + Asserted against ``_finalize_numeric`` directly rather than through + ``fd.clean``. The end-to-end route is **order-dependent**: the same + ``fd.clean(...)`` call raises in a fresh process but does not after certain + other work has happened in the same process, so an ``fd.clean``-level + ``pytest.raises`` here was genuinely flaky under randomised test ordering. + The unit-level behaviour is stable, and it is the defect. See + ``test_the_end_to_end_route_to_this_crash_is_order_dependent``. + """ + # Three spellings of the same failure across pandas/numpy versions: object + # dtype falls back to Python's ``%`` ("unsupported operand type(s)"), a + # numpy complex array hits the ufunc ("ufunc 'remainder' not supported"), + # and pandas 1.x raises its own ("can't mod complex numbers"). The point is + # that it raises at all. + with pytest.raises(TypeError, match="remainder|unsupported operand|mod complex"): + _finalize_numeric(pd.Series([complex(1, 2), 3], dtype=object)) + + # An all-complex column is declined before reaching the finalizer, which is + # the behaviour the mixed column should have as well. + out = fd.clean(pd.DataFrame({"v": [complex(1, 2), complex(3, 4)]}), verbose=False) + assert str(out["v"].dtype) == "complex128" + + +def test_the_end_to_end_route_to_this_crash_is_order_dependent(): + """Records an unexplained order dependence rather than hiding it. + + In a fresh process ``fd.clean`` on a complex-beside-text column raises the + ``TypeError`` above. After some other pandas work in the same process it + does not — the column stops being routed to the numeric path. Both + ``pd.api.types.infer_dtype`` (always ``"mixed"``) and ``_finalize_numeric`` + (always raises) were checked and are stable, so the divergence is upstream + of the finalizer in ``clean``'s routing. The mechanism was not identified. + + This test asserts only what is stable in either state: the frame is + returned unchanged, or the call raises ``TypeError`` — never a silently + coerced column that has lost the complex value. + """ + frame = pd.DataFrame({"v": [complex(1, 2), "abc", "3"]}) + try: + out = fd.clean(frame, verbose=False) + except TypeError as exc: + assert any(m in str(exc) for m in ("remainder", "unsupported operand", "mod complex")) + return + assert out["v"].tolist() == [complex(1, 2), "abc", "3"] + +def test_date_objects_with_a_missing_cell_still_normalize_to_datetime64(): + """``infer_dtype(s, skipna=True)`` is what makes a column of ``date`` + objects report as ``"date"`` even when a cell is missing. Inferring with + the missing cell included reports ``"mixed"``, which sends the column + through the *text* heuristics — and those look for date-shaped strings, + find none (these are ``date`` objects, not text), and leave the column as + object. One missing cell would then decide the dtype of the column. + """ + values = [dt.date(2021, 1, 5), None, dt.date(2021, 3, 9)] + s = clean1(values, drop_empty_rows=False) + assert str(s.dtype).startswith("datetime64") + assert s.isna().tolist() == [False, True, False] + assert s.tolist()[0] == pd.Timestamp("2021-01-05") + + values = [dt.datetime(2021, 1, 5, 9, 30), pd.NaT, dt.datetime(2021, 3, 9, 10, 0)] + s = clean1(values, drop_empty_rows=False) + assert str(s.dtype).startswith("datetime64") + assert s.tolist()[2] == pd.Timestamp("2021-03-09 10:00") diff --git a/tests/test_dtypes_protection_mutants.py b/tests/test_dtypes_protection_mutants.py new file mode 100644 index 0000000..ef3f414 --- /dev/null +++ b/tests/test_dtypes_protection_mutants.py @@ -0,0 +1,346 @@ +"""Protection and text-shape boundaries in the dtype step that nothing else pins. + +Three decision points in :mod:`freshdata.steps.dtypes` are reachable but +untested elsewhere in the suite: + +* ``refine_numeric_after_semantic`` — the post-semantic numeric retry. It owns + a second copy of the dtype-level leading-zero veto and its own dtype gate, + and no test calls it directly (the cleaner only reaches it when a *numeric* + semantic repair was applied earlier in the same run). +* ``_looks_dateish`` — the "nothing usable to look at" arm of the cheap date + pre-screen, which must answer *no*, not *yes*. +* ``_record_coerced`` — the day/month-ambiguity note, for a coercion casualty + that is not text at all. + +The last test proves by execution that ``_fits_int64``'s exception guard +cannot be reached from ``_finalize_numeric``. + +Everything here asserts current behaviour; nothing in ``src/`` is modified. +""" + +import datetime as dt +import inspect +import sys +from decimal import Decimal +from fractions import Fraction + +import numpy as np +import pandas as pd + +from freshdata._util import PANDAS_MAJOR +from freshdata.config import CleanConfig +from freshdata.report import CleanReport +from freshdata.steps import dtypes as dtypes_mod +from freshdata.steps.dtypes import ( + _INT64_MAX, + _INT64_MIN, + _finalize_numeric, + _fits_int64, + _looks_dateish, + fix_dtypes, + refine_numeric_after_semantic, + suggest_conversion, +) + + +def _refine(values, **options): + """Run the post-semantic numeric retry on a one-column frame.""" + df = pd.DataFrame({"v": values}) + report = CleanReport() + out = refine_numeric_after_semantic(df, ["v"], CleanConfig(**options), report) + return out["v"], report + + +# -------------------------------------------------------------------------- +# refine_numeric_after_semantic: which columns it is allowed to touch +# -------------------------------------------------------------------------- + + +def test_refine_numeric_retry_accepts_a_nullable_string_column(): + """The retry's dtype gate is object *or* StringDtype, never both at once. + + A column the semantic stage repaired can be ``string`` dtype rather than + ``object``; requiring both dtypes would silently skip every such column + (no ``StringDtype`` is also an object dtype). + """ + column, report = _refine(pd.Series(["1", "2", "3", "4", "5", "6"], dtype="string")) + assert column.dtype == "int64" + assert column.tolist() == [1, 2, 3, 4, 5, 6] + assert [a.step for a in report.actions] == ["fix_dtypes"] + + +def test_refine_numeric_retry_accepts_a_plain_object_column(): + """The same retry on the object-dtype half of that gate.""" + column, report = _refine(pd.Series(["1", "2", "3", "4", "5", "6"], dtype=object)) + assert column.dtype == "int64" + assert column.tolist() == [1, 2, 3, 4, 5, 6] + assert "after semantic repair" in report.actions[0].description + + +# -------------------------------------------------------------------------- +# refine_numeric_after_semantic: the leading-zero veto it carries itself +# -------------------------------------------------------------------------- + + +ZIPS = ["01234", "02115", "03301", "04401", "05501"] + + +def test_refine_numeric_retry_keeps_zero_padded_ids_as_text(): + """The retry re-applies the dtype-level leading-zero veto. + + Without it a zero-padded identifier column that the semantic stage touched + would be converted to ``int64`` after the first dtype pass had already + protected it, dropping the padding. + """ + column, report = _refine(pd.Series(ZIPS, dtype=object)) + assert column.tolist() == ZIPS + assert report.actions == [] + + +def test_refine_numeric_retry_honours_the_leading_zero_opt_out(): + """The veto is *gated on the flag*, not unconditional. + + Counterpart to the test above: with ``preserve_leading_zeros=False`` the + very same column must convert, which is what makes the flag meaningful. + """ + column, report = _refine(pd.Series(ZIPS, dtype=object), preserve_leading_zeros=False) + assert column.dtype == "int64" + assert column.tolist() == [1234, 2115, 3301, 4401, 5501] + assert "after semantic repair" in report.actions[0].description + + +# -------------------------------------------------------------------------- +# _looks_dateish: "nothing to look at" must mean no +# -------------------------------------------------------------------------- + + +def test_dateish_screen_rejects_a_sample_with_no_usable_text_at_all(): + """No inspectable string means "not date-shaped", not "date-shaped". + + A mixed column of ``datetime.date`` objects and an integer offers the + pre-screen nothing to read. Answering *yes* would hand the column to + ``to_datetime``, which reinterprets the bare integer as a nanosecond + epoch and fabricates a 1970 timestamp for it. + """ + values = pd.Series( + [dt.date(2021, 1, 5), dt.date(2021, 2, 6), dt.date(2021, 3, 7), 42], + dtype=object, + ) + assert pd.api.types.infer_dtype(values, skipna=True) == "mixed-integer" + assert _looks_dateish(values) is False + assert suggest_conversion(values, CleanConfig()) == ("none", None, 0) + + report = CleanReport() + out = fix_dtypes(pd.DataFrame({"v": values}), CleanConfig(), report) + assert out["v"].tolist() == values.tolist() + assert report.actions == [] + # What the screen prevents: 42 would have become + # 1970-01-01T00:00:00.000000042. Illustrative only, and pandas 1.x coerces + # the bare int to NaT instead, so it is asserted on pandas 2+ only. + if PANDAS_MAJOR >= 2: + assert pd.to_datetime(values, errors="coerce").iloc[3] == pd.Timestamp(42) + + +def test_dateish_screen_rejects_a_sample_whose_text_is_all_too_long_to_read(): + """The other way the inspectable set empties out: every value over the cap. + + Each value here is a perfectly parseable timestamp, so a *yes* would + convert the column; the screen still has to say no because it never looked + at any of them. + """ + too_long = [f"2021-01-0{i}T14:30:00.123456+00:00{' ' * 9}" for i in range(1, 6)] + assert {len(v) for v in too_long} == {41} + values = pd.Series(too_long, dtype=object) + assert _looks_dateish(values) is False + assert suggest_conversion(values, CleanConfig()) == ("none", None, 0) + # The values themselves are not the problem -- only their length is. + # ``format="mixed"`` was added in pandas 2.0, so this is asserted there. + if PANDAS_MAJOR >= 2: + assert pd.to_datetime(values, format="mixed", errors="coerce").notna().all() + + +# -------------------------------------------------------------------------- +# _record_coerced: the day/month-ambiguity note +# -------------------------------------------------------------------------- + + +def _iso_dates_plus(extra): + """19 unambiguous ISO dates plus one trailing value, as object dtype.""" + values = [f"2021-{m:02d}-{m:02d}" for m in range(1, 13)] + values += [f"2022-{m:02d}-{m:02d}" for m in range(1, 8)] + return pd.Series([*values, extra], dtype=object) + + +def test_a_non_text_casualty_is_never_reported_as_an_ambiguous_date(): + """Only *text* can carry a day/month reading. + + A ``datetime.time`` cell in a date column coerces to ``NaT`` and is + recorded as a casualty, but "pass dayfirst=True or dayfirst=False" is + useless advice for it, so the note must not fire. + """ + values = _iso_dates_plus(dt.time(12, 0)) + report = CleanReport() + out = fix_dtypes(pd.DataFrame({"v": values}), CleanConfig(), report) + + assert pd.api.types.is_datetime64_any_dtype(out["v"].dtype) + assert report.coerced_cells["v"] == {19: dt.time(12, 0)} + assert [a.description for a in report.actions] == [ + "converted to datetime64[ns] (1 unparseable value(s) set to missing)" + ] + assert not any("ambiguous" in a.description for a in report.actions) + + +def test_a_text_casualty_that_is_ambiguous_still_raises_the_note(): + """Counterpart: a genuinely ambiguous date string does raise the note.""" + values = _iso_dates_plus("01/02/2023") + report = CleanReport() + out = fix_dtypes(pd.DataFrame({"v": values}), CleanConfig(), report) + + assert pd.api.types.is_datetime64_any_dtype(out["v"].dtype) + assert report.coerced_cells["v"] == {19: "01/02/2023"} + notes = [a for a in report.actions if "ambiguous" in a.description] + assert len(notes) == 1 + assert notes[0].count == 1 + assert "dayfirst" in notes[0].rationale + + +# -------------------------------------------------------------------------- +# _fits_int64: the exception guard is unreachable from _finalize_numeric +# -------------------------------------------------------------------------- + + +def _adversarial_numeric_series(): + """Series built to break ``int(min)`` / ``int(max)`` if anything can. + + Deliberately not a pool of ordinary scalars: non-finite floats in every + float container pandas offers (numpy, ``float32``, masked ``Float64``, + object), integers far outside int64 in both directions, + ``Decimal``/``Fraction`` payloads, the unsigned extreme, and the exact + int64 bounds. + """ + inf, ninf, nan = float("inf"), float("-inf"), float("nan") + return [ + ("float64 with +inf", pd.Series([1.0, inf])), + ("float64 with -inf", pd.Series([1.0, ninf])), + ("float64 all inf", pd.Series([inf, inf])), + ("float64 inf and nan", pd.Series([inf, nan, 1.0])), + ("float32 with inf", pd.Series(np.array([1.0, np.float32("inf")], dtype="float32"))), + ("Float64 with inf", pd.Series([1.0, inf], dtype="Float64")), + ("Float64 inf only", pd.Series([inf], dtype="Float64")), + ("Float64 NA and inf", pd.Series([pd.NA, inf], dtype="Float64")), + ("object with inf", pd.Series([1, inf], dtype=object)), + ("object with -inf", pd.Series([1, ninf], dtype=object)), + ("object int beyond int64", pd.Series([2**70, 1], dtype=object)), + ("object int below int64", pd.Series([-(2**70), 1], dtype=object)), + ("object Decimal", pd.Series([Decimal("1"), Decimal("2")], dtype=object)), + ("object Fraction", pd.Series([Fraction(2, 1), Fraction(4, 1)], dtype=object)), + ("object bool", pd.Series([True, False], dtype=object)), + ("int64 exact bounds", pd.Series([_INT64_MIN, _INT64_MAX])), + ("uint64 max", pd.Series([np.iinfo(np.uint64).max], dtype="uint64")), + ("Int64 with NA", pd.Series([1, pd.NA], dtype="Int64")), + ("float64 near overflow", pd.Series([1e308, 1.0])), + ("float64 exactly 2**63", pd.Series([float(2**63)])), + ("float64 exactly -2**63", pd.Series([float(-(2**63))])), + ("empty float64", pd.Series([], dtype="float64")), + ] + + +def _fits_int64_with_flipped_guard(nonnull): + """``_fits_int64`` with its exception guard answering the opposite way.""" + try: + return int(nonnull.min()) >= _INT64_MIN and int(nonnull.max()) <= _INT64_MAX + except (OverflowError, ValueError): + return True + + +def test_int64_fit_guard_never_runs_and_cannot_change_finalize_numeric(): + """Execution proof that ``_fits_int64``'s ``except`` arm is dead code here. + + ``_finalize_numeric`` consults ``_fits_int64`` only after + ``(nonnull % 1 == 0).all()`` has held, and that is false for every + non-finite value (``inf % 1`` is ``nan``), so ``int(min)``/``int(max)`` + only ever see finite, exactly convertible numbers. + + Two independent checks, both by execution rather than argument: + + 1. a line trace over the whole adversarial pool records zero hits on the + guard's ``return`` line, while recording many hits on the function; + 2. swapping in a ``_fits_int64`` whose guard answers the opposite way + leaves every ``_finalize_numeric`` result identical. + """ + pool = _adversarial_numeric_series() + source_file = inspect.getsourcefile(_fits_int64) + lines, first_line = inspect.getsourcelines(_fits_int64) + guard_line = first_line + len(lines) - 1 + assert lines[-1].strip() in ("return False", "return True") + + hits = {} + + def tracer(frame, event, _arg): + if frame.f_code.co_filename != source_file: + return None + if event == "line": + hits[frame.f_lineno] = hits.get(frame.f_lineno, 0) + 1 + return tracer + + real, flipped, reached = {}, {}, 0 + previous = sys.gettrace() + sys.settrace(tracer) + try: + for name, series in pool: + before = sum(hits.values()) + real[name] = _finalize_numeric(series) + if sum(hits.values()) > before: + reached += 1 + finally: + sys.settrace(previous) + + original = dtypes_mod._fits_int64 + dtypes_mod._fits_int64 = _fits_int64_with_flipped_guard + try: + for name, series in pool: + flipped[name] = _finalize_numeric(series) + finally: + dtypes_mod._fits_int64 = original + + # The pool really does drive the code under test (not a vacuous proof). + assert reached >= 10, f"only {reached} of {len(pool)} series ran _finalize_numeric" + assert hits.get(guard_line, 0) == 0 + + for name, _series in pool: + assert real[name].dtype == flipped[name].dtype, name + assert real[name].astype(object).equals(flipped[name].astype(object)), name + + +def test_integral_values_are_always_exactly_convertible_to_int(): + """The property that guard unreachability rests on, checked value by value. + + For every adversarial cell: if it passes ``v % 1 == 0`` then ``int(v)`` + succeeds. Cells whose ``% 1`` raises never reach the guard either, because + ``_finalize_numeric`` evaluates that test first and does not catch it. + """ + inf, ninf, nan = float("inf"), float("-inf"), float("nan") + cells = [ + inf, ninf, nan, 0.0, -0.0, 1.0, -1.0, 1e308, -1e308, float(2**63), + float(-(2**63)), 2**70, -(2**70), np.float32("inf"), np.float64("inf"), + np.float64("nan"), np.int64(_INT64_MIN), np.int64(_INT64_MAX), + np.uint64(np.iinfo(np.uint64).max), True, False, + Decimal("1"), Decimal("1.5"), Fraction(2, 1), Fraction(1, 2), + ] + integral, convertible, non_integral = 0, 0, 0 + for cell in cells: + series = pd.Series([cell], dtype=object) + try: + is_integral = bool((series % 1 == 0).all()) + except Exception: + continue # `% 1` raised: _finalize_numeric propagates, guard unreached + if not is_integral: + non_integral += 1 + continue + integral += 1 + int(series.min()) # must not raise — this is the claim under test + int(series.max()) + convertible += 1 + assert integral >= 12 + assert non_integral >= 5 # the non-finite cells really are screened out + assert convertible == integral