From 74cbb89c9923cc9ec7c79d156133824ecf91eef2 Mon Sep 17 00:00:00 2001 From: Kevin Costner <120246174+kevincostner17@users.noreply.github.com> Date: Sun, 20 Sep 2026 21:49:53 +0530 Subject: [PATCH] test(semantic): raise experts.py mutation kill rate 44.4% -> 63.2% `semantic/experts.py` is where semantic repairs are proposed, and it killed only 111 of 250 mutants. This adds 60 tests across four themes, killing 32 of 37 targeted mutants; the other 5 are proven equivalent with executable proofs, not assertions. Expert applicability guards. `info.free_text or info.identifier_like or info.boolean_like` could become `and`, and `numeric_like and not free_text` could become `or`, with nothing failing. These are the guards that keep an expert off identifier and free-text columns, so flipping them is the "ID-protection removed" class: an expert that runs on an identifier column rewrites "007" to 7, and the damage is unrecoverable from the output alone. Date day/month disambiguation. The `a > 12 and b <= 12` family decides whether 05/12 is May 12th or 5th December. Every boundary at 12 was movable. Three of the branch-3 mutants cannot be killed: they differ only in states already claimed by the earlier branches, proven by differential over the complete input domain (180,000 inputs each, zero differences). Currency and number parsing, including the ambiguity flag for "1,000" with no currency code -- the corpus trap this parser exists to handle. Two mutants here are equivalent: one comparison is unreachable outside a branch that guarantees both operands differ, and one `return True` is dead code, confirmed by line-level trace over 610,436 calls recording zero executions. Allowed values and the category tie-break, plus the frozen-ness of _DateResolution and dropna in the value counter. Also corrects PR #491. That PR reported 98.3% for scoring.py, but its verdicts came from a harness run with a stale-bytecode defect; the true figure at that commit was 96.6%. The masked survivor was `sort_keys=True` in features_hash -- the property that makes the digest a function of feature content rather than of dict literal order. A test for it is included here, and scoring.py now genuinely measures 98.3%. Tests only; no src change. --- tests/test_experts_currency_mutants.py | 199 ++++++++++++++++ tests/test_experts_date_mutants.py | 308 +++++++++++++++++++++++++ tests/test_experts_guard_mutants.py | 209 +++++++++++++++++ tests/test_experts_value_mutants.py | 242 +++++++++++++++++++ tests/test_scoring_boundaries.py | 19 ++ 5 files changed, 977 insertions(+) create mode 100644 tests/test_experts_currency_mutants.py create mode 100644 tests/test_experts_date_mutants.py create mode 100644 tests/test_experts_guard_mutants.py create mode 100644 tests/test_experts_value_mutants.py diff --git a/tests/test_experts_currency_mutants.py b/tests/test_experts_currency_mutants.py new file mode 100644 index 0000000..e2bdb39 --- /dev/null +++ b/tests/test_experts_currency_mutants.py @@ -0,0 +1,199 @@ +"""Currency parsing: the decisions `test_currency_locale.py` does not pin. + +Mutation testing over :mod:`freshdata.semantic.experts` found that the currency +parser still passes its suite when several of its locale decisions are inverted. +The existing suite checks the *values* that well-formed amounts parse to; it +does not check the boundaries of the grouping validator, the "repeated +separator can only be grouping" rule, or the ``ambiguous`` flag that decides +whether a cell is auto-repaired or routed to a human. Each group below pins one +of those, and each fails if the corresponding decision is flipped. + +What is at stake in each group: + +* **Grouping boundaries** -- ``_valid_grouping`` is the only thing standing + between ``$1.2.3`` and a silent reading of 123. If its first-group length + check is off by one, ordinary ``$123,456.78`` stops parsing; if its + reject-branches are inverted, malformed money is coerced to a plausible + number instead of being left alone. Both directions are pinned here. +* **Digit gate** -- ``_split_amount`` refuses a body with no digits *before* + handing it to ``float()``. Without that gate ``float()`` happily accepts + ``"inf"`` and ``"nan"``, so a junk cell would become an infinite monetary + amount rather than ``None``. +* **The ambiguity flag** -- ``"1,000"`` with no currency to appeal to is either + one thousand or 1.0 with a decimal comma, and nothing in the string settles + it. ``_split_amount`` returns ``ambiguous=True`` so the caller can route the + cell to review. If that flag is flipped to ``False`` the value is silently + auto-applied at high confidence, which is exactly the thousand-fold error + FD2-002 was about -- only this time with no signal that a guess was made. + Nothing else in the suite asserts on the flag's ``True`` case. + +These call the private ``_split_amount`` / ``_valid_grouping`` deliberately: +``_split_amount``'s no-currency branch is not reachable through +``parse_currency_parts`` (see the last test), so testing it at the public API +alone cannot pin it. +""" + +from __future__ import annotations + +import pytest + +from freshdata.semantic.experts import ( + _split_amount, + _valid_grouping, + parse_currency, + parse_currency_parts, +) + +# -- the ambiguity flag ------------------------------------------------------ + + +@pytest.mark.parametrize( + ("body", "expected"), + [ + # One thousand under US convention, 1.0 under European convention. + ("1,000", (1000.0, True)), + ("1.000", (1.0, True)), + ("1,200", (1200.0, True)), + ("1.200", (1.2, True)), + ], +) +def test_a_grouped_amount_without_a_currency_code_is_flagged_ambiguous(body, expected): + """No currency and a 3-digit tail: the reading is a guess, and says so. + + The value returned is the dot-as-decimal reading, but the second element + must stay ``True`` so the caller routes the cell to a human instead of + auto-applying a reading that is a coin flip. + """ + assert _split_amount(body, None) == expected + + +def test_a_currency_code_removes_the_ambiguity_flag(): + """Once a currency is known the reading is decided, not guessed.""" + assert _split_amount("1,000", "USD") == (1000.0, False) + assert _split_amount("1.000", "USD") == (1.0, False) + # CHF writes the decimal comma, so its readings are the mirror image. + assert _split_amount("1,000", "CHF") == (1.0, False) + assert _split_amount("1.000", "CHF") == (1000.0, False) + + +def test_a_known_currency_reads_a_three_digit_tail_by_its_own_convention(): + """``$1.000`` is one dollar, not a thousand: USD writes the decimal dot.""" + assert parse_currency_parts("$1.000") == (1.0, False) + assert parse_currency_parts("$1.250") == (1.25, False) + assert parse_currency_parts("EUR 1.000") == (1000.0, False) + + +# -- grouping boundaries ----------------------------------------------------- + + +@pytest.mark.parametrize( + ("part", "sep"), + [("123,456", ","), ("999,999,999", ","), ("123.456", ".")], +) +def test_a_three_digit_first_group_is_still_valid_grouping(part, sep): + """The first group may be *up to* three digits, and three is allowed.""" + assert _valid_grouping(part, sep) is True + + +def test_an_amount_whose_first_group_is_exactly_three_digits_parses(): + """The commonest shape of all -- rejecting it would break ordinary money.""" + assert parse_currency("$123,456.78") == 123456.78 + assert parse_currency("€123.456,78") == 123456.78 + + +@pytest.mark.parametrize("part", ["1234,567", "12345,678"]) +def test_a_first_group_longer_than_three_digits_is_not_grouping(part): + assert _valid_grouping(part, ",") is False + + +def test_an_over_long_first_group_is_rejected_rather_than_coerced(): + """``$1234,567.00`` is not 1234567.00 under either convention. + + Accepting it would mean inventing a grouping that the writer did not use, + so the parser must decline and leave the cell to ordinary dtype repair. + """ + assert parse_currency("$1234,567.00") is None + + +def test_a_thousands_group_containing_a_non_digit_is_rejected(): + """``float()`` accepts underscores between digits; grouping must not. + + ``float("1_00.25")`` is 100.25, so without the ``isdigit`` check on every + group after the first, ``$1,_00.25`` would parse to 100.25 instead of being + reported unparseable. + """ + assert _valid_grouping("1,_00", ",") is False + assert parse_currency("$1,_00.25") is None + + +# -- a repeated separator can only be grouping ------------------------------- + + +@pytest.mark.parametrize( + ("text", "expected"), + [ + ("$1.234.567", 1234567.0), + ("$1.234.567,89", 1234567.89), + ("€1,234,567", 1234567.0), + ], +) +def test_a_separator_used_more_than_once_is_read_as_grouping(text, expected): + """A decimal separator appears at most once, so twice means grouping. + + This holds even when the separator is the one the currency normally uses as + a decimal point: ``$1.234.567`` is a million, not a malformed 1.234. + """ + assert parse_currency(text) == expected + + +# -- the digit gate before float() ------------------------------------------- + + +@pytest.mark.parametrize("body", ["", "inf", "nan", "-inf", "infinity", ".", ",", "-"]) +def test_a_body_with_no_digits_is_rejected_before_float_sees_it(body): + """``float("inf")`` succeeds; an amount column must not inherit that. + + Every one of these must come back ``(None, False)`` -- no value, and no + claim that an ambiguity was resolved. + """ + assert _split_amount(body, "USD") == (None, False) + assert _split_amount(body, None) == (None, False) + + +def test_a_marker_with_no_number_reports_no_value_and_no_ambiguity(): + assert parse_currency_parts("$") == (None, False) + assert parse_currency_parts("EUR") == (None, False) + + +def test_malformed_grouping_reports_no_value_and_no_ambiguity(): + """Rejection is not ambiguity: there is no reading to route for review.""" + assert parse_currency_parts("$1.2.3") == (None, False) + assert parse_currency_parts("$1,20.50") == (None, False) + assert parse_currency_parts("$1234,567.00") == (None, False) + + +def test_a_string_without_a_currency_marker_reports_no_ambiguity(): + """Not currency at all, so there is nothing for a human to adjudicate.""" + assert parse_currency_parts("1,200") == (None, False) + assert parse_currency_parts("1.200") == (None, False) + + +# -- documented, not endorsed ------------------------------------------------ + + +@pytest.mark.parametrize( + ("text", "expected"), + [("CAD 1.200", 1.2), ("AUD 1.200", 1.2), ("CNY 1.200", 1.2), ("$1.200", 1.2)], +) +def test_an_ambiguous_amount_reaching_the_public_parser_is_never_flagged(text, expected): + """Current behaviour: ``ambiguous=True`` cannot surface through the API. + + ``parse_currency_parts`` requires a currency marker, and every marker it + accepts is one ``detect_currency`` also resolves, so ``_split_amount`` is + never called with ``code=None`` from here. The "report it ambiguous rather + than guess" branch is therefore unreachable in production, and a currency + outside ``_COMMA_DECIMAL_CURRENCIES`` is guessed as dot-decimal instead -- + contrary to what the module comment on that table says. Pinned as the + behaviour that exists today; see the note filed with this suite. + """ + assert parse_currency_parts(text) == (expected, False) diff --git a/tests/test_experts_date_mutants.py b/tests/test_experts_date_mutants.py new file mode 100644 index 0000000..a174296 --- /dev/null +++ b/tests/test_experts_date_mutants.py @@ -0,0 +1,308 @@ +"""Boundary tests for the day/month disambiguation in ``semantic/experts.py``. + +Mutation testing of ``_resolve_date`` and ``looks_like_date_value`` left a +cluster of survivors on the one decision that turns ``"05/12/2024"`` into +either May 12th or the 5th of December. Nothing else in the suite noticed when +the boundary was moved, so a mutant that misreads the order writes a *wrong +date* into the user's data and reports it as an unambiguous, low-risk, +auto-appliable repair. That is the worst failure mode this library has: a +silent semantic corruption that looks like a confident fix. + +The value ``12`` is the whole crux, because it is the largest legal month: + + a > 12, b <= 12 -> the first token cannot be a month; day-first, certain + b > 12, a <= 12 -> the second token cannot be a month; month-first, certain + a > 12, b > 12 -> neither token can be a month; not a date at all + otherwise -> both <= 12, so only the ``dayfirst`` hint can decide, + and with no hint the reading stays ambiguous: a + guess is surfaced for audit at confidence 0.75 / + risk "high" so the policy gate never applies it + +Each mutant loosened one of those comparisons (``>`` -> ``>=``, ``<=`` -> ``<``, +``and`` -> ``or``), which either promotes a genuinely ambiguous value to a +confident wrong answer or demotes a certain one to a guess. The tests below pin +the behaviour at exactly 12 on both sides, and at 12 against 13. + +The second group covers ``looks_like_date_value``, the cheap shape probe that +decides whether a column is even *eligible* for date repair. Its boolean +returns and its ``or``-chain of date regexes could all be flipped unnoticed: +an ``and``-chain there would require a string to match all three mutually +exclusive patterns at once, so no value would ever look like a date and the +date expert would silently go dark on every dataset. + +Mutants killed here (ids from the repo's mutation harness): +``cmp#16``, ``cmp#18``, ``cmp#19``, ``bool#12``, ``const_bool#19``.. +``const_bool#23`` and ``const_bool#26``. ``cmp#20``, ``cmp#21`` and ``bool#16`` +are proven equivalent in the last test rather than left unexplained. + +(Re-running that harness needs ``PYTHONDONTWRITEBYTECODE=1``: every mutant of +this module unparses to within a byte or two of the same size and they are +written within the same mtime second, so CPython happily reuses the previous +mutant's cached bytecode and the reported verdicts drift between runs.) +""" + +from __future__ import annotations + +import pandas as pd +import pytest + +from freshdata.semantic.experts import ( + _NUMERIC_DATE_RE, + _resolve_date, + is_plain_number, + looks_like_date_value, +) + + +def _resolve(raw: str, dayfirst: bool | None = None): + return _resolve_date(raw, dayfirst=dayfirst, reference_date=None) + + +# -- numeric day/month order: the boundary at 12 ------------------------------ + + +def test_a_first_token_of_exactly_twelve_is_still_an_ambiguous_month(): + """``12/05`` must not be read as day 12 just because 12 is not below 13. + + Kills ``cmp#16`` (``a > 12`` -> ``a >= 12``): the mutant takes the + "first token cannot be a month" branch for a token that can, and returns + 2024-05-12 at confidence 0.95 / risk "low" -- auto-appliable, and the + wrong month. + """ + res = _resolve("12/05/2024") + assert res is not None + assert res.value == pd.Timestamp(2024, 12, 5) + assert res.confidence == 0.75 + assert res.risk == "high" + assert "ambiguous" in res.detail + + +def test_a_first_token_of_exactly_twelve_obeys_an_explicit_dayfirst_hint(): + """With a hint, ``12/05`` reads both ways -- and only the hint decides. + + Also kills ``cmp#16``: the mutant would answer 2024-05-12 for *both* + hints, ignoring ``dayfirst=False`` entirely. + """ + assert _resolve("12/05/2024", dayfirst=True).value == pd.Timestamp(2024, 5, 12) + assert _resolve("12/05/2024", dayfirst=False).value == pd.Timestamp(2024, 12, 5) + + +def test_a_second_token_of_exactly_twelve_is_still_an_ambiguous_day(): + """``05/12`` must not be read as month 5 just because 12 is not below 13. + + Kills ``cmp#18`` (``b > 12`` -> ``b >= 12``): the mutant takes the + "second token cannot be a month" branch and reports the US reading as + certain, so a European ``5 December`` column is silently rewritten to + May 12th at risk "low". + """ + res = _resolve("05/12/2024") + assert res is not None + assert res.value == pd.Timestamp(2024, 5, 12) + assert res.confidence == 0.75 + assert res.risk == "high" + assert "ambiguous" in res.detail + + +def test_a_second_token_of_exactly_twelve_obeys_an_explicit_dayfirst_hint(): + """Also kills ``cmp#18``: under the mutant ``dayfirst=True`` is ignored.""" + assert _resolve("05/12/2024", dayfirst=True).value == pd.Timestamp(2024, 12, 5) + assert _resolve("05/12/2024", dayfirst=False).value == pd.Timestamp(2024, 5, 12) + + +def test_both_tokens_exactly_twelve_stay_ambiguous_without_a_hint(): + """``12/12`` is the fixed point of the boundary: neither token is excluded. + + The two readings happen to coincide, but the *decision* must still be the + ambiguous one, because the rule is about which token is the month. + """ + res = _resolve("12/12/2024") + assert res is not None + assert res.value == pd.Timestamp(2024, 12, 12) + assert res.confidence == 0.75 + assert res.risk == "high" + + +def test_a_first_token_of_exactly_twelve_resolves_when_the_second_exceeds_twelve(): + """``12/13`` is certain: 13 cannot be a month, so 12 is the month. + + Kills ``cmp#19`` (``a <= 12`` -> ``a < 12``): the mutant refuses the + month-first branch for a first token of exactly 12, drops through to the + no-hint fallback and downgrades a *certain* date to a 0.75/"high" guess, + so a perfectly resolvable column stops being repaired. + """ + res = _resolve("12/13/2024") + assert res is not None + assert res.value == pd.Timestamp(2024, 12, 13) + assert res.confidence == 0.95 + assert res.risk == "low" + assert "unambiguously" in res.detail + + +@pytest.mark.parametrize("raw", ["12/13/2024", "12/31/2024", "12-25-2024"]) +def test_a_first_token_of_twelve_against_a_larger_second_ignores_the_dayfirst_hint(raw): + """Evidence beats convention: 13/31/25 cannot be a month under any hint. + + Also kills ``cmp#19``: under the mutant these fall through to the hint + branches, where ``dayfirst=True`` reads them as day 12 of month 13/31/25 + -- an impossible date, silently returned as "not a date" (``None``). + """ + expected = _resolve(raw).value + assert expected is not None + assert _resolve(raw, dayfirst=True).value == expected + assert _resolve(raw, dayfirst=False).value == expected + + +def test_a_second_token_of_exactly_twelve_resolves_when_the_first_exceeds_twelve(): + """``13/12`` is the mirror case: 13 cannot be a month, so 12 is.""" + res = _resolve("13/12/2024") + assert res is not None + assert res.value == pd.Timestamp(2024, 12, 13) + assert res.confidence == 0.95 + assert res.risk == "low" + assert "unambiguously" in res.detail + + +def test_two_tokens_that_both_exceed_twelve_are_not_a_date(): + """``13/13`` has no month at all, so the value is left alone entirely.""" + assert _resolve("13/13/2024") is None + assert _resolve("13/13/2024", dayfirst=True) is None + assert _resolve("31/31/2024", dayfirst=False) is None + + +def test_an_explicit_dayfirst_false_resolves_an_otherwise_ambiguous_value(): + """``dayfirst=False`` is a real branch, not a synonym for "no hint". + + Kills ``const_bool#26`` (``dayfirst is False`` -> ``dayfirst is True``): + the mutant's condition can never hold (an earlier branch already claimed + ``dayfirst is True``), so an explicit month-first hint silently decays + into the ambiguous fallback. The guessed *value* is identical there, which + is exactly why no existing test caught it -- only the confidence and risk + change, and with them whether the repair is ever applied. + """ + res = _resolve("05/06/2024", dayfirst=False) + assert res is not None + assert res.value == pd.Timestamp(2024, 5, 6) + assert res.confidence == 0.95 + assert res.risk == "low" + assert "unambiguously" in res.detail + + hinted_the_other_way = _resolve("05/06/2024", dayfirst=True) + assert hinted_the_other_way.value == pd.Timestamp(2024, 6, 5) + assert hinted_the_other_way.confidence == 0.95 + + no_hint = _resolve("05/06/2024") + assert no_hint.value == pd.Timestamp(2024, 5, 6) # same guess, but only a guess + assert no_hint.confidence == 0.75 + assert no_hint.risk == "high" + + +# -- the eligibility probe ---------------------------------------------------- + + +@pytest.mark.parametrize("value", [None, 12, 20240512, 3.5, True, ["2024-05-12"]]) +def test_a_non_string_never_looks_like_a_date_value(value): + """Kills ``const_bool#20``: the mutant calls every non-string a date, + so an integer column of ids becomes eligible for date repair.""" + assert looks_like_date_value(value) is False + + +@pytest.mark.parametrize("text", ["", " ", "\t\n", " "]) +def test_a_blank_string_never_looks_like_a_date_value(text): + """Kills ``const_bool#21``: the mutant makes empty cells look like dates.""" + assert looks_like_date_value(text) is False + + +@pytest.mark.parametrize("text", ["today", "Yesterday", " TOMORROW ", "Today"]) +def test_relative_date_phrases_look_like_date_values(text): + """Kills ``const_bool#22``: the mutant hides ``today``/``yesterday`` from + the eligibility probe, so columns full of relative phrases -- the very + values that most need review -- are never even considered.""" + assert looks_like_date_value(text) is True + + +@pytest.mark.parametrize( + "text", + [ + "2024-05-12", # only _ISO_DATE_RE matches + "2024/05/12", # only _ISO_SLASH_DATE_RE matches + "05/12/2024", # only _NUMERIC_DATE_RE matches + "5-12-2024", # only _NUMERIC_DATE_RE matches, dash form + ], +) +def test_each_numeric_date_shape_looks_like_a_date_value_on_its_own(text): + """Kills ``bool#12`` and ``const_bool#23``. + + The three patterns are mutually exclusive, so the ``and``-chain mutant + (``bool#12``) can never be satisfied by any string, and flipping the + ``return True`` (``const_bool#23``) has the same effect: every numeric + date shape stops looking like a date and the date expert goes dark. + """ + assert looks_like_date_value(text) is True + + +@pytest.mark.parametrize("text", ["May 12, 2024", "12 May 2024", "12 Sept. 2024"]) +def test_month_name_dates_look_like_date_values(text): + assert looks_like_date_value(text) is True + + +@pytest.mark.parametrize("text", ["hello", "12", "2024", "SKU-30", "not a date"]) +def test_non_date_strings_do_not_look_like_date_values(text): + assert looks_like_date_value(text) is False + + +@pytest.mark.parametrize("value", [None, [1], {"a": 1}, object(), pd.Timestamp("2024-01-01")]) +def test_values_of_an_unhandled_type_are_not_plain_numbers(value): + """Kills ``const_bool#19``: the fall-through of ``is_plain_number`` must be + ``False``. The mutant declares every unhandled type -- ``None``, lists, + timestamps -- an ordinary number, which mis-routes those columns to the + numeric experts.""" + assert is_plain_number(value) is False + + +# -- equivalence proof -------------------------------------------------------- + + +def test_the_neither_token_is_a_month_branch_cannot_be_loosened(): + """Proof that three surviving mutants are equivalent, not missing tests. + + ``_resolve_date`` decides the numeric order with:: + + if a > 12 and b <= 12: # (1) day-first, certain + elif b > 12 and a <= 12: # (2) month-first, certain + elif a > 12 and b > 12: # (3) neither token can be a month + + Three mutants of branch (3) survive every test, and none can be killed: + + * ``cmp#20`` ``a >= 12 and b > 12`` + * ``cmp#21`` ``a > 12 and b >= 12`` + * ``bool#16`` ``a > 12 or b > 12`` + + Branch (3) is only ever evaluated when (1) and (2) are both false, and on + that reachable set all four predicates coincide: + + * ``cmp#20`` differs from the original only at ``a == 12 and b > 12`` -- + but that state satisfies (2) (``b > 12 and 12 <= 12``), so (3) is never + reached there; + * ``cmp#21`` differs only at ``a > 12 and b == 12`` -- that state + satisfies (1) (``a > 12 and 12 <= 12``); + * ``bool#16`` differs only when exactly one token exceeds 12 -- those are + precisely the states claimed by (1) and (2). + + ``_NUMERIC_DATE_RE`` captures ``\\d{1,2}`` for both tokens, so the entire + input domain is ``0..99`` squared and the argument is checked by + exhaustion below rather than only argued in prose. + """ + assert _NUMERIC_DATE_RE.match("100/05/2024") is None # domain really is 0..99 + assert _NUMERIC_DATE_RE.match("99/99/2024") is not None + + reached = 0 + for a in range(100): + for b in range(100): + if (a > 12 and b <= 12) or (b > 12 and a <= 12): + continue # branch (1) or (2) returns first + reached += 1 + original = a > 12 and b > 12 + assert (a >= 12 and b > 12) == original, (a, b) # cmp#20 + assert (a > 12 and b >= 12) == original, (a, b) # cmp#21 + assert (a > 12 or b > 12) == original, (a, b) # bool#16 + assert reached == 13 * 13 + 87 * 87 # both <= 12, or both > 12 diff --git a/tests/test_experts_guard_mutants.py b/tests/test_experts_guard_mutants.py new file mode 100644 index 0000000..ef9e4d3 --- /dev/null +++ b/tests/test_experts_guard_mutants.py @@ -0,0 +1,209 @@ +"""The `applies()` guards that decide whether a semantic expert runs at all. + +Mutation testing of `freshdata.semantic.experts` left the *entry* guards of the +value experts uncovered: the `and` that pairs a shape flag with `not +free_text`, the `or` chain that vetoes free-text/identifier/boolean columns, +and the `return False` early exits those guards use. Flipping any of them is +the "identifier-protection removed" mutation class, and it is the most +dangerous one in the library: a value expert that runs on an identifier column +will happily rewrite `"007"` to `7` and `"0001"` to `1`, silently destroying +leading zeros in customer ids, SKUs, ZIP codes and account numbers. The damage +is unrecoverable from the output alone -- nothing downstream can tell whether +`7` was ever `"007"`. + +The identifier carve-out has five independent protection layers; these guards +are one of them, and a layer that is untested is a layer that is not there. +Each test below kills a specific confirmed survivor and names it, so that if +an assertion is ever weakened the reason it existed is on the page. + +Sibling suites: `test_policy_guard_mutants.py` (the policy gate and the +byte-identity guard) and `test_guard_protected.py` (end-to-end protection). +""" + +from __future__ import annotations + +import pandas as pd +import pytest + +from freshdata.semantic.experts import ( + CategorySynonymExpert, + CurrencyStringExpert, + DatePhraseExpert, + IdentifierProtectionExpert, + SpelledNumberExpert, + looks_like_identifier_value, +) +from freshdata.semantic.types import SemanticColumnInfo + + +def _info(**kw) -> SemanticColumnInfo: + """A column info with every shape flag off; override only what matters.""" + base = { + "name": "cust_id", + "role": "categorical", + "n_nonnull": 3, + "nunique": 3, + "high_cardinality": False, + "preserve": False, + "free_text": False, + "numeric_like": False, + "boolean_like": False, + "money_like": False, + "unit_like": False, + "identifier_like": False, + } + base.update(kw) + return SemanticColumnInfo(**base) + + +# -- a shape flag is necessary, never merely sufficient-by-absence ---------- + + +def test_the_spelled_number_expert_stays_off_a_column_that_is_not_numeric_like(): + """Kills experts bool#20 (`numeric_like and not free_text` -> `or not`). + + With `or`, every column that merely fails to be free text -- which is most + columns, including every identifier column -- satisfied the guard, and the + number-word rewriter ran on ids, SKUs and postcodes. + """ + assert SpelledNumberExpert().applies(_info(role="text")) is False + assert SpelledNumberExpert().applies(_info(identifier_like=True)) is False + + +def test_the_spelled_number_expert_runs_on_a_plain_numeric_column(): + """The guard must still let the expert do its job.""" + assert SpelledNumberExpert().applies(_info(numeric_like=True)) is True + + +def test_the_spelled_number_expert_stays_off_free_text_even_when_numeric_like(): + assert SpelledNumberExpert().applies(_info(numeric_like=True, free_text=True)) is False + + +def test_the_currency_expert_stays_off_a_column_that_is_not_money_like(): + """Kills experts bool#23 (`money_like and not free_text` -> `or not`). + + Same failure shape as bool#20: `or` turned "not free text" into a licence + to strip currency formatting from any column, identifiers included. + """ + assert CurrencyStringExpert().applies(_info(role="text")) is False + assert CurrencyStringExpert().applies(_info(identifier_like=True)) is False + + +def test_the_currency_expert_runs_on_a_plain_money_column(): + assert CurrencyStringExpert().applies(_info(money_like=True)) is True + + +def test_the_currency_expert_stays_off_free_text_even_when_money_like(): + assert CurrencyStringExpert().applies(_info(money_like=True, free_text=True)) is False + + +# -- the category veto needs ONE reason, not all three ---------------------- + + +@pytest.mark.parametrize("flag", ["free_text", "identifier_like", "boolean_like"]) +def test_the_category_expert_is_vetoed_by_any_single_disqualifying_flag(flag): + """Kills experts bool#28 (`free_text or identifier_like or boolean_like` + -> all `and`) and const_bool#32 (that branch's `return False` -> `True`). + + Under `and` the veto fired only for a column that was free text *and* an + identifier *and* boolean -- i.e. never -- so the category normalizer ran on + identifier columns and folded `"007"`/`"07"` case-and-whitespace variants + together. Under the constant flip the veto branch *admitted* the column it + had just disqualified. Each flag on its own must stop the expert. + """ + assert CategorySynonymExpert().applies(_info(**{flag: True})) is False + + +def test_the_category_expert_runs_on_an_ordinary_categorical_column(): + """The veto must not be the whole guard: a clean categorical still applies.""" + assert CategorySynonymExpert().applies(_info(role="categorical")) is True + + +def test_the_category_expert_defers_to_the_reference_expert_when_allowed_values_exist(): + """Kills experts const_bool#33 (the `allowed_values` `return False` -> + `True`). + + An explicit reference list is handled by `ReferenceExpert`, which has fuzzy + matching and ambiguity handling. Flipping this exit made both experts + propose on the same values, producing duplicate and potentially conflicting + repairs for one cell. + """ + assert CategorySynonymExpert().applies(_info(allowed_values=("active", "inactive"))) is False + + +# -- the date expert is vetoed by identifier-likeness on its own ------------ + + +def test_the_date_expert_stays_off_an_identifier_column_that_also_reads_as_a_date(): + """Kills experts bool#33 (`date_like and not free_text and not + identifier_like` -> all `or`). + + Under `or` the guard was satisfied by *not* being an identifier, or by not + being free text, so it passed for essentially every column -- and an + identifier column of numeric codes that happens to parse as dates would be + converted to timestamps outright. Date-shaped identifiers are common + (`"20240115"`-style batch codes), which is why this veto exists. + """ + assert DatePhraseExpert().applies(_info(date_like=True, identifier_like=True)) is False + assert DatePhraseExpert().applies(_info(date_like=True, free_text=True)) is False + assert DatePhraseExpert().applies(_info(identifier_like=True)) is False + + +def test_the_date_expert_needs_the_column_to_actually_read_as_a_date(): + """The `or` mutant also let a column with no date evidence through.""" + assert DatePhraseExpert().applies(_info()) is False + + +def test_the_date_expert_runs_on_a_clean_date_column(): + assert DatePhraseExpert().applies(_info(date_like=True)) is True + + +# -- the leading-zero identifier test --------------------------------------- + + +def test_a_single_zero_is_not_an_identifier_but_a_zero_padded_number_is(): + """Kills experts cmp#22 (`len(s) > 1` -> `len(s) >= 1`). + + The length check is what makes this the *leading-zero* test: a zero is only + a pad when something follows it. With `>=` the bare string `"0"` -- an + ordinary value in a count, flag or score column -- was classified as an + identifier, so the spelled-number and category experts silently skipped it + while repairing every value around it, leaving the column half-normalized. + `"0"` is the only input where the two versions differ. + """ + assert looks_like_identifier_value("0") is False + assert looks_like_identifier_value("00") is True + assert looks_like_identifier_value("007") is True + assert looks_like_identifier_value("7") is False + + +def test_the_leading_zero_test_ignores_surrounding_whitespace(): + """The value is stripped first, so padding cannot smuggle a code through.""" + assert looks_like_identifier_value(" 007 ") is True + assert looks_like_identifier_value(" 0 ") is False + + +# -- the protective veto record is only emitted when it is earned ----------- + + +def test_the_identifier_veto_records_nothing_for_values_no_expert_would_touch(): + """Kills experts const_bool#37 (`transformable`'s final `return False` -> + `True`). + + `IdentifierProtectionExpert` exists to leave an audit entry when protection + actually did something. With the flip every non-string value counted as + transformable, so a plain integer id column produced a protective record + describing a veto that never had anything to veto -- audit noise that makes + the real protective records harder to find. + """ + info = _info(identifier_like=True) + assert IdentifierProtectionExpert().propose(pd.Series([1, 2, 3]), info) == [] + + +def test_the_identifier_veto_records_a_skip_when_a_value_expert_would_have_struck(): + """The counterpart: zero-padded codes are exactly what protection is for.""" + info = _info(identifier_like=True) + proposals = IdentifierProtectionExpert().propose(pd.Series(["007", "008"]), info) + assert len(proposals) == 1 + assert proposals[0].issue_type == "identifier_like" + assert proposals[0].proposed_value is None # a no-op on the data by design diff --git a/tests/test_experts_value_mutants.py b/tests/test_experts_value_mutants.py new file mode 100644 index 0000000..0413bea --- /dev/null +++ b/tests/test_experts_value_mutants.py @@ -0,0 +1,242 @@ +"""Value-expert properties that the suite asserted nowhere. + +Mutation testing ``freshdata.semantic.experts`` left four survivors in the +value-repair path. Each one is a silent correctness hazard: + +* ``cmp#26`` -- the ``canon`` tie-break in :class:`CategorySynonymExpert` + (``count > best[1]`` -> ``count >= best[1]``). It picks which surface + spelling of a casefold key becomes the canonical one. Flipping the operator + flips the *direction* of every tied normalization (``'usa' -> 'USA'`` becomes + ``'USA' -> 'usa'``), which rewrites user data whenever the confidence gate + lets the proposal auto-apply. Nothing asserted the direction. + +* ``cmp#27`` -- ``allowed[key] != value`` -> ``== value``. The inverted form + emits a no-op "repair" for every value that already *is* its allowed value, + and stops repairing the values that actually deviate. Nothing asserted that + an already-conforming value is left alone. + +* ``const_bool#24`` -- :class:`_DateResolution` is ``@dataclass(frozen=True)``. + It is the audit record for one date decision (value, confidence, risk, + detail); immutability is what keeps a resolution from being edited after the + policy gate has scored it. Nothing asserted the frozen-ness. + +* ``const_bool#31`` -- ``_value_counts`` uses ``value_counts(dropna=True)``. + Counting missing cells as if they were a distinct category would inflate a + column's apparent dominant value and feed NaN into experts that assume + ``isinstance(v, str)``. Nothing asserted that nulls are excluded. + +The tie-break tests double as characterization of a suspected defect: the +winner of a tie is decided by *row order*, not by the data. See +``test_tied_spellings_make_the_first_appearing_row_win_the_canonical_form``. +""" + +from __future__ import annotations + +import dataclasses + +import numpy as np +import pandas as pd +import pytest + +import freshdata as fd +from freshdata.semantic.experts import ( + CategorySynonymExpert, + _DateResolution, + _value_counts, +) +from freshdata.semantic.types import SemanticColumnInfo + + +def _info(**kw) -> SemanticColumnInfo: + base = { + "name": "country", + "role": "categorical", + "n_nonnull": 6, + "nunique": 2, + "high_cardinality": False, + "preserve": False, + "free_text": False, + "numeric_like": False, + "boolean_like": False, + "money_like": False, + "unit_like": False, + "identifier_like": False, + } + base.update(kw) + return SemanticColumnInfo(**base) + + +def _pairs(proposals) -> list[tuple[object, object]]: + return [(p.raw_value, p.proposed_value) for p in proposals] + + +# --- cmp#26: the canonical-spelling tie-break ------------------------------- # +def test_the_more_frequent_spelling_wins_the_canonical_form_outright(): + """A strict majority must decide the canonical form, not the tie-break. + + Guards the ``count > best[1]`` comparison from degenerating into an + order-only rule: ``'USA'`` appears once and ``'usa'`` three times, so the + proposal must run towards ``'usa'`` even though ``'USA'`` is the first row. + """ + series = pd.Series(["USA", "usa", "usa", "usa"]) + proposals = CategorySynonymExpert().propose(series, _info(n_nonnull=4)) + assert _pairs(proposals) == [("USA", "usa")] + + +def test_tied_spellings_make_the_first_appearing_row_win_the_canonical_form(): + """Characterizes the tie-break: on an exact tie the first row wins. + + ``count > best[1]`` keeps the first spelling that ``value_counts`` yields, + and pandas breaks a count tie by order of first appearance. So a 3-vs-3 + ``'USA'``/``'usa'`` column normalizes towards whichever spelling the file + happened to list first. + + This is a **suspected defect**, not a property worth having: the two frames + below hold the same multiset of values and differ only in row order, yet + the expert proposes opposite repairs. The test pins the behaviour so the + mutant (``count >= best[1]``, which makes the *last* spelling win) dies and + so any deliberate fix has to come past this assertion. + """ + expert = CategorySynonymExpert() + upper_first = pd.Series(["USA", "USA", "USA", "usa", "usa", "usa"]) + lower_first = pd.Series(["usa", "usa", "usa", "USA", "USA", "USA"]) + + assert _pairs(expert.propose(upper_first, _info())) == [("usa", "USA")] + assert _pairs(expert.propose(lower_first, _info())) == [("USA", "usa")] + + +def test_row_order_alone_flips_an_auto_applied_tied_normalization(): + """The tie-break reaches user data once the confidence gate opens. + + The tied case scores 0.93/medium, below the 0.95 default, so by default it + only ever surfaces as a *suggestion* whose direction flips with row order. + Lowering ``semantic_auto_threshold`` to 0.90 -- a documented knob -- makes + the same proposal auto-apply, and two row permutations of one column then + produce two different cleaned frames. Recorded as the public-API evidence + for the defect described above. + """ + upper_first = pd.DataFrame({"country": ["USA", "USA", "USA", "usa", "usa", "usa"]}) + lower_first = pd.DataFrame({"country": ["usa", "usa", "usa", "USA", "USA", "USA"]}) + kwargs = { + "semantic_mode": "auto", + "semantic_auto_threshold": 0.90, + "return_report": True, + "verbose": False, + } + + out_upper, report_upper = fd.clean(upper_first, **kwargs) + out_lower, report_lower = fd.clean(lower_first, **kwargs) + + assert list(out_upper["country"]) == ["USA"] * 6 + assert list(out_lower["country"]) == ["usa"] * 6 + statuses = {a.status for r in (report_upper, report_lower) for a in r if a.step == "semantic"} + assert statuses == {"automatic"} + + +def test_a_tied_normalization_is_only_suggested_under_the_default_threshold(): + """At stock settings the tie never rewrites data -- it is reported instead.""" + df = pd.DataFrame({"country": ["USA", "USA", "USA", "usa", "usa", "usa"]}) + out, report = fd.clean(df, semantic_mode="auto", return_report=True, verbose=False) + + assert list(out["country"]) == ["USA", "USA", "USA", "usa", "usa", "usa"] + semantic = [a for a in report if a.step == "semantic"] + assert [a.status for a in semantic] == ["suggested"] + assert semantic[0].metadata["raw_value"] == "usa" + assert semantic[0].metadata["proposed_value"] == "USA" + + +# --- cmp#27: the allowed-values check --------------------------------------- # +def test_a_value_that_already_equals_its_allowed_value_is_left_alone(): + """No proposal may be emitted for a value already in canonical form. + + ``allowed[key] != value`` is what suppresses the no-op; inverting it to + ``== value`` would emit ``'Red' -> 'Red'`` repairs for clean data. + """ + series = pd.Series(["Red", "Red", "Blue", "Blue"]) + info = _info(name="colour", n_nonnull=4, allowed_values=("Red", "Blue")) + assert CategorySynonymExpert().propose(series, info) == [] + + +def test_a_deviating_value_is_mapped_onto_its_allowed_value(): + """The other half of the same branch: deviations *are* repaired, at 0.96.""" + series = pd.Series(["red", "red", "Blue", "Blue"]) + info = _info(name="colour", n_nonnull=4, allowed_values=("Red", "Blue")) + proposals = CategorySynonymExpert().propose(series, info) + + assert _pairs(proposals) == [("red", "Red")] + assert proposals[0].confidence == pytest.approx(0.96) + assert any(e.kind == "context_hint" for e in proposals[0].evidence) + + +def test_an_allowed_values_column_is_routed_away_from_the_synonym_expert(): + """``applies`` hands explicit reference lists to ReferenceExpert instead. + + Documents why the two tests above must call ``propose`` directly: the + allowed-values branch is unreachable through this expert's own gate. + """ + expert = CategorySynonymExpert() + assert expert.applies(_info(allowed_values=("Red", "Blue"))) is False + assert expert.applies(_info()) is True + + +# --- const_bool#24: the date-resolution audit record ------------------------ # +def test_a_date_resolution_record_cannot_be_edited_after_construction(): + """``_DateResolution`` is the audit trail for one date decision. + + If it were mutable, a downstream caller could lower a risk or raise a + confidence after the fact and the report would no longer describe what was + actually decided. + """ + resolution = _DateResolution(None, 0.50, "high", "no reference_date supplied") + + assert dataclasses.is_dataclass(resolution) + assert dataclasses.fields(_DateResolution) # guards against an empty record + for field, new_value in ( + ("value", pd.Timestamp("2024-01-01")), + ("confidence", 0.99), + ("risk", "low"), + ("detail", "rewritten"), + ): + with pytest.raises(dataclasses.FrozenInstanceError): + setattr(resolution, field, new_value) + + assert resolution.value is None + assert resolution.confidence == pytest.approx(0.50) + assert resolution.risk == "high" + + +# --- const_bool#31: missing values are not a category ----------------------- # +def test_value_counts_excludes_missing_cells_from_the_distinct_table(): + """Nulls must never become a counted category. + + ``dropna=False`` would give NaN its own row, so a column that is mostly + empty would look like it has a dominant ``NaN`` value -- and experts that + iterate the table assuming ``isinstance(raw, str)`` would silently skip the + real data while the share-based evidence was computed against a phantom. + """ + series = pd.Series(["a", "a", "b", None, np.nan]) + counts = _value_counts(series) + + assert list(counts.index) == ["a", "b"] + assert counts.to_dict() == {"a": 2, "b": 1} + assert int(counts.sum()) == 3 + assert not any(pd.isna(v) for v in counts.index) + + +def test_value_counts_drops_missing_cells_even_when_they_are_the_majority(): + """The all-but-one-null case: the table must not be led by NaN.""" + series = pd.Series([1.0, 1.0, np.nan, np.nan, np.nan]) + counts = _value_counts(series) + + assert len(counts) == 1 + assert counts.index[0] == 1.0 + assert int(counts.iloc[0]) == 2 + + +def test_value_counts_prefers_a_precomputed_native_table_over_rescanning(): + """The attrs fast path short-circuits before the pandas call it mutates.""" + precomputed = pd.Series({"x": 7, "y": 2}, dtype="int64") + series = pd.Series(["x", "y", None]) + series.attrs["fd_value_counts"] = precomputed + + assert _value_counts(series) is precomputed diff --git a/tests/test_scoring_boundaries.py b/tests/test_scoring_boundaries.py index 782b333..d09ca2d 100644 --- a/tests/test_scoring_boundaries.py +++ b/tests/test_scoring_boundaries.py @@ -498,3 +498,22 @@ def test_the_zero_span_guard_in_the_interpolator_is_unreachable(): hi = bisect.bisect_right(xs, raw) lo = hi - 1 assert xs[hi] - xs[lo] > 0, (xs, raw) + + +def test_the_feature_hash_does_not_depend_on_key_insertion_order(): + """The hash must be canonical, not merely deterministic. + + ``features_hash`` exists so a report consumer can compare two actions' + evidence. ``json.dumps(..., sort_keys=True)`` is what makes the digest a + function of the feature *content*; without it the digest also encodes the + order the dict literal happens to be written in, so reordering that + literal would silently invalidate every previously published hash while + still looking perfectly deterministic within any one build. + + Mutation testing caught this: flipping ``sort_keys`` to ``False`` survived + every test, because each one only ever compared hashes built in one order. + """ + a = {"raw_score": 0.9, "backend": "deterministic", "risk": "low"} + b = {"risk": "low", "backend": "deterministic", "raw_score": 0.9} + assert list(a) != list(b) # genuinely different insertion order + assert features_hash(a) == features_hash(b)