diff --git a/buckaroo/customizations/pd_stats_v2.py b/buckaroo/customizations/pd_stats_v2.py index 68d281fb7..4156c770e 100644 --- a/buckaroo/customizations/pd_stats_v2.py +++ b/buckaroo/customizations/pd_stats_v2.py @@ -248,6 +248,8 @@ def histogram(value_counts: pd.Series, nan_per: float, is_numeric: bool, length: def pd_cleaning_stats(value_counts: pd.Series, length: int) -> PdCleaningResult: """Compute int parsing stats for cleaning.""" vc = value_counts + if vc.empty: + return {'int_parse_fail': 1.0, 'int_parse': 0.0} coerced_ser = pd.to_numeric(vc.index.values, errors='coerce', downcast='integer', dtype_backend='pyarrow') nan_sum = (pd.Series(coerced_ser).isna() * 1 * vc.values).sum() return {'int_parse_fail': nan_sum / length, 'int_parse': (length - nan_sum) / length} diff --git a/tests/unit/test_pd_stats_v2.py b/tests/unit/test_pd_stats_v2.py index de5f3e7f8..334483f29 100644 --- a/tests/unit/test_pd_stats_v2.py +++ b/tests/unit/test_pd_stats_v2.py @@ -12,7 +12,7 @@ from buckaroo.pluggable_analysis_framework.stat_pipeline import StatPipeline from buckaroo.pluggable_analysis_framework.utils import PERVERSE_DF -from buckaroo.customizations.pd_stats_v2 import (typing_stats, _type, base_summary_stats, numeric_stats, computed_default_summary_stats, histogram_series, histogram, pd_cleaning_stats, heuristic_fracs, orig_col_name, PD_ANALYSIS_V2) +from buckaroo.customizations.pd_stats_v2 import (typing_stats, _type, base_summary_stats, numeric_stats, computed_default_summary_stats, histogram_series, histogram, pd_cleaning_stats, heuristic_fracs, orig_col_name, PD_ANALYSIS_V2, PD_AUTOCLEAN_DEFAULT_V2) # ============================================================================ @@ -335,6 +335,18 @@ def test_string_column(self): assert result['int_parse'] > 0 assert result['int_parse_fail'] > 0 + def test_unhashable_column_is_not_integer_parseable(self): + result = pd_cleaning_stats(pd.Series(dtype=object), length=4) + assert result['int_parse'] == 0.0 + assert result['int_parse_fail'] == 1.0 + + def test_default_autoclean_skips_safe_int_for_unhashable_column(self): + pipeline = StatPipeline(PD_AUTOCLEAN_DEFAULT_V2, unit_test=False) + summary, _ = pipeline.process_df(pd.DataFrame({'lists': [['a'], ['b']]})) + column_summary = next(iter(summary.values())) + assert column_summary['orig_col_name'] == 'lists' + assert column_summary['cleaning_ops'] == [] + # ============================================================================ # Tests: heuristic_fracs @@ -436,4 +448,3 @@ def test_unit_test_runs(self): pipeline = StatPipeline(PD_ANALYSIS_V2, unit_test=True) passed, errors = pipeline._unit_test_result # Some errors may occur on edge cases, but shouldn't crash -