diff --git a/src/freshdata/enterprise/config.py b/src/freshdata/enterprise/config.py index 31e2e7c..433fcd7 100644 --- a/src/freshdata/enterprise/config.py +++ b/src/freshdata/enterprise/config.py @@ -13,6 +13,7 @@ from __future__ import annotations import dataclasses +import math import secrets from dataclasses import dataclass, field from typing import Any, Literal @@ -178,7 +179,7 @@ class TrustScoreWeights: """Relative weights blending the four trust dimensions into one score. Weights need not sum to 1 — :meth:`normalized` rescales them. Each must be - non-negative and at least one must be positive. + finite and non-negative, and at least one must be positive. """ completeness: float = 0.30 @@ -189,6 +190,8 @@ class TrustScoreWeights: def __post_init__(self) -> None: for name in ("completeness", "validity", "uniqueness", "consistency"): value = getattr(self, name) + if not math.isfinite(value): + raise ValueError(f"{name} weight must be finite, got {value!r}") if value < 0: raise ValueError(f"{name} weight must be >= 0, got {value!r}") if self.completeness + self.validity + self.uniqueness + self.consistency <= 0: @@ -338,6 +341,13 @@ class AnonymizationConfig: Mirrors the masking knobs on :class:`MaskingRule` for callers that want to drive :func:`freshdata.enterprise.privacy.anonymize` without the legacy rule object. ``preserve_format`` selects surrogate/FPE shape preservation. + + .. warning:: + Not yet consumed by any pipeline. Passing it via + :attr:`EnterpriseConfig.anonymization` makes + :func:`~freshdata.enterprise.clean_enterprise` raise :class:`ValueError` + rather than silently return unmasked data. Use :class:`MaskingRule` or + :class:`PIIDetectionConfig` instead. """ strategy: Literal[ @@ -534,6 +544,12 @@ class EnterpriseConfig: Bundles the feature toggles and sub-configs consumed by :func:`freshdata.enterprise.clean_enterprise`. Frozen and hashable, so a single instance can be shared across threads or reused for many frames. + + ``anonymization`` is **unsupported**: it is accepted and type-checked so + configs still construct, but no pipeline applies it, so + :func:`~freshdata.enterprise.clean_enterprise` raises :class:`ValueError` + when it is non-empty. Use ``masking`` (:class:`MaskingRule`) or ``privacy`` + (:class:`PIIDetectionConfig` with ``enable_privacy_detection=True``) instead. """ actor: str | None = None @@ -552,6 +568,7 @@ class EnterpriseConfig: # --- new enterprise capabilities (all opt-in, backward compatible) --- drift: DriftConfig | None = None privacy: PIIDetectionConfig | None = None + #: Unsupported: ``clean_enterprise`` raises ``ValueError`` if non-empty. anonymization: tuple[AnonymizationConfig, ...] = () k_anonymity: KAnonymityConfig | None = None entity_resolution: EntityResolutionConfig | None = None diff --git a/src/freshdata/enterprise/interface.py b/src/freshdata/enterprise/interface.py index 90c5fa3..762ec44 100644 --- a/src/freshdata/enterprise/interface.py +++ b/src/freshdata/enterprise/interface.py @@ -211,6 +211,24 @@ def _gate_and_fold_profile( return resolved, gate, fold_profile_options(resolved, dict(clean_options), gate) +def _resolve_enterprise_config(enterprise: EnterpriseConfig | None) -> EnterpriseConfig: + """Return the effective config, failing closed on ``anonymization`` (#247). + + The field is accepted by the config but no pipeline stage applies it, so a + non-empty value would silently return unmasked PII. + """ + ec = enterprise or EnterpriseConfig() + if ec.anonymization: + raise ValueError( + "EnterpriseConfig.anonymization is not supported by clean_enterprise and " + "would not be applied, leaving PII unmasked. Use " + "EnterpriseConfig(masking=(MaskingRule(...),)) for column-targeted masking, " + "or privacy=PIIDetectionConfig(...) with enable_privacy_detection=True " + "for detection-driven anonymization." + ) + return ec + + def clean_enterprise( df: Any, *, @@ -235,8 +253,13 @@ def clean_enterprise( ``profile`` (a :class:`~freshdata.learning.LearningProfile` or path to a ``.fdprofile``) replays a learned profile with the same drift gating and option-folding as :func:`freshdata.clean`. + + Raises :class:`ValueError` if ``enterprise.anonymization`` is non-empty: that + field is not applied by the pipeline, so it fails closed instead of returning + unmasked data. Use ``masking=`` (:class:`MaskingRule`) or ``privacy=`` + (:class:`PIIDetectionConfig`) instead. """ - ec = enterprise or EnterpriseConfig() + ec = _resolve_enterprise_config(enterprise) profile, profile_gate, clean_options = _gate_and_fold_profile(df, profile, clean_options) cc = merge_options(clean_config, **clean_options) who = actor or ec.actor or ec.lineage.actor diff --git a/tests/test_enterprise_config.py b/tests/test_enterprise_config.py index e23f10e..4a446ff 100644 --- a/tests/test_enterprise_config.py +++ b/tests/test_enterprise_config.py @@ -3,6 +3,7 @@ import pytest from freshdata.enterprise import ( + AnonymizationConfig, ClusterConfig, EnterpriseConfig, LineageConfig, @@ -81,6 +82,14 @@ def test_trust_weights_all_zero_rejected(): TrustScoreWeights(completeness=0, validity=0, uniqueness=0, consistency=0) +@pytest.mark.parametrize("name", ["completeness", "validity", "uniqueness", "consistency"]) +@pytest.mark.parametrize("bad", [float("nan"), float("inf"), float("-inf")]) +def test_trust_weights_non_finite_rejected(name, bad): + # Regression for #277: NaN slipped past `< 0` and yielded a NaN trust score. + with pytest.raises(ValueError, match=f"{name} weight must be finite"): + TrustScoreWeights(**{name: bad}) + + # -- SemanticValidatorConfig ---------------------------------------------- def test_semantic_reference_config_coerces_tuple(): @@ -133,6 +142,15 @@ def test_enterprise_config_invalid(kwargs, exc, match): EnterpriseConfig(**kwargs) +def test_enterprise_config_still_constructs_with_anonymization(): + # #247: the field stays accepted (and type-checked) so configs construct; + # clean_enterprise is what fails closed on it. + ec = EnterpriseConfig(anonymization=[AnonymizationConfig(strategy="redact")]) + assert ec.anonymization == (AnonymizationConfig(strategy="redact"),) + with pytest.raises(TypeError, match="AnonymizationConfig"): + EnterpriseConfig(anonymization=("nope",)) + + def test_enterprise_config_is_hashable(): # Frozen + tuple fields => usable as a dict key / in a set. ec = EnterpriseConfig(masking=(MaskingRule(name="p", columns=("e",)),)) diff --git a/tests/test_enterprise_interface.py b/tests/test_enterprise_interface.py index dd178d7..7e9251e 100644 --- a/tests/test_enterprise_interface.py +++ b/tests/test_enterprise_interface.py @@ -7,11 +7,13 @@ from freshdata.adapters.polars import is_polars_frame from freshdata.enterprise import ( + AnonymizationConfig, ClusterConfig, EnterpriseConfig, MaskingRule, SemanticValidatorConfig, clean_enterprise, + cli, ) @@ -120,3 +122,38 @@ def test_clean_options_forwarded_and_validated(raw): def test_clean_enterprise_actor_propagates_to_lineage(raw): result = clean_enterprise(raw, enterprise=_full_config(), strategy="balanced", actor="bob") assert result.lineage.events[0].who == "bob" + + +# -- EnterpriseConfig.anonymization fails closed (#247) -------------------- + +_ANON = (AnonymizationConfig(strategy="redact"),) + + +@pytest.mark.parametrize("enable_masking", [True, False]) +def test_clean_enterprise_rejects_unsupported_anonymization(raw, enable_masking): + # The field is never applied, so it must raise rather than return raw PII. + ec = EnterpriseConfig(anonymization=_ANON, enable_masking=enable_masking) + with pytest.raises(ValueError, match=r"anonymization is not supported.*masking=.*privacy="): + clean_enterprise(raw, enterprise=ec, verbose=False) + + +def test_clean_enterprise_rejects_anonymization_on_polars_input(raw): + pl = pytest.importorskip("polars") + ec = EnterpriseConfig(anonymization=_ANON) + with pytest.raises(ValueError, match="anonymization is not supported"): + clean_enterprise(pl.from_pandas(raw), enterprise=ec, verbose=False) + + +def test_cli_clean_fails_closed_on_anonymization(raw, tmp_path, monkeypatch, capsys): + # The CLI builds its config then calls clean_enterprise; the guard must surface + # as a one-line error with a non-zero exit and no output file written. + monkeypatch.setattr( + cli, "_build_enterprise", lambda spec: EnterpriseConfig(anonymization=_ANON) + ) + src, cfg, out = tmp_path / "in.csv", tmp_path / "cfg.json", tmp_path / "out.csv" + raw.to_csv(src, index=False) + cfg.write_text(json.dumps({"enterprise": {}})) + code = cli.main(["clean", str(src), "-o", str(out), "--config", str(cfg), "--quiet"]) + assert code != 0 + assert "anonymization is not supported" in capsys.readouterr().err + assert not out.exists()