Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 18 additions & 1 deletion src/freshdata/enterprise/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
from __future__ import annotations

import dataclasses
import math
import secrets
from dataclasses import dataclass, field
from typing import Any, Literal
Expand Down Expand Up @@ -178,7 +179,7 @@ class TrustScoreWeights:
"""Relative weights blending the four trust dimensions into one score.

Weights need not sum to 1 — :meth:`normalized` rescales them. Each must be
non-negative and at least one must be positive.
finite and non-negative, and at least one must be positive.
"""

completeness: float = 0.30
Expand All @@ -189,6 +190,8 @@ class TrustScoreWeights:
def __post_init__(self) -> None:
for name in ("completeness", "validity", "uniqueness", "consistency"):
value = getattr(self, name)
if not math.isfinite(value):
raise ValueError(f"{name} weight must be finite, got {value!r}")
if value < 0:
raise ValueError(f"{name} weight must be >= 0, got {value!r}")
if self.completeness + self.validity + self.uniqueness + self.consistency <= 0:
Expand Down Expand Up @@ -338,6 +341,13 @@ class AnonymizationConfig:
Mirrors the masking knobs on :class:`MaskingRule` for callers that want to
drive :func:`freshdata.enterprise.privacy.anonymize` without the legacy
rule object. ``preserve_format`` selects surrogate/FPE shape preservation.

.. warning::
Not yet consumed by any pipeline. Passing it via
:attr:`EnterpriseConfig.anonymization` makes
:func:`~freshdata.enterprise.clean_enterprise` raise :class:`ValueError`
rather than silently return unmasked data. Use :class:`MaskingRule` or
:class:`PIIDetectionConfig` instead.
"""

strategy: Literal[
Expand Down Expand Up @@ -534,6 +544,12 @@ class EnterpriseConfig:
Bundles the feature toggles and sub-configs consumed by
:func:`freshdata.enterprise.clean_enterprise`. Frozen and hashable, so a
single instance can be shared across threads or reused for many frames.

``anonymization`` is **unsupported**: it is accepted and type-checked so
configs still construct, but no pipeline applies it, so
:func:`~freshdata.enterprise.clean_enterprise` raises :class:`ValueError`
when it is non-empty. Use ``masking`` (:class:`MaskingRule`) or ``privacy``
(:class:`PIIDetectionConfig` with ``enable_privacy_detection=True``) instead.
"""

actor: str | None = None
Expand All @@ -552,6 +568,7 @@ class EnterpriseConfig:
# --- new enterprise capabilities (all opt-in, backward compatible) ---
drift: DriftConfig | None = None
privacy: PIIDetectionConfig | None = None
#: Unsupported: ``clean_enterprise`` raises ``ValueError`` if non-empty.
anonymization: tuple[AnonymizationConfig, ...] = ()
k_anonymity: KAnonymityConfig | None = None
entity_resolution: EntityResolutionConfig | None = None
Expand Down
25 changes: 24 additions & 1 deletion src/freshdata/enterprise/interface.py
Original file line number Diff line number Diff line change
Expand Up @@ -211,6 +211,24 @@ def _gate_and_fold_profile(
return resolved, gate, fold_profile_options(resolved, dict(clean_options), gate)


def _resolve_enterprise_config(enterprise: EnterpriseConfig | None) -> EnterpriseConfig:
"""Return the effective config, failing closed on ``anonymization`` (#247).

The field is accepted by the config but no pipeline stage applies it, so a
non-empty value would silently return unmasked PII.
"""
ec = enterprise or EnterpriseConfig()
if ec.anonymization:
raise ValueError(
"EnterpriseConfig.anonymization is not supported by clean_enterprise and "
"would not be applied, leaving PII unmasked. Use "
"EnterpriseConfig(masking=(MaskingRule(...),)) for column-targeted masking, "
"or privacy=PIIDetectionConfig(...) with enable_privacy_detection=True "
"for detection-driven anonymization."
)
return ec


def clean_enterprise(
df: Any,
*,
Expand All @@ -235,8 +253,13 @@ def clean_enterprise(
``profile`` (a :class:`~freshdata.learning.LearningProfile` or path to a
``.fdprofile``) replays a learned profile with the same drift gating and
option-folding as :func:`freshdata.clean`.

Raises :class:`ValueError` if ``enterprise.anonymization`` is non-empty: that
field is not applied by the pipeline, so it fails closed instead of returning
unmasked data. Use ``masking=`` (:class:`MaskingRule`) or ``privacy=``
(:class:`PIIDetectionConfig`) instead.
"""
ec = enterprise or EnterpriseConfig()
ec = _resolve_enterprise_config(enterprise)
profile, profile_gate, clean_options = _gate_and_fold_profile(df, profile, clean_options)
cc = merge_options(clean_config, **clean_options)
who = actor or ec.actor or ec.lineage.actor
Expand Down
18 changes: 18 additions & 0 deletions tests/test_enterprise_config.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import pytest

from freshdata.enterprise import (
AnonymizationConfig,
ClusterConfig,
EnterpriseConfig,
LineageConfig,
Expand Down Expand Up @@ -81,6 +82,14 @@ def test_trust_weights_all_zero_rejected():
TrustScoreWeights(completeness=0, validity=0, uniqueness=0, consistency=0)


@pytest.mark.parametrize("name", ["completeness", "validity", "uniqueness", "consistency"])
@pytest.mark.parametrize("bad", [float("nan"), float("inf"), float("-inf")])
def test_trust_weights_non_finite_rejected(name, bad):
# Regression for #277: NaN slipped past `< 0` and yielded a NaN trust score.
with pytest.raises(ValueError, match=f"{name} weight must be finite"):
TrustScoreWeights(**{name: bad})


# -- SemanticValidatorConfig ----------------------------------------------

def test_semantic_reference_config_coerces_tuple():
Expand Down Expand Up @@ -133,6 +142,15 @@ def test_enterprise_config_invalid(kwargs, exc, match):
EnterpriseConfig(**kwargs)


def test_enterprise_config_still_constructs_with_anonymization():
# #247: the field stays accepted (and type-checked) so configs construct;
# clean_enterprise is what fails closed on it.
ec = EnterpriseConfig(anonymization=[AnonymizationConfig(strategy="redact")])
assert ec.anonymization == (AnonymizationConfig(strategy="redact"),)
with pytest.raises(TypeError, match="AnonymizationConfig"):
EnterpriseConfig(anonymization=("nope",))


def test_enterprise_config_is_hashable():
# Frozen + tuple fields => usable as a dict key / in a set.
ec = EnterpriseConfig(masking=(MaskingRule(name="p", columns=("e",)),))
Expand Down
37 changes: 37 additions & 0 deletions tests/test_enterprise_interface.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,11 +7,13 @@

from freshdata.adapters.polars import is_polars_frame
from freshdata.enterprise import (
AnonymizationConfig,
ClusterConfig,
EnterpriseConfig,
MaskingRule,
SemanticValidatorConfig,
clean_enterprise,
cli,
)


Expand Down Expand Up @@ -120,3 +122,38 @@ def test_clean_options_forwarded_and_validated(raw):
def test_clean_enterprise_actor_propagates_to_lineage(raw):
result = clean_enterprise(raw, enterprise=_full_config(), strategy="balanced", actor="bob")
assert result.lineage.events[0].who == "bob"


# -- EnterpriseConfig.anonymization fails closed (#247) --------------------

_ANON = (AnonymizationConfig(strategy="redact"),)


@pytest.mark.parametrize("enable_masking", [True, False])
def test_clean_enterprise_rejects_unsupported_anonymization(raw, enable_masking):
# The field is never applied, so it must raise rather than return raw PII.
ec = EnterpriseConfig(anonymization=_ANON, enable_masking=enable_masking)
with pytest.raises(ValueError, match=r"anonymization is not supported.*masking=.*privacy="):
clean_enterprise(raw, enterprise=ec, verbose=False)


def test_clean_enterprise_rejects_anonymization_on_polars_input(raw):
pl = pytest.importorskip("polars")
ec = EnterpriseConfig(anonymization=_ANON)
with pytest.raises(ValueError, match="anonymization is not supported"):
clean_enterprise(pl.from_pandas(raw), enterprise=ec, verbose=False)


def test_cli_clean_fails_closed_on_anonymization(raw, tmp_path, monkeypatch, capsys):
# The CLI builds its config then calls clean_enterprise; the guard must surface
# as a one-line error with a non-zero exit and no output file written.
monkeypatch.setattr(
cli, "_build_enterprise", lambda spec: EnterpriseConfig(anonymization=_ANON)
)
src, cfg, out = tmp_path / "in.csv", tmp_path / "cfg.json", tmp_path / "out.csv"
raw.to_csv(src, index=False)
cfg.write_text(json.dumps({"enterprise": {}}))
code = cli.main(["clean", str(src), "-o", str(out), "--config", str(cfg), "--quiet"])
assert code != 0
assert "anonymization is not supported" in capsys.readouterr().err
assert not out.exists()
Loading