From 00fd5a95fa3aa15d07efd6f320219919122be47f Mon Sep 17 00:00:00 2001 From: Kevin Costner <120246174+kevincostner17@users.noreply.github.com> Date: Tue, 15 Sep 2026 02:49:50 +0530 Subject: [PATCH] fix(compliance): base HIPAA/GDPR reports on real column and masking evidence HIPAA Safe Harbor passed without dataframe= even when identifier columns existed (#245). Without a source frame the adapter only saw columns that cleaning touched, so untouched identifiers such as ssn or patient_email were invisible and all 18 identifiers came back not_detected. CleanReport now records input_columns (the post-rename column list) in run_pipeline and execute_plan, and the compliance context uses it when dataframe= is absent. When neither is available the HIPAA report sets coverage_verifiable=False, adds a warning and does not pass. input_columns is not added to to_dict(), so the JSON schema and golden snapshots are unchanged. HIPAA identifier hints matched raw substrings (#283), so "ip" flagged description and shipping_cost, "date" flagged last_updated and "sin" flagged business_unit. Column names are now tokenized on separators, camelCase and letter/digit boundaries. Every hint matches as a whole token or token sequence. Hints of four characters or fewer match only that way. Longer hints may also match inside run-together names (patientemail, dateofbirth), but not starting part-way through a token (ip_address no longer matches ship_address). "surname" and "zipcode" hints were added so those common run-together names are still detected. The GDPR Article 30 record always listed "Hash-salt PII masking (SHA-256 + salt)" (#287). security_measures is now built from evidence: a masking entry appears only when columns were masked, with the recorded strategies and no cryptographic claims, and the Data Trust Score entry appears only when a trust score was available. Closes #245 Closes #283 Closes #287 --- docs/compliance.md | 4 +- src/freshdata/cleaner.py | 3 + src/freshdata/compliance/_adapter.py | 30 +++++- src/freshdata/compliance/_gdpr.py | 26 +++-- src/freshdata/compliance/_hipaa.py | 71 ++++++++++++-- src/freshdata/repairplan.py | 1 + src/freshdata/report.py | 6 ++ tests/test_compliance/test_gdpr.py | 30 ++++++ tests/test_compliance/test_hipaa.py | 141 +++++++++++++++++++++++++++ 9 files changed, 294 insertions(+), 18 deletions(-) diff --git a/docs/compliance.md b/docs/compliance.md index 1d297619..223cf7ba 100644 --- a/docs/compliance.md +++ b/docs/compliance.md @@ -103,7 +103,9 @@ Two optional, keyword-only arguments add evidence when available: - **`dataframe=`** — the source frame. Recovers per-column roles and missing ratios via [`freshdata.infer_roles`](api-reference.md), sharpening the HIPAA and - ALCOA reports. + ALCOA reports. Without it, the full column list comes from the report's + `input_columns` (recorded by `freshdata.clean`). If neither is available, the + HIPAA report sets `coverage_verifiable` to `False`, adds a warning, and does not pass. - **`enterprise_result=`** — an [enterprise](feature-overview.md) result supplying the 0–100 Data Trust Score, PII-masking events, and fuzzy-clustering lineage. diff --git a/src/freshdata/cleaner.py b/src/freshdata/cleaner.py index 5012aa24..0d9802d3 100644 --- a/src/freshdata/cleaner.py +++ b/src/freshdata/cleaner.py @@ -114,6 +114,9 @@ def run_pipeline( # noqa: PLR0915 - fixed-order pipeline orchestration if config.column_names: out = normalize_column_names(out, report) _emit_progress(progress_callback, "column_names", "after", out) + # Full column evidence for compliance reports, in the report's (post-rename) + # namespace, so untouched columns are still visible without the source frame. + report.input_columns = [str(c) for c in out.columns] # Hard protected-column guard (context policy / mutable=False): fold the # protected set into preserve_columns so drop/impute logic honors it, and diff --git a/src/freshdata/compliance/_adapter.py b/src/freshdata/compliance/_adapter.py index bec92dcb..f0e38807 100644 --- a/src/freshdata/compliance/_adapter.py +++ b/src/freshdata/compliance/_adapter.py @@ -13,7 +13,7 @@ from __future__ import annotations import logging -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import Any import pandas as pd @@ -158,6 +158,12 @@ class ComplianceContext: core_action_count: int clean_report: Any input_dataframe_hash: str | None = None + #: ``True`` when ``all_columns`` is the full column list (from ``dataframe=`` + #: or the report's recorded ``input_columns``); ``False`` when it was only + #: inferred from actions/masked columns, so untouched columns are invisible. + columns_complete: bool = False + #: ``{column: strategy}`` recorded by an enterprise mask report, when known. + mask_strategies: dict[str, str] = field(default_factory=dict) # --------------------------------------------------------------------------- # @@ -297,6 +303,9 @@ def build_context( # Trust score: EnterpriseResult -> domain score (0–1, scaled) -> config. trust_score = _resolve_trust_score(enterprise, clean_report, config) + all_columns, columns_complete = _resolve_all_columns( + clean_report, dataframe, core_actions, masked_columns + ) return ComplianceContext( session_id=new_session_id(), @@ -308,10 +317,12 @@ def build_context( roles=roles, missing_ratio=missing_ratio, domain_sensitive_columns=domain_sensitive, - all_columns=_resolve_all_columns(clean_report, dataframe, core_actions, masked_columns), + all_columns=all_columns, core_action_count=len(core_actions), clean_report=clean_report, input_dataframe_hash=config.input_dataframe_hash, + columns_complete=columns_complete, + mask_strategies={c: str(s) for c, s in mask_meta.items() if s}, ) @@ -335,9 +346,18 @@ def _resolve_all_columns( dataframe: pd.DataFrame | None, core_actions: list[Any], masked_columns: set[str], -) -> list[str]: +) -> tuple[list[str], bool]: + """Return ``(columns, complete)``. + + ``complete`` is ``True`` only when the full column list is known: from the + source ``dataframe`` or the report's recorded ``input_columns``. Otherwise + columns are inferred from what cleaning touched, which misses untouched ones. + """ if dataframe is not None: - return [str(c) for c in dataframe.columns] + return [str(c) for c in dataframe.columns], True + recorded = getattr(clean_report, "input_columns", None) or [] + if recorded: + return [str(c) for c in recorded], True columns: set[str] = set(masked_columns) for action in core_actions: column = getattr(action, "column", None) @@ -345,4 +365,4 @@ def _resolve_all_columns( columns.add(column) for attr in ("columns_dropped", "columns_imputed", "columns_preserved"): columns.update(getattr(clean_report, attr, []) or []) - return sorted(columns) + return sorted(columns), False diff --git a/src/freshdata/compliance/_gdpr.py b/src/freshdata/compliance/_gdpr.py index 9d560339..20e84347 100644 --- a/src/freshdata/compliance/_gdpr.py +++ b/src/freshdata/compliance/_gdpr.py @@ -23,11 +23,25 @@ "drop_row": "Art.5(1)(e) storage limitation (empty/irreparable record)", "pii_mask": "Art.5(1)(f) integrity and confidentiality", } -_SECURITY_MEASURES = [ - "Hash-salt PII masking (SHA-256 + salt)", - "Audit trail generation", - "Access-gated Data Trust Score", -] + + +def _security_measures(ctx: ComplianceContext) -> list[str]: + """Security measures actually evidenced by this run (never a fixed list). + + Masking is listed only when columns were masked, naming the strategies that + were recorded; no cryptographic properties are asserted. + """ + measures: list[str] = [] + masked = set(ctx.masked_columns) + masked.update(a.column for a in ctx.actions if a.action_type == "pii_mask" and a.column) + if masked: + strategies = sorted({s for c, s in ctx.mask_strategies.items() if c in masked and s}) + detail = f" (strategies: {', '.join(strategies)})" if strategies else "" + measures.append(f"PII masking/anonymisation applied to {len(masked)} column(s){detail}") + measures.append("Audit trail generation") + if ctx.trust_score_available: + measures.append("Data Trust Score computed") + return measures def _personal_data_categories(ctx: ComplianceContext) -> list[str]: @@ -62,7 +76,7 @@ def generate_gdpr(ctx: ComplianceContext, config: ComplianceConfig) -> Framework "third_country_transfers": False, "third_country_transfers_note": ("freshdata processes in-memory; no network transfer."), "retention_days": config.retention_days, - "security_measures": list(_SECURITY_MEASURES), + "security_measures": _security_measures(ctx), "automated_decision_making": True, "automated_decision_making_note": ("Column-level cleaning decisions are fully automated."), "safeguards": ( diff --git a/src/freshdata/compliance/_hipaa.py b/src/freshdata/compliance/_hipaa.py index 90532510..ca1c9f9b 100644 --- a/src/freshdata/compliance/_hipaa.py +++ b/src/freshdata/compliance/_hipaa.py @@ -5,6 +5,7 @@ from __future__ import annotations +import re from typing import Any from ._adapter import ComplianceContext @@ -25,12 +26,19 @@ "names": { "id": 1, "description": "Names", - "detection_hints": ["name", "first_name", "last_name", "full_name", "patient_name"], + "detection_hints": [ + "name", + "first_name", + "last_name", + "full_name", + "patient_name", + "surname", + ], }, "geographic": { "id": 2, "description": "Geographic subdivisions smaller than state", - "detection_hints": ["address", "street", "city", "county", "zip", "postal"], + "detection_hints": ["address", "street", "city", "county", "zip", "zipcode", "postal"], }, "dates": { "id": 3, @@ -126,6 +134,46 @@ } +#: Hints this short (``ip``, ``sin``, ``date``, ...) are ambiguous inside other +#: words, so they only match as whole name tokens. +_SHORT_HINT_MAX_LEN = 4 +_TOKEN_BOUNDARY = re.compile( + r"(?<=[a-z])(?=[A-Z])" # camelCase + r"|(?<=[A-Z])(?=[A-Z][a-z]{2})" # acronym followed by a word: IPAddress + r"|(?<=[A-Za-z])(?=[0-9])|(?<=[0-9])(?=[A-Za-z])" # letter/digit: zip5, ipv4 +) +_SEPARATORS = re.compile(r"[\W_]+") + + +def _tokens(name: object) -> tuple[str, ...]: + """Split a column name into lowercase tokens (separators, camelCase, digits).""" + spaced = _TOKEN_BOUNDARY.sub(" ", str(name)) + return tuple(t.lower() for t in _SEPARATORS.split(spaced) if t) + + +def _hint_matches(hint_tokens: tuple[str, ...], col_tokens: tuple[str, ...]) -> bool: + """Whether the hint (as ``hint_tokens``) names a column with ``col_tokens``. + + Every hint matches as a whole token or token sequence (``date_of_birth`` + matches ``DateOfBirth``). Longer hints may also appear inside run-together + names (``patientemail``) but never starting part-way through a token they + extend past (``ip_address`` does not match ``ship_address``). + """ + n = len(hint_tokens) + if any(col_tokens[i : i + n] == hint_tokens for i in range(len(col_tokens) - n + 1)): + return True + compact = "".join(hint_tokens) + if len(compact) <= _SHORT_HINT_MAX_LEN: + return False + joined = "".join(col_tokens) + offset = 0 + for token in col_tokens: + if compact in token or joined.startswith(compact, offset): + return True + offset += len(token) + return False + + def _known_columns(ctx: ComplianceContext) -> list[str]: columns: set[str] = set(ctx.all_columns) | set(ctx.masked_columns) columns.update(a.column for a in ctx.actions if a.column) @@ -134,6 +182,7 @@ def _known_columns(ctx: ComplianceContext) -> list[str]: def generate_hipaa(ctx: ComplianceContext, config: ComplianceConfig) -> FrameworkReport: known_columns = _known_columns(ctx) + column_tokens = {col: _tokens(col) for col in known_columns} masked = set(ctx.masked_columns) identifier_coverage: dict[str, dict] = {} @@ -141,9 +190,11 @@ def generate_hipaa(ctx: ComplianceContext, config: ComplianceConfig) -> Framewor gaps: list[str] = [] for key, spec in HIPAA_IDENTIFIERS.items(): - hints = spec["detection_hints"] + hint_tokens = [_tokens(hint) for hint in spec["detection_hints"]] columns_found = [ - col for col in known_columns if any(hint in col.lower() for hint in hints) + col + for col in known_columns + if any(_hint_matches(tokens, column_tokens[col]) for tokens in hint_tokens) ] columns_masked = [col for col in columns_found if col in masked] @@ -180,6 +231,13 @@ def generate_hipaa(ctx: ComplianceContext, config: ComplianceConfig) -> Framewor raise ComplianceGapError(f"HIPAA Safe Harbor gaps detected: {gaps}") errors = [f"Identifier {key!r} detected but not addressed (no PII masking)." for key in gaps] + warnings: list[str] = [] + if not ctx.columns_complete: + warnings.append( + "Column coverage not verifiable without dataframe=: the clean report records " + "no input column list, so identifier columns that cleaning did not touch " + "cannot be detected. Pass the source frame as dataframe= to verify coverage." + ) data = { "report_id": new_entry_id("HIPAA"), @@ -188,13 +246,14 @@ def generate_hipaa(ctx: ComplianceContext, config: ComplianceConfig) -> Framewor "identifier_coverage": identifier_coverage, "summary": summary, "gaps": gaps, + "coverage_verifiable": ctx.columns_complete, "caveat": _HIPAA_CAVEAT, } return FrameworkReport( framework_key=FRAMEWORK_KEY, framework_name=FRAMEWORK_NAME, - passed=not gaps, - warnings=[], + passed=not gaps and ctx.columns_complete, + warnings=warnings, errors=errors, data=data, ) diff --git a/src/freshdata/repairplan.py b/src/freshdata/repairplan.py index 4a671d07..4ae47e58 100644 --- a/src/freshdata/repairplan.py +++ b/src/freshdata/repairplan.py @@ -719,6 +719,7 @@ def execute_plan( rows_before=len(df), cols_before=df.shape[1], missing_before=int(df.isna().sum().sum()), + input_columns=[str(c) for c in df.columns], ) out = df.copy(deep=False) guard_snapshot = snapshot_protected( diff --git a/src/freshdata/report.py b/src/freshdata/report.py index f7daacae..86fb134f 100644 --- a/src/freshdata/report.py +++ b/src/freshdata/report.py @@ -126,6 +126,12 @@ class CleanReport(HtmlReprMixin): columns_dropped: list[str] = field(default_factory=list) columns_imputed: list[str] = field(default_factory=list) columns_preserved: list[str] = field(default_factory=list) + #: Every column the pipeline received, in input order, under the names used + #: throughout this report (i.e. after column-name normalization). Empty when + #: the producer did not record it. Compliance generators use it as column + #: evidence when no source ``dataframe=`` is supplied. Not part of + #: :meth:`to_dict` (the stable audit payload is unchanged). + input_columns: list[str] = field(default_factory=list) warnings: list[str] = field(default_factory=list) recommendations: list[str] = field(default_factory=list) #: Per-cell record of values that ``fix_dtypes`` coerced to missing because diff --git a/tests/test_compliance/test_gdpr.py b/tests/test_compliance/test_gdpr.py index 43c57fd1..75649b5e 100644 --- a/tests/test_compliance/test_gdpr.py +++ b/tests/test_compliance/test_gdpr.py @@ -53,3 +53,33 @@ def test_article_30_record_fields(make_report): assert article_30["third_country_transfers"] is False assert article_30["automated_decision_making"] is True assert gdpr.data["caveat"] + + +# --- #287: security measures reflect evidence, not a constant list ------------ + + +def test_security_measures_omit_masking_when_nothing_masked(sample_report, sample_df): + gdpr = _gdpr(sample_report, dataframe=sample_df) + measures = gdpr.data["article_30"]["security_measures"] + assert not any("mask" in m.lower() for m in measures) + assert not any("sha-256" in m.lower() or "salt" in m.lower() for m in measures) + assert "Audit trail generation" in measures + # No enterprise trust score was supplied, so none is claimed. + assert not any("trust score" in m.lower() for m in measures) + + +def test_security_measures_list_masking_from_config(make_report): + report = make_report({"step": "missing", "column": "age", "count": 0}) + gdpr = _gdpr(report, config=ComplianceConfig(masked_columns=["email", "ssn"])) + measures = gdpr.data["article_30"]["security_measures"] + assert measures[0] == "PII masking/anonymisation applied to 2 column(s)" + + +def test_security_measures_name_recorded_strategies(sample_report, enterprise_stub): + result = enterprise_stub(sample_report, overall=90.0, masked=["email"]) + gdpr = _gdpr(result) + measures = gdpr.data["article_30"]["security_measures"] + assert measures[0] == ( + "PII masking/anonymisation applied to 1 column(s) (strategies: sha256+salt)" + ) + assert "Data Trust Score computed" in measures diff --git a/tests/test_compliance/test_hipaa.py b/tests/test_compliance/test_hipaa.py index a2c104e9..3e57a712 100644 --- a/tests/test_compliance/test_hipaa.py +++ b/tests/test_compliance/test_hipaa.py @@ -2,8 +2,10 @@ from __future__ import annotations +import pandas as pd import pytest +import freshdata as fd from freshdata.compliance import ( ComplianceConfig, ComplianceGapError, @@ -59,3 +61,142 @@ def test_gap_sets_passed_false_and_errors(sample_report, sample_df): assert hipaa.passed is False assert hipaa.errors assert hipaa.data["caveat"] + + +# --- #245: coverage without dataframe= ----------------------------------------- + + +def test_clean_records_input_columns_after_rename(): + raw = pd.DataFrame({"Patient Email": ["a@b.com", "c@d.com"], "v": [1, 2]}) + _, report = fd.clean(raw, return_report=True, verbose=False) + assert report.input_columns == ["patient_email", "v"] + assert "input_columns" not in report.to_dict() # stable audit payload unchanged + + +def test_untouched_identifier_columns_fail_without_dataframe(): + raw = pd.DataFrame( + {"ssn": ["123-45-6789", "987-65-4321"], "patient_email": ["a@b.com", "c@d.com"]} + ) + _, report = fd.clean(raw, return_report=True, verbose=False) + no_df = _hipaa(report) + with_df = _hipaa(report, dataframe=raw) + assert no_df.passed is False + assert no_df.data["gaps"] == with_df.data["gaps"] == ["email", "ssn"] + assert no_df.data["coverage_verifiable"] is True + assert no_df.warnings == [] + + +def test_masked_identifiers_pass_without_dataframe(): + raw = pd.DataFrame({"email": ["a@b.com", "c@d.com"], "v": [1, 2]}) + _, report = fd.clean(raw, return_report=True, verbose=False) + hipaa = _hipaa(report, config=ComplianceConfig(masked_columns=["email"])) + assert hipaa.passed is True + assert hipaa.data["coverage_verifiable"] is True + + +def test_no_column_evidence_is_unverifiable_and_not_passed(make_report): + report = make_report({"step": "missing", "column": "age", "count": 0}) + hipaa = _hipaa(report) + assert hipaa.data["gaps"] == [] + assert hipaa.data["coverage_verifiable"] is False + assert hipaa.passed is False + assert any("not verifiable without dataframe=" in w for w in hipaa.warnings) + assert hipaa.errors == [] + + +def test_dataframe_makes_synthetic_report_verifiable(make_report): + report = make_report({"step": "missing", "column": "age", "count": 0}) + hipaa = _hipaa(report, dataframe=pd.DataFrame({"age": [1, 2]})) + assert hipaa.data["coverage_verifiable"] is True + assert hipaa.passed is True + assert hipaa.warnings == [] + + +# --- #283: identifier hints match name tokens, not arbitrary substrings ------- + + +@pytest.mark.parametrize( + "column", + [ + "description", + "shipping_cost", + "last_updated", + "business_unit", + "hotel", + "cancelled", + "province", + "fluid", + "surface", + "backlinks", + "filename", + ], +) +def test_short_hints_do_not_match_inside_words(make_report, column): + hipaa = _hipaa(make_report(), dataframe=pd.DataFrame({column: [1, 2]})) + found = {k: v["columns_found"] for k, v in hipaa.data["identifier_coverage"].items()} + assert not any(found.values()), found + assert hipaa.passed is True + + +def test_issue_283_repro_passes(make_report): + df = pd.DataFrame( + { + "description": ["ok", "fine"], + "last_updated": ["y", "n"], + "shipping_cost": [1, 2], + "business_unit": ["a", "b"], + } + ) + _, report = fd.clean(df, return_report=True, verbose=False) + hipaa = _hipaa(report, dataframe=df) + assert hipaa.passed is True + assert hipaa.data["gaps"] == [] + + +@pytest.mark.parametrize( + ("column", "identifier"), + [ + ("ip", "ip_addresses"), + ("client_ip", "ip_addresses"), + ("IPAddress", "ip_addresses"), + ("IPv4Address", "ip_addresses"), + ("date_of_birth", "dates"), + ("DateOfBirth", "dates"), + ("dateofbirth", "dates"), + ("visit_date", "dates"), + ("patientemail", "email"), + ("PatientEmail", "email"), + ("SSN", "ssn"), + ("home_tel", "phone"), + ("zip5", "geographic"), + ("zipcode", "geographic"), + ("surname", "names"), + ("firstName", "names"), + ("vehicle_vin", "vehicle_identifiers"), + ("record_uid", "other_unique"), + ], +) +def test_identifier_hints_still_detect_real_identifiers(make_report, column, identifier): + hipaa = _hipaa(make_report(), dataframe=pd.DataFrame({column: [1, 2]})) + assert column in hipaa.data["identifier_coverage"][identifier]["columns_found"] + assert hipaa.passed is False + + +def test_long_hint_does_not_start_mid_token(make_report): + hipaa = _hipaa(make_report(), dataframe=pd.DataFrame({"ship_address": [1, 2]})) + coverage = hipaa.data["identifier_coverage"] + assert coverage["ip_addresses"]["columns_found"] == [] + assert coverage["geographic"]["columns_found"] == ["ship_address"] + + +def test_apply_plan_records_input_columns(): + df = pd.DataFrame( + { + "ssn": ["123-45-6789", "987-65-4321", "111-22-3333", "444-55-6666"], + "email_addr": ["a@@b.com", "x @ y.com", "ok@ok.com", "junk"], + } + ) + plan = fd.suggest_plan(df, semantic_mode="auto", verbose=False).repair_plan + _, report = fd.apply_plan(df, plan) + assert report.input_columns == ["ssn", "email_addr"] + assert _hipaa(report).data["coverage_verifiable"] is True