fix(duckdb): exact metadata scan, all-empty columns, ordered dedup, Unicode whitespace

src/freshdata/_util.py

@@ -235,3 +235,13 @@ def mask_sensitive_value(value: object) -> str:
     """
     digest = hashlib.sha256(repr(value).encode("utf-8")).hexdigest()[:8]
     return f"[SENSITIVE:{digest}]"
+
+
+#: Every character ``str.isspace`` accepts, i.e. what ``str.strip()`` removes.
+#: Native engines strip exactly this set so they match the pandas reference
+#: (RE2's ``\s`` is ASCII-only; Rust's whitespace excludes ``\x1c``-``\x1f``).
+PY_WHITESPACE = (
+    "\t\n\x0b\x0c\r\x1c\x1d\x1e\x1f \x85\xa0\u1680"
+    "\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200a"
+    "\u2028\u2029\u202f\u205f\u3000"
+)

src/freshdata/execution/_metadata.py

@@ -2,8 +2,8 @@
 
 :class:`ColumnMetadata` is everything the planner and the selector need to make
 decisions without materialising a dataset. Each scanner uses the cheapest path
-its backend offers: pandas describe on a sample, polars lazy aggregates, DuckDB
-``SUMMARIZE``, or the Parquet footer.
+its backend offers: pandas describe on a sample, polars lazy aggregates, one
+DuckDB aggregate query, or the Parquet footer.
 """
 
 from __future__ import annotations
@@ -21,6 +21,15 @@
 _SAMPLE_FRAC = 0.10
 
 
+def _quote_identifier(name: str) -> str:
+    return '"' + name.replace('"', '""') + '"'
+
+
+def is_duckdb_float(native_dtype: str) -> bool:
+    """True for DuckDB floating-point types, the only ones that can hold ``NaN``."""
+    return native_dtype.upper() in ("FLOAT", "DOUBLE", "REAL", "FLOAT4", "FLOAT8")
+
+
 def _canonical_dtype(kind: str) -> str:
     """Map an arbitrary dtype string to freshdata's canonical buckets."""
     k = kind.lower()
@@ -50,6 +59,8 @@ class ColumnMetadata:
     is_numeric: bool = False
     is_string: bool = False
     sample_values: list[Any] = field(default_factory=list)
+    #: The backend's own type name (e.g. DuckDB ``DOUBLE``); empty when unknown.
+    native_dtype: str = ""
 
     @property
     def is_empty(self) -> bool:
@@ -135,34 +146,40 @@ def from_polars_lazy(lf: Any) -> list[ColumnMetadata]:
 
     @staticmethod
     def from_duckdb(conn: Any, table_name: str) -> list[ColumnMetadata]:
-        """Scan a registered DuckDB table/view via ``SUMMARIZE`` (no Python scan)."""
-        require_duckdb()
-        summary = conn.execute(f"SUMMARIZE {table_name}").fetchall()
-        cols = [d[0] for d in conn.execute(f"SUMMARIZE {table_name}").description]
-        idx = {name: i for i, name in enumerate(cols)}
+        """Scan a registered DuckDB table/view with one aggregate query (no Python scan).
 
-        (n,) = conn.execute(f"SELECT COUNT(*) FROM {table_name}").fetchone()
-        n = int(n)
+        Null counts are exact, and float ``NaN`` counts as missing as it does in
+        pandas. ``SUMMARIZE`` is not used: its ``stddev_samp`` raises on
+        non-finite floats and it only reports a rounded null percentage.
+        """
+        require_duckdb()
+        described = conn.execute(f"DESCRIBE {table_name}").fetchall()
+        columns = [(str(r[0]), str(r[1])) for r in described]
+        aggs = ["COUNT(*)"]
+        for name, native in columns:
+            col = _quote_identifier(name)
+            present = f"CASE WHEN NOT isnan({col}) THEN 1 END" if is_duckdb_float(native) else col
+            aggs.append(f"COUNT({present})")
+            aggs.append(f"approx_count_distinct({col})")
+        row = conn.execute(f"SELECT {', '.join(aggs)} FROM {table_name}").fetchone()
+        n = int(row[0])
 
         out: list[ColumnMetadata] = []
-        for r in summary:
-            name = r[idx["column_name"]]
-            dtype = str(r[idx["column_type"]])
-            null_pct = r[idx.get("null_percentage", -1)] if "null_percentage" in idx else None
-            null_ratio = float(null_pct) / 100.0 if null_pct is not None else 0.0
-            approx_unique = r[idx["approx_unique"]] if "approx_unique" in idx else -1
-            canonical = _canonical_dtype(dtype)
-            non_null = int(round(n * (1.0 - null_ratio)))
+        for i, (name, native) in enumerate(columns):
+            non_null = int(row[1 + 2 * i])
+            approx_unique = row[2 + 2 * i]
+            canonical = _canonical_dtype(native)
             out.append(
                 ColumnMetadata(
-                    name=str(name),
+                    name=name,
                     dtype_str=canonical,
                     row_count=n,
-                    null_ratio=null_ratio,
+                    null_ratio=0.0 if n == 0 else 1.0 - non_null / n,
                     non_null_count=non_null,
                     n_unique=int(approx_unique) if approx_unique is not None else -1,
                     is_numeric=canonical in ("int64", "float64"),
                     is_string=canonical == "string",
+                    native_dtype=native,
                 )
             )
         return out

src/freshdata/execution/_report.py

@@ -78,6 +78,24 @@ def finalize_report(report: CleanReport, cleaned: Any, started: float) -> CleanR
     return report
 
 
+def zero_column_frame(backend: str, output_format: str, report: CleanReport) -> Any:
+    """pandas result for a native run that dropped every column as empty.
+
+    The pandas pipeline keeps the row count on a zero-column frame, so the native
+    backends return the same. Polars and Arrow frames cannot hold rows without
+    columns; for those output formats the difference is recorded on the report.
+    """
+    import pandas as pd
+
+    if output_format != "pandas":
+        report.record_backend_difference(
+            backend, "drop_empty_columns",
+            f"every column was empty; a zero-column {output_format} result cannot carry "
+            f"the row count ({report.rows_before} row(s) in pandas output)",
+        )
+    return pd.DataFrame(index=pd.RangeIndex(report.rows_before))
+
+
 def finalize_report_native(report: CleanReport, started: float) -> CleanReport:
     """Finalize a report whose result was returned as a native, un-materialized
     handle (a DuckDB relation or a Polars ``LazyFrame``).