From 6b2d0a605fd5e8e43ae098a3dd43d2d10047783b Mon Sep 17 00:00:00 2001 From: Muhammad Ali Siddiqui Date: Sun, 20 Sep 2026 21:39:23 +0500 Subject: [PATCH 1/2] fix: handle zero-column DuckDB sources --- src/freshdata/execution/backends/_duckdb.py | 15 ++++++++++++++- tests/test_execution/test_duckdb_engine.py | 6 ++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/src/freshdata/execution/backends/_duckdb.py b/src/freshdata/execution/backends/_duckdb.py index 169200e0..507ec950 100644 --- a/src/freshdata/execution/backends/_duckdb.py +++ b/src/freshdata/execution/backends/_duckdb.py @@ -42,6 +42,7 @@ ) from .._plan import PlanGenerator from .._report import finalize_report, finalize_report_native, init_report, zero_column_frame +from ...report import CleanReport from .._spill import create_run_spill_dir, remove_run_spill_dir from ._pandas import materialize_to_pandas @@ -49,7 +50,6 @@ import pandas as pd from ...config import CleanConfig - from ...report import CleanReport from .._config import EngineConfig log = logging.getLogger("freshdata.execution.duckdb") @@ -147,6 +147,19 @@ def execute( plan_cols = self._peek_columns(source) plan = PlanGenerator(config).plan(plan_cols) + + if not plan_cols: + started = time.perf_counter() + report = CleanReport( + rows_before=len(source), + cols_before=0, + memory_before=self._memory_before(source), + ) + report.backend = "duckdb" + cleaned = zero_column_frame("duckdb", engine_config.output_format, report) + finalize_report(report, cleaned, started) + return cleaned, report + reason = plan.fallback_reason or pandas_ingest_fallback_reason(source, self.name) if reason is None and self._pandas_index_forces_fallback(source): reason = "pandas index semantics" diff --git a/tests/test_execution/test_duckdb_engine.py b/tests/test_execution/test_duckdb_engine.py index 604400fa..0e697cd3 100644 --- a/tests/test_execution/test_duckdb_engine.py +++ b/tests/test_execution/test_duckdb_engine.py @@ -52,6 +52,12 @@ def test_drop_empty_column(native_config): assert "empty" not in out.columns +def test_zero_column_dataframe(native_config): + df = pd.DataFrame(index=range(3)) + out = fd.clean(df, config=native_config, engine="duckdb") + assert out.shape == (3, 0) + + def test_drop_duplicates(native_config): df = pd.DataFrame({"a": [1, 1, 2, 2, 3], "b": ["x", "x", "y", "y", "z"]}) out = fd.clean(df, config=native_config, engine="duckdb", drop_duplicates=True) From c49cb4694e6c88816bbbd3ae1f16e02896c97ce8 Mon Sep 17 00:00:00 2001 From: Kevin Costner <120246174+kevincostner17@users.noreply.github.com> Date: Mon, 21 Sep 2026 00:01:41 +0530 Subject: [PATCH 2/2] fix(duckdb): route a zero-column source through the disclosed pandas fallback The zero-column short-circuit ran ahead of the backend's fallback chain, so it bypassed both `pandas_ingest_fallback_reason` and `_pandas_index_forces_fallback`. Three consequences: - A zero-column frame with a non-RangeIndex worked before this change (the index guard routed it to pandas and the labels survived) and came back with a rebuilt RangeIndex after it, silently dropping the labels and breaking the parity the `native_config` fixture exists to enforce. - `output_format="duckdb"` still raised, now a RuntimeError from `_require_recorded_fallback`, since a materialized frame may stand in for a native handle only after a fallback the report discloses. - `rows_before=len(source)` is only meaningful for pandas: a zero-column LazyFrame raised TypeError and a parquet path would have passed its character count off as a row count. A source DuckDB cannot register is an ingest limitation, so name it as one and let the existing chain handle it. The pandas reference already keeps the rows and the index on a zero-column frame, and the fallback is disclosed in the report and still blocked by `fallback_policy="error"`. Fixes #457 Co-Authored-By: Claude Opus 5 (1M context) --- src/freshdata/execution/backends/_duckdb.py | 21 ++++-------- tests/test_execution/test_duckdb_engine.py | 36 +++++++++++++++++++++ 2 files changed, 43 insertions(+), 14 deletions(-) diff --git a/src/freshdata/execution/backends/_duckdb.py b/src/freshdata/execution/backends/_duckdb.py index 507ec950..b69d2e33 100644 --- a/src/freshdata/execution/backends/_duckdb.py +++ b/src/freshdata/execution/backends/_duckdb.py @@ -42,7 +42,6 @@ ) from .._plan import PlanGenerator from .._report import finalize_report, finalize_report_native, init_report, zero_column_frame -from ...report import CleanReport from .._spill import create_run_spill_dir, remove_run_spill_dir from ._pandas import materialize_to_pandas @@ -50,6 +49,7 @@ import pandas as pd from ...config import CleanConfig + from ...report import CleanReport from .._config import EngineConfig log = logging.getLogger("freshdata.execution.duckdb") @@ -147,20 +147,13 @@ def execute( plan_cols = self._peek_columns(source) plan = PlanGenerator(config).plan(plan_cols) - - if not plan_cols: - started = time.perf_counter() - report = CleanReport( - rows_before=len(source), - cols_before=0, - memory_before=self._memory_before(source), - ) - report.backend = "duckdb" - cleaned = zero_column_frame("duckdb", engine_config.output_format, report) - finalize_report(report, cleaned, started) - return cleaned, report - reason = plan.fallback_reason or pandas_ingest_fallback_reason(source, self.name) + if reason is None and not plan_cols: + # DuckDB cannot register a frame without columns ("Need a DataFrame + # with at least one column"). The pandas reference keeps the rows and + # the index on a zero-column frame, so disclose the fallback and let + # it produce the result. + reason = "zero-column source" if reason is None and self._pandas_index_forces_fallback(source): reason = "pandas index semantics" if reason is not None: diff --git a/tests/test_execution/test_duckdb_engine.py b/tests/test_execution/test_duckdb_engine.py index 0e697cd3..45fe0cd2 100644 --- a/tests/test_execution/test_duckdb_engine.py +++ b/tests/test_execution/test_duckdb_engine.py @@ -58,6 +58,42 @@ def test_zero_column_dataframe(native_config): assert out.shape == (3, 0) +def test_zero_column_dataframe_discloses_the_pandas_fallback(native_config): + df = pd.DataFrame(index=range(3)) + out, report = fd.clean(df, config=native_config, engine="duckdb", return_report=True) + assert out.shape == (3, 0) + (event,) = report.fallback_events + assert "zero-column source" in event["fallback_reason"] + + +def test_zero_column_dataframe_keeps_a_non_range_index(native_config): + # The pandas reference keeps the rows *and* the index labels on a + # zero-column frame; the native backend must agree with it rather than + # rebuilding a RangeIndex. + df = pd.DataFrame(index=["a", "b", "c"]) + out = fd.clean(df, config=native_config, engine="duckdb") + reference = fd.clean(df, config=native_config, engine="pandas") + # check_frame_type=False: the pandas engine hands back a CleanResult wrapper + # while the fallback path unwraps to a plain DataFrame. Only the contents matter. + pd.testing.assert_frame_equal(out, reference, check_frame_type=False) + assert list(out.index) == ["a", "b", "c"] + + +def test_zero_column_dataframe_with_a_native_handle_request(native_config): + # A zero-column source cannot become a DuckDB relation at all; the + # disclosed fallback is what lets a materialized pandas frame through. + out, report = fd.clean( + pd.DataFrame(index=range(3)), + config=native_config, + engine="duckdb", + output_format="duckdb", + return_report=True, + ) + assert isinstance(out, pd.DataFrame) + assert out.shape == (3, 0) + assert report.fallback_events + + def test_drop_duplicates(native_config): df = pd.DataFrame({"a": [1, 1, 2, 2, 3], "b": ["x", "x", "y", "y", "z"]}) out = fd.clean(df, config=native_config, engine="duckdb", drop_duplicates=True)