From 301d5382f578cb94365e023aeebbe007316401df Mon Sep 17 00:00:00 2001
From: Steve Vaughan Jr <s_vaughan@apple.com>
Date: Fri, 21 Aug 2026 13:55:26 -0400
Subject: [PATCH] fix: ignore reader-side parquet.hadoop.vectored.io.enabled in
 Iceberg native-write detection
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Avoid mis-classifying Compatible Iceberg V2 writes as Unsupported due to
well-known parquet reader settings.

CometIcebergNativeWrite's `requireNoParquetHadoopConfOverrides` (added
in #5298) walks the entire session Hadoop Configuration and returns an
`Unsupported` fall-back reason for any key starting with `parquet.`.
The intent is sound (a `parquet.*` key in the Hadoop conf reaches
iceberg-java's writer but not Comet's native writer), but the
predicate is too broad: it checks reader-side `parquet.*` keys the
same as writer-side ones.

`parquet.hadoop.vectored.io.enabled` is a reader-side vectored-IO
knob:

- Declared in parquet-hadoop as
  `org.apache.parquet.hadoop.ParquetInputFormat.HADOOP_VECTORED_IO_ENABLED`
  with default `true` in parquet-hadoop 1.16+.
- Only consulted by parquet-mr's Hadoop reader path (via
  `HadoopReadOptions.useHadoopVectoredIo`); iceberg-java's writer
  never reads it.

Any environment that seeds it into the session Hadoop configuration
trips this rule, and every otherwise-Compatible Iceberg V2 write is
silently mis-classified as Unsupported — the native writer is
disabled without the user having changed any writer-relevant setting.

Add an `IgnoredHadoopParquetConfKeys` set containing
`parquet.hadoop.vectored.io.enabled` and consult it inside the
`parquet.*` walk before emitting the fall-back reason. The broad
`startsWith("parquet.")` gate remains in place for every
writer-relevant knob.

Add a regression test that pins
`parquet.hadoop.vectored.io.enabled=true` in the session Hadoop
configuration via `withSQLConf` and asserts Compatible. The existing
negative test ("fall-back: parquet.* key in the session Hadoop
configuration", keyed on `parquet.block.size`) continues to exercise
the rejection path for real writer-relevant `parquet.*` keys.
---
 .../serde/operator/CometIcebergNativeWrite.scala   | 13 ++++++++++++-
 .../comet/CometIcebergWriteDetectionSuite.scala    | 14 ++++++++++++++
 2 files changed, 26 insertions(+), 1 deletion(-)

diff --git a/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala b/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala
index 5867bbb3f79..b6fc9fd62bf 100644
--- a/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala
+++ b/spark/src/main/scala/org/apache/comet/serde/operator/CometIcebergNativeWrite.scala
@@ -89,6 +89,16 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] with
   private val ParquetWritePropertyPrefix = "write.parquet."
   private val ParquetMrPropertyPrefix = "parquet."
 
+  // Hadoop-side `parquet.*` keys that iceberg-java's writer never consumes, so seeing them
+  // in the session Hadoop configuration does not indicate the native writer would diverge.
+  // `parquet.hadoop.vectored.io.enabled` is a reader-side vectored-IO knob declared by
+  // parquet-hadoop as `ParquetInputFormat.HADOOP_VECTORED_IO_ENABLED` (default `true` in
+  // parquet-hadoop 1.16+) and only consulted by parquet-mr's Hadoop reader path. Keep it
+  // out of the writer-compatibility gate so that environments which seed it into the
+  // session Hadoop configuration do not silently disable native Iceberg writes.
+  private val IgnoredHadoopParquetConfKeys: Set[String] = Set(
+    "parquet.hadoop.vectored.io.enabled")
+
   private lazy val vettedParquetWriteKeys: Set[String] = Set(
     PropertyKeys.ParquetCompressionCodec,
     PropertyKeys.ParquetCompressionLevel,
@@ -271,7 +281,8 @@ object CometIcebergNativeWrite extends CometOperatorSerde[IcebergWriteExec] with
   private val requireNoParquetHadoopConfOverrides: TriggerRule = ctx =>
     ctx.hadoopConf.asScala
       .map(_.getKey)
-      .find(_.startsWith(ParquetMrPropertyPrefix))
+      .filter(_.startsWith(ParquetMrPropertyPrefix))
+      .find(k => !IgnoredHadoopParquetConfKeys.contains(k))
       .map(k => s"Hadoop configuration sets $k (reaches iceberg-java's writer but not native)")
 
   private val requireSupportedStorageScheme: TriggerRule = ctx =>
diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala
index 71da5160f79..264b8d26171 100644
--- a/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala
+++ b/spark/src/test/scala/org/apache/comet/CometIcebergWriteDetectionSuite.scala
@@ -357,6 +357,20 @@ class CometIcebergWriteDetectionSuite extends CometTestBase with CometIcebergTes
     }
   }
 
+  // parquet.hadoop.vectored.io.enabled is a reader-side vectored-IO knob declared by
+  // parquet-hadoop (ParquetInputFormat.HADOOP_VECTORED_IO_ENABLED, default true in
+  // parquet-hadoop 1.16+). iceberg-java's writer never consumes it, so it must not
+  // disable native Iceberg writes when it happens to be present in the session
+  // Hadoop configuration.
+  test("Compatible when only parquet.hadoop.vectored.io.enabled is set in Hadoop configuration") {
+    withDetectionCatalog { dir =>
+      createTable(dir, "vectored_io_only", partitionSpec = "")
+      withSQLConf("parquet.hadoop.vectored.io.enabled" -> "true") {
+        assertSupportLevelIs[Compatible]("vectored_io_only")
+      }
+    }
+  }
+
   test("fall-back: io-impl set") {
     withDetectionCatalog { dir =>
       createTable(