fix: reject duplicate Parquet field names before decoding

docs/source/user-guide/latest/compatibility/scans.md

@@ -62,6 +62,20 @@ The following limitation may produce incorrect results without falling back to S
 
 The following limitations raise an error at scan time rather than falling back to Spark:
 
+- Selecting a field by name when multiple physical siblings match, including inside structs,
+  arrays, and maps. Comet raises a duplicate-field error instead of resolving the collision.
+  Checks cover referenced columns, including predicates; unselected roots do not prevent
+  reading a unique field by name or field ID. Exact-name projections of unique children in
+  structs and arrays of structs remain supported. Casts that cannot use this pruning reject
+  byte-identical duplicate siblings anywhere in the decoded physical subtree, including maps.
+  Field-ID resolution retains precedence, but selecting a byte-identically duplicated physical
+  root name still raises a duplicate-field error, even when the requested field is renamed.
+  Names in separate groups do not collide. Spark may read a duplicate-bearing file with an
+  explicit schema in case-sensitive mode, but its choice of sibling depends on the field shape
+  and can produce unexpected values. Spark rejects schema inference from a single file with
+  duplicate names; inference across files can depend on merge order.
+  Resolution is tracked in [#5884](https://github.com/apache/datafusion-comet/issues/5884),
+  with mixed-type behavior in [#5964](https://github.com/apache/datafusion-comet/issues/5964).
 - Invalid UTF-8 bytes in `STRING` columns. Spark permits arbitrary byte sequences in a `STRING`
   column (for example from `CAST(X'C1' AS STRING)`), but Comet's native execution path is built on
   Arrow, whose string type is strictly UTF-8. Reading a Parquet file whose `STRING` column contains

native/core/src/execution/operators/iceberg_scan.rs

@@ -619,6 +619,61 @@ mod tests {
         FileIOBuilder::new(Arc::new(OpenDalStorageFactory::Fs)).build()
     }
 
+    #[test]
+    fn issue_5783_projection_rejects_selected_duplicate_root() {
+        use arrow::array::Int64Array;
+        use arrow::datatypes::{DataType, Field, Schema as ArrowSchema};
+        use datafusion_physical_expr_adapter::PhysicalExprAdapterFactory;
+
+        let physical = Arc::new(ArrowSchema::new(vec![
+            Field::new("a", DataType::Int64, false),
+            Field::new("a", DataType::Int64, false),
+            Field::new("b", DataType::Int64, false),
+        ]));
+        let mut options = super::SparkParquetOptions::new(super::EvalMode::Legacy, "UTC", false);
+        options.case_sensitive = true;
+        let factory = super::SparkPhysicalExprAdapterFactory::new(options, None);
+        for name in ["a", "b"] {
+            let target = Arc::new(ArrowSchema::new(vec![Field::new(
+                name,
+                DataType::Int64,
+                false,
+            )]));
+            let adapter = factory
+                .create(Arc::clone(&target), Arc::clone(&physical))
+                .unwrap();
+            let result = super::build_projection_expressions(&target, &adapter);
+            if name == "a" {
+                let error = result
+                    .expect_err("selected root must be ambiguous")
+                    .to_string();
+                assert!(error.contains("duplicate"), "{error}");
+            } else {
+                let batch = super::RecordBatch::try_new(
+                    Arc::clone(&physical),
+                    vec![
+                        Arc::new(Int64Array::from(vec![1])),
+                        Arc::new(Int64Array::from(vec![2])),
+                        Arc::new(Int64Array::from(vec![3])),
+                    ],
+                )
+                .unwrap();
+                let output =
+                    super::adapt_batch_with_expressions(batch, &target, &result.unwrap()).unwrap();
+                assert_eq!(output.num_rows(), 1);
+                assert_eq!(
+                    output
+                        .column(0)
+                        .as_any()
+                        .downcast_ref::<Int64Array>()
+                        .unwrap()
+                        .value(0),
+                    3
+                );
+            }
+        }
+    }
+
     fn task_with_deletes(deletes: Vec<FileScanTaskDeleteFile>) -> FileScanTask {
         FileScanTask::builder()
             .with_file_size_in_bytes(0)

native/core/src/parquet/parquet_exec.rs

@@ -177,7 +177,7 @@ pub(crate) fn init_datasource_exec(
     // `store_sales`), the page index is re-fetched, uncached, on every open (comet#3978).
     // `EagerPageIndexReaderFactory` forces the page index to load on the first fetch and be
     // cached with the footer, at the cost of losing the skip's benefit when it would have
-    // applied. Filed upstream as apache/datafusion#23978; revert this once that's fixed.
+    // applied. Filed upstream as apache/datafusion#23978.
     //
     // Preserve bytes_scanned's existing requested data/Bloom-filter range accounting. Footer
     // and page-index reads through get_metadata bypass it, and coalescing may fetch extra bytes.

native/core/src/parquet/parquet_support.rs

@@ -61,6 +61,10 @@ use super::objectstore::s3_blob_fs_support::{
     normalize_object_store_url, NormalizedObjectStoreUrl,
 };
 
+pub(crate) fn duplicate_parquet_field_error(name: &str) -> DataFusionError {
+    DataFusionError::Execution(format!("Found duplicate Parquet field name '{name}'"))
+}
+
 // This file originates from cast.rs. While developing native scan support and implementing
 // SparkSchemaAdapter we observed that Spark's type conversion logic on Parquet reads does not
 // always align to the CAST expression's logic, so it was duplicated here to adapt its behavior.
@@ -420,8 +424,8 @@ fn field_id(field: &arrow::datatypes::Field) -> Option<i32> {
 /// ID-bearing requested fields match ONLY by ID (a missing ID is a missing column, never a name
 /// fallback); other fields match by name, folded with the same `toLowerCase(Locale.ROOT)` fold
 /// the top-level schema adapter uses when `case_sensitive` is false. A requested field whose
-/// folded name matches more than one file field in case-insensitive mode raises Spark's
-/// `foundDuplicateFieldInCaseInsensitiveModeError`.
+/// folded name matches more than one file field is rejected. Case-insensitive matching retains
+/// Spark's `foundDuplicateFieldInCaseInsensitiveModeError`.
 ///
 /// Shared by the runtime convert (`parquet_convert_struct_to_struct`) and the plan-time
 /// conversion check in `schema_adapter`, so both resolve nested fields identically.
@@ -473,14 +477,11 @@ pub(crate) fn match_struct_fields(
                 // falling back to name match.
                 (true, Some(id)) => Ok(from_id_to_index.get(&id).copied()),
                 _ => match folded_to_indices.get(to_folded[to_pos].as_str()) {
-                    // Mirror Spark's `foundDuplicateFieldInCaseInsensitiveModeError`: a
-                    // requested field matching more than one file field is ambiguous. Gated on
-                    // case-insensitive mode to match the top-level check (which only runs when
-                    // `!case_sensitive`): when case-sensitive the fold is identity, so a
-                    // collision means byte-identical sibling names, and raising an error whose
-                    // message says "in case-insensitive mode" would be wrong. Fall through to
-                    // the first match in that case.
-                    Some(indices) if indices.len() > 1 && !parquet_options.case_sensitive => {
+                    // Reject selected ambiguity before a decoder can multiply rows.
+                    Some(indices) if indices.len() > 1 => {
+                        if parquet_options.case_sensitive {
+                            return Err(duplicate_parquet_field_error(to_field.name()));
+                        }
                         let matched: Vec<&str> = indices
                             .iter()
                             .map(|&i| from_fields[i].name().as_str())