diff --git a/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexPartitionReaderFactory.java b/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexPartitionReaderFactory.java index e187e4863b1..acf72a37d96 100644 --- a/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexPartitionReaderFactory.java +++ b/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexPartitionReaderFactory.java @@ -15,6 +15,7 @@ import org.apache.spark.sql.connector.read.InputPartition; import org.apache.spark.sql.connector.read.PartitionReader; import org.apache.spark.sql.connector.read.PartitionReaderFactory; +import org.apache.spark.sql.util.CaseInsensitiveStringMap; import org.apache.spark.sql.vectorized.ColumnarBatch; /** @@ -46,7 +47,10 @@ public PartitionReader createReader(InputPartition partition) { @Override public PartitionReader createColumnarReader(InputPartition partition) { - NativeRuntime.setWorkerThreads(Integer.parseInt(formatOptions.getOrDefault("vortex.workerThreads", "4"))); + // Spark lower-cases the keys of the options map it hands to Table#newScanBuilder, so the + // option arrives spelled vortex.workerthreads and a case-sensitive lookup never finds it. + CaseInsensitiveStringMap options = new CaseInsensitiveStringMap(formatOptions); + NativeRuntime.setWorkerThreads(options.getInt("vortex.workerThreads", 4)); VortexFilePartition spark = (VortexFilePartition) partition; return new VortexPartitionReader(spark, dataColumnNames, formatOptions, pushedPredicates); }