From 02466f949f5cfaead9bb170d9ea446e3673cbd4e Mon Sep 17 00:00:00 2001 From: jackylee Date: Tue, 4 Aug 2026 19:29:16 +0800 Subject: [PATCH] fix(spark): resolve vortex.workerThreads case-insensitively A read option set as vortex.workerThreads never reached the native runtime. VortexTable.newScanBuilder merges the CaseInsensitiveStringMap Spark hands it into the format options; that map lower-cases its keys, so the option arrives spelled vortex.workerthreads while VortexScanBuilder seeds the camelCase spelling with the default of 4. The case-sensitive lookup always read the default one, silently ignoring the user's value. Wrap the options in a CaseInsensitiveStringMap and read through getInt. Signed-off-by: jackylee --- .../dev/vortex/spark/read/VortexPartitionReaderFactory.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexPartitionReaderFactory.java b/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexPartitionReaderFactory.java index e187e4863b1..acf72a37d96 100644 --- a/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexPartitionReaderFactory.java +++ b/java/vortex-spark/src/main/java/dev/vortex/spark/read/VortexPartitionReaderFactory.java @@ -15,6 +15,7 @@ import org.apache.spark.sql.connector.read.InputPartition; import org.apache.spark.sql.connector.read.PartitionReader; import org.apache.spark.sql.connector.read.PartitionReaderFactory; +import org.apache.spark.sql.util.CaseInsensitiveStringMap; import org.apache.spark.sql.vectorized.ColumnarBatch; /** @@ -46,7 +47,10 @@ public PartitionReader createReader(InputPartition partition) { @Override public PartitionReader createColumnarReader(InputPartition partition) { - NativeRuntime.setWorkerThreads(Integer.parseInt(formatOptions.getOrDefault("vortex.workerThreads", "4"))); + // Spark lower-cases the keys of the options map it hands to Table#newScanBuilder, so the + // option arrives spelled vortex.workerthreads and a case-sensitive lookup never finds it. + CaseInsensitiveStringMap options = new CaseInsensitiveStringMap(formatOptions); + NativeRuntime.setWorkerThreads(options.getInt("vortex.workerThreads", 4)); VortexFilePartition spark = (VortexFilePartition) partition; return new VortexPartitionReader(spark, dataColumnNames, formatOptions, pushedPredicates); }