From 2a61ac1f16fa54c38c9cfcfd28cdf00a219bfac1 Mon Sep 17 00:00:00 2001 From: Selim Soufargi Date: Thu, 18 Jun 2026 22:32:09 +0200 Subject: [PATCH 01/30] detect source table type instead of setting it manually in the config --- .../xtable/conversion/DetectSourceType.java | 65 +++++++++++++++++++ .../xtable/conversion/ExternalTable.java | 2 +- .../apache/xtable/conversion/SourceTable.java | 12 +++- 3 files changed, 76 insertions(+), 3 deletions(-) create mode 100644 xtable-api/src/main/java/org/apache/xtable/conversion/DetectSourceType.java diff --git a/xtable-api/src/main/java/org/apache/xtable/conversion/DetectSourceType.java b/xtable-api/src/main/java/org/apache/xtable/conversion/DetectSourceType.java new file mode 100644 index 000000000..c25b9ca15 --- /dev/null +++ b/xtable-api/src/main/java/org/apache/xtable/conversion/DetectSourceType.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.xtable.conversion; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; + +public class DetectSourceType { + // helper method to detect input format + static String safeDetectFormat(String basePath, CatalogConfig catalogConfig) { + try { + Configuration conf = new Configuration(); + // map props to hadoop's + if (catalogConfig != null && catalogConfig.getCatalogOptions() != null) { + catalogConfig.getCatalogOptions().forEach(conf::set); + } + return DetectSourceType.detectFormat(ExternalTable.sanitizeBasePath(basePath), conf); + } catch (IOException e) { + return "UNKNOWN"; + } + } + + public static String detectFormat(String pathStr, Configuration conf) throws IOException { + Path basePath = new Path(pathStr); + FileSystem fs = basePath.getFileSystem(conf); + + if (!fs.exists(basePath)) { + return "UNKNOWN"; + } + + if (fs.exists(new Path(basePath, "_delta_log"))) { + return "DELTA"; + } + + if (fs.exists(new Path(basePath, ".hoodie"))) { + return "HUDI"; + } + + Path icebergMetaDir = new Path(basePath, "metadata"); + if (fs.exists(icebergMetaDir)) { + return "ICEBERG"; + } + + return "UNKNOWN"; + } +} diff --git a/xtable-api/src/main/java/org/apache/xtable/conversion/ExternalTable.java b/xtable-api/src/main/java/org/apache/xtable/conversion/ExternalTable.java index a6c97d8fa..8a4450b79 100644 --- a/xtable-api/src/main/java/org/apache/xtable/conversion/ExternalTable.java +++ b/xtable-api/src/main/java/org/apache/xtable/conversion/ExternalTable.java @@ -65,7 +65,7 @@ class ExternalTable { this.additionalProperties = additionalProperties; } - protected String sanitizeBasePath(String tableBasePath) { + public static String sanitizeBasePath(String tableBasePath) { Path path = new Path(tableBasePath); Preconditions.checkArgument(path.isAbsolute(), "Table base path must be absolute"); if (path.isAbsoluteAndSchemeAuthorityNull()) { diff --git a/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTable.java b/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTable.java index f3e1c3599..702ba13a7 100644 --- a/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTable.java +++ b/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTable.java @@ -34,13 +34,21 @@ public class SourceTable extends ExternalTable { @Builder(toBuilder = true) public SourceTable( String name, - String formatName, + String formatName, // can be omitted in the yaml config file String basePath, String dataPath, String[] namespace, CatalogConfig catalogConfig, Properties additionalProperties) { - super(name, formatName, basePath, namespace, catalogConfig, additionalProperties); + super( + name, + formatName != null + ? formatName + : DetectSourceType.safeDetectFormat(basePath, catalogConfig), + basePath, + namespace, + catalogConfig, + additionalProperties); this.dataPath = dataPath == null ? this.getBasePath() : sanitizeBasePath(dataPath); } } From d6174c4f26cdf5aa9de5a1e2c8187b0eda83e8c5 Mon Sep 17 00:00:00 2001 From: Selim Soufargi Date: Thu, 18 Jun 2026 22:58:00 +0200 Subject: [PATCH 02/30] test --- .../apache/xtable/utilities/ITRunSync.java | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/xtable-utilities/src/test/java/org/apache/xtable/utilities/ITRunSync.java b/xtable-utilities/src/test/java/org/apache/xtable/utilities/ITRunSync.java index f18ce867f..fbb06fef0 100644 --- a/xtable-utilities/src/test/java/org/apache/xtable/utilities/ITRunSync.java +++ b/xtable-utilities/src/test/java/org/apache/xtable/utilities/ITRunSync.java @@ -93,6 +93,21 @@ void testContinuousSyncMode(@TempDir Path tempDir) throws IOException { } } + @Test + void testSingleSyncModeWithoutInputTableFormat(@TempDir Path tempDir) throws IOException { + String tableName = "test-table"; + try (GenericTable table = + TestJavaHudiTable.forStandardSchema( + tableName, tempDir, null, HoodieTableType.COPY_ON_WRITE)) { + table.insertRows(10); + File configFile = writeConfigFileWithoutSourceTableFormat(tempDir, table, tableName); + String[] args = new String[] {"--datasetConfig", configFile.getPath()}; + RunSync.main(args); + Path icebergMetadataPath = Paths.get(URI.create(table.getBasePath() + "/metadata")); + waitForNumIcebergCommits(icebergMetadataPath, 3); + } + } + private static File writeConfigFile(Path tempDir, GenericTable table, String tableName) throws IOException { RunSync.DatasetConfig config = @@ -111,6 +126,23 @@ private static File writeConfigFile(Path tempDir, GenericTable table, String tab return configFile; } + private static File writeConfigFileWithoutSourceTableFormat( + Path tempDir, GenericTable table, String tableName) throws IOException { + RunSync.DatasetConfig config = + RunSync.DatasetConfig.builder() + .targetFormats(Collections.singletonList("ICEBERG")) + .datasets( + Collections.singletonList( + RunSync.DatasetConfig.Table.builder() + .tableBasePath(table.getBasePath()) + .tableName(tableName) + .build())) + .build(); + File configFile = new File(tempDir + "config.yaml"); + RunSync.YAML_MAPPER.writeValue(configFile, config); + return configFile; + } + @SneakyThrows private static void waitForNumIcebergCommits(Path metadataPath, int count) { long start = System.currentTimeMillis(); From 4495f8a6857b91734fef1f7649cb93ac74fabfdc Mon Sep 17 00:00:00 2001 From: Selim Soufargi Date: Thu, 18 Jun 2026 23:52:44 +0200 Subject: [PATCH 03/30] fix test --- .../xtable/conversion/DetectSourceType.java | 3 +- .../org/apache/xtable/utilities/RunSync.java | 33 +++++++++++-------- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/xtable-api/src/main/java/org/apache/xtable/conversion/DetectSourceType.java b/xtable-api/src/main/java/org/apache/xtable/conversion/DetectSourceType.java index c25b9ca15..389a23dc9 100644 --- a/xtable-api/src/main/java/org/apache/xtable/conversion/DetectSourceType.java +++ b/xtable-api/src/main/java/org/apache/xtable/conversion/DetectSourceType.java @@ -40,7 +40,8 @@ static String safeDetectFormat(String basePath, CatalogConfig catalogConfig) { } public static String detectFormat(String pathStr, Configuration conf) throws IOException { - Path basePath = new Path(pathStr); + String sanitizeBasePath = ExternalTable.sanitizeBasePath(pathStr); + Path basePath = new Path(sanitizeBasePath); FileSystem fs = basePath.getFileSystem(conf); if (!fs.exists(basePath)) { diff --git a/xtable-utilities/src/main/java/org/apache/xtable/utilities/RunSync.java b/xtable-utilities/src/main/java/org/apache/xtable/utilities/RunSync.java index facbcf3a6..1dc692c66 100644 --- a/xtable-utilities/src/main/java/org/apache/xtable/utilities/RunSync.java +++ b/xtable-utilities/src/main/java/org/apache/xtable/utilities/RunSync.java @@ -52,12 +52,7 @@ import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import com.google.common.annotations.VisibleForTesting; -import org.apache.xtable.conversion.CatalogConfig; -import org.apache.xtable.conversion.ConversionConfig; -import org.apache.xtable.conversion.ConversionController; -import org.apache.xtable.conversion.ConversionSourceProvider; -import org.apache.xtable.conversion.SourceTable; -import org.apache.xtable.conversion.TargetTable; +import org.apache.xtable.conversion.*; import org.apache.xtable.hudi.HudiSourceConfig; import org.apache.xtable.iceberg.IcebergCatalogConfig; import org.apache.xtable.model.storage.TableFormat; @@ -214,18 +209,30 @@ static CatalogConfig getIcebergCatalogConfig(String icebergCatalogConfigPath) th static ConversionSourceProvider getConversionSourceProvider( String conversionProviderConfigpath, DatasetConfig datasetConfig, Configuration hadoopConf) throws IOException { - // Process source format String sourceFormat = datasetConfig.sourceFormat; byte[] customConfig = getCustomConfigurations(conversionProviderConfigpath); TableFormatConverters tableFormatConverters = loadTableFormatConversionConfigs(customConfig); + if (sourceFormat == null + && datasetConfig.getDatasets() != null + && !datasetConfig.getDatasets().isEmpty()) { + DatasetConfig.Table firstTable = datasetConfig.getDatasets().get(0); + if (firstTable.getTableBasePath() != null) { + try { + String tablePath = firstTable.getTableBasePath(); + sourceFormat = DetectSourceType.detectFormat(tablePath, hadoopConf); + log.info( + "Source format was omitted in config. Auto-detected table format: {}", sourceFormat); + } catch (Exception e) { + throw new IllegalArgumentException( + String.format( + "Source format %s is not supported. Known source and target formats are %s", + sourceFormat, tableFormatConverters.getTableFormatConverters().keySet())); + } + } + } + TableFormatConverters.ConversionConfig sourceConversionConfig = tableFormatConverters.getTableFormatConverters().get(sourceFormat); - if (sourceConversionConfig == null) { - throw new IllegalArgumentException( - String.format( - "Source format %s is not supported. Known source and target formats are %s", - sourceFormat, tableFormatConverters.getTableFormatConverters().keySet())); - } String sourceProviderClass = sourceConversionConfig.conversionSourceProviderClass; ConversionSourceProvider conversionSourceProvider = ReflectionUtils.createInstanceOfClass(sourceProviderClass); From 9d6f3898a56a693886bfc2a503a4534be9f8e1d6 Mon Sep 17 00:00:00 2001 From: Selim Soufargi Date: Fri, 19 Jun 2026 00:24:43 +0200 Subject: [PATCH 04/30] fix imports in test --- .../main/java/org/apache/xtable/utilities/RunSync.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/xtable-utilities/src/main/java/org/apache/xtable/utilities/RunSync.java b/xtable-utilities/src/main/java/org/apache/xtable/utilities/RunSync.java index 1dc692c66..2b4a4b271 100644 --- a/xtable-utilities/src/main/java/org/apache/xtable/utilities/RunSync.java +++ b/xtable-utilities/src/main/java/org/apache/xtable/utilities/RunSync.java @@ -52,7 +52,13 @@ import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; import com.google.common.annotations.VisibleForTesting; -import org.apache.xtable.conversion.*; +import org.apache.xtable.conversion.CatalogConfig; +import org.apache.xtable.conversion.ConversionConfig; +import org.apache.xtable.conversion.ConversionController; +import org.apache.xtable.conversion.ConversionSourceProvider; +import org.apache.xtable.conversion.SourceTable; +import org.apache.xtable.conversion.DetectSourceType; +import org.apache.xtable.conversion.TargetTable; import org.apache.xtable.hudi.HudiSourceConfig; import org.apache.xtable.iceberg.IcebergCatalogConfig; import org.apache.xtable.model.storage.TableFormat; From 5830ee5632184d5d0f7995a74e50f3524de7d813 Mon Sep 17 00:00:00 2001 From: Selim Soufargi Date: Fri, 19 Jun 2026 00:50:12 +0200 Subject: [PATCH 05/30] fix imports in test --- .../src/main/java/org/apache/xtable/utilities/RunSync.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xtable-utilities/src/main/java/org/apache/xtable/utilities/RunSync.java b/xtable-utilities/src/main/java/org/apache/xtable/utilities/RunSync.java index 2b4a4b271..55b9bdac0 100644 --- a/xtable-utilities/src/main/java/org/apache/xtable/utilities/RunSync.java +++ b/xtable-utilities/src/main/java/org/apache/xtable/utilities/RunSync.java @@ -56,8 +56,8 @@ import org.apache.xtable.conversion.ConversionConfig; import org.apache.xtable.conversion.ConversionController; import org.apache.xtable.conversion.ConversionSourceProvider; -import org.apache.xtable.conversion.SourceTable; import org.apache.xtable.conversion.DetectSourceType; +import org.apache.xtable.conversion.SourceTable; import org.apache.xtable.conversion.TargetTable; import org.apache.xtable.hudi.HudiSourceConfig; import org.apache.xtable.iceberg.IcebergCatalogConfig; From a06de3f634c96fa70c51da46499b54ef1e8f1b47 Mon Sep 17 00:00:00 2001 From: Selim Soufargi Date: Fri, 19 Jun 2026 08:55:05 +0200 Subject: [PATCH 06/30] fixes --- xtable-api/pom.xml | 8 ++++++ .../xtable/conversion/DetectSourceType.java | 27 +++++++++---------- .../apache/xtable/conversion/SourceTable.java | 19 +++++++------ .../org/apache/xtable/utilities/RunSync.java | 8 +++--- 4 files changed, 36 insertions(+), 26 deletions(-) diff --git a/xtable-api/pom.xml b/xtable-api/pom.xml index 43aa7bace..08d39da44 100644 --- a/xtable-api/pom.xml +++ b/xtable-api/pom.xml @@ -88,5 +88,13 @@ org.mockito mockito-junit-jupiter + + org.apache.iceberg + iceberg-core + + + org.apache.iceberg + iceberg-api + diff --git a/xtable-api/src/main/java/org/apache/xtable/conversion/DetectSourceType.java b/xtable-api/src/main/java/org/apache/xtable/conversion/DetectSourceType.java index 389a23dc9..c22b39300 100644 --- a/xtable-api/src/main/java/org/apache/xtable/conversion/DetectSourceType.java +++ b/xtable-api/src/main/java/org/apache/xtable/conversion/DetectSourceType.java @@ -24,20 +24,11 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.iceberg.hadoop.HadoopTables; +import org.apache.iceberg.Table; + public class DetectSourceType { // helper method to detect input format - static String safeDetectFormat(String basePath, CatalogConfig catalogConfig) { - try { - Configuration conf = new Configuration(); - // map props to hadoop's - if (catalogConfig != null && catalogConfig.getCatalogOptions() != null) { - catalogConfig.getCatalogOptions().forEach(conf::set); - } - return DetectSourceType.detectFormat(ExternalTable.sanitizeBasePath(basePath), conf); - } catch (IOException e) { - return "UNKNOWN"; - } - } public static String detectFormat(String pathStr, Configuration conf) throws IOException { String sanitizeBasePath = ExternalTable.sanitizeBasePath(pathStr); @@ -56,9 +47,15 @@ public static String detectFormat(String pathStr, Configuration conf) throws IOE return "HUDI"; } - Path icebergMetaDir = new Path(basePath, "metadata"); - if (fs.exists(icebergMetaDir)) { - return "ICEBERG"; + // workaround for: .metadata can be set elsewhere + try { + HadoopTables tables = new HadoopTables(conf); + // if the path points to a valid Iceberg table (even with a custom metadata location), + Table table = tables.load(pathStr); + if (table != null) { + return "ICEBERG"; + } + } catch (Exception e) { } return "UNKNOWN"; diff --git a/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTable.java b/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTable.java index 702ba13a7..379db1b5d 100644 --- a/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTable.java +++ b/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTable.java @@ -18,12 +18,14 @@ package org.apache.xtable.conversion; +import java.io.IOException; import java.util.Properties; import lombok.Builder; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.NonNull; +import org.apache.hadoop.conf.Configuration; @EqualsAndHashCode(callSuper = true) @Getter @@ -33,18 +35,19 @@ public class SourceTable extends ExternalTable { @Builder(toBuilder = true) public SourceTable( - String name, - String formatName, // can be omitted in the yaml config file - String basePath, - String dataPath, - String[] namespace, - CatalogConfig catalogConfig, - Properties additionalProperties) { + String name, + String formatName, // can be omitted in the yaml config file + String basePath, + String dataPath, + String[] namespace, + CatalogConfig catalogConfig, + Properties additionalProperties, Configuration hadoopConf + ) throws IOException { super( name, formatName != null ? formatName - : DetectSourceType.safeDetectFormat(basePath, catalogConfig), + : DetectSourceType.detectFormat(basePath, hadoopConf), basePath, namespace, catalogConfig, diff --git a/xtable-utilities/src/main/java/org/apache/xtable/utilities/RunSync.java b/xtable-utilities/src/main/java/org/apache/xtable/utilities/RunSync.java index 55b9bdac0..e2c2a3582 100644 --- a/xtable-utilities/src/main/java/org/apache/xtable/utilities/RunSync.java +++ b/xtable-utilities/src/main/java/org/apache/xtable/utilities/RunSync.java @@ -122,7 +122,8 @@ static SourceTable sourceTableBuilder( @NonNull DatasetConfig.Table table, CatalogConfig catalogConfig, @NonNull DatasetConfig datasetConfig, - Properties sourceProperties) { + Properties sourceProperties, + Configuration hadoopConf) throws IOException { SourceTable sourceTable = SourceTable.builder() .name(table.getTableName()) @@ -132,6 +133,7 @@ static SourceTable sourceTableBuilder( .catalogConfig(catalogConfig) .additionalProperties(sourceProperties) .formatName(datasetConfig.sourceFormat) + .hadoopConf(hadoopConf) .build(); return sourceTable; } @@ -161,7 +163,7 @@ static void syncTableMetdata( List tableFormatList, CatalogConfig catalogConfig, Configuration hadoopConf, - ConversionSourceProvider conversionSourceProvider) { + ConversionSourceProvider conversionSourceProvider) throws IOException { ConversionController conversionController = new ConversionController(hadoopConf); for (DatasetConfig.Table table : datasetConfig.getDatasets()) { log.info( @@ -175,7 +177,7 @@ static void syncTableMetdata( } SourceTable sourceTable = - sourceTableBuilder(table, catalogConfig, datasetConfig, sourceProperties); + sourceTableBuilder(table, catalogConfig, datasetConfig, sourceProperties, hadoopConf); List targetTables = targetTableBuilder(table, catalogConfig, tableFormatList); ConversionConfig conversionConfig = ConversionConfig.builder() From 1d528bb7e33288bde78ba44351d65b4444ecb4b8 Mon Sep 17 00:00:00 2001 From: Selim Soufargi Date: Fri, 19 Jun 2026 08:56:26 +0200 Subject: [PATCH 07/30] spotless --- .../xtable/conversion/DetectSourceType.java | 2 +- .../apache/xtable/conversion/SourceTable.java | 22 +++++++++---------- .../org/apache/xtable/utilities/RunSync.java | 6 +++-- 3 files changed, 16 insertions(+), 14 deletions(-) diff --git a/xtable-api/src/main/java/org/apache/xtable/conversion/DetectSourceType.java b/xtable-api/src/main/java/org/apache/xtable/conversion/DetectSourceType.java index c22b39300..d552d1b4d 100644 --- a/xtable-api/src/main/java/org/apache/xtable/conversion/DetectSourceType.java +++ b/xtable-api/src/main/java/org/apache/xtable/conversion/DetectSourceType.java @@ -24,8 +24,8 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.iceberg.hadoop.HadoopTables; import org.apache.iceberg.Table; +import org.apache.iceberg.hadoop.HadoopTables; public class DetectSourceType { // helper method to detect input format diff --git a/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTable.java b/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTable.java index 379db1b5d..d5391c251 100644 --- a/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTable.java +++ b/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTable.java @@ -25,6 +25,7 @@ import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.NonNull; + import org.apache.hadoop.conf.Configuration; @EqualsAndHashCode(callSuper = true) @@ -35,19 +36,18 @@ public class SourceTable extends ExternalTable { @Builder(toBuilder = true) public SourceTable( - String name, - String formatName, // can be omitted in the yaml config file - String basePath, - String dataPath, - String[] namespace, - CatalogConfig catalogConfig, - Properties additionalProperties, Configuration hadoopConf - ) throws IOException { + String name, + String formatName, // can be omitted in the yaml config file + String basePath, + String dataPath, + String[] namespace, + CatalogConfig catalogConfig, + Properties additionalProperties, + Configuration hadoopConf) + throws IOException { super( name, - formatName != null - ? formatName - : DetectSourceType.detectFormat(basePath, hadoopConf), + formatName != null ? formatName : DetectSourceType.detectFormat(basePath, hadoopConf), basePath, namespace, catalogConfig, diff --git a/xtable-utilities/src/main/java/org/apache/xtable/utilities/RunSync.java b/xtable-utilities/src/main/java/org/apache/xtable/utilities/RunSync.java index e2c2a3582..95460bc5a 100644 --- a/xtable-utilities/src/main/java/org/apache/xtable/utilities/RunSync.java +++ b/xtable-utilities/src/main/java/org/apache/xtable/utilities/RunSync.java @@ -123,7 +123,8 @@ static SourceTable sourceTableBuilder( CatalogConfig catalogConfig, @NonNull DatasetConfig datasetConfig, Properties sourceProperties, - Configuration hadoopConf) throws IOException { + Configuration hadoopConf) + throws IOException { SourceTable sourceTable = SourceTable.builder() .name(table.getTableName()) @@ -163,7 +164,8 @@ static void syncTableMetdata( List tableFormatList, CatalogConfig catalogConfig, Configuration hadoopConf, - ConversionSourceProvider conversionSourceProvider) throws IOException { + ConversionSourceProvider conversionSourceProvider) + throws IOException { ConversionController conversionController = new ConversionController(hadoopConf); for (DatasetConfig.Table table : datasetConfig.getDatasets()) { log.info( From 096d0e947ab83dd31c606781b32d9d7225532831 Mon Sep 17 00:00:00 2001 From: Selim Soufargi Date: Fri, 19 Jun 2026 09:10:00 +0200 Subject: [PATCH 08/30] fix for hadoop conf in SourceTable --- .../apache/xtable/conversion/SourceTable.java | 58 ++++++++----------- 1 file changed, 24 insertions(+), 34 deletions(-) diff --git a/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTable.java b/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTable.java index d5391c251..5fd1665d0 100644 --- a/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTable.java +++ b/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTable.java @@ -1,24 +1,7 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - package org.apache.xtable.conversion; import java.io.IOException; +import java.io.UncheckedIOException; import java.util.Properties; import lombok.Builder; @@ -36,22 +19,29 @@ public class SourceTable extends ExternalTable { @Builder(toBuilder = true) public SourceTable( - String name, - String formatName, // can be omitted in the yaml config file - String basePath, - String dataPath, - String[] namespace, - CatalogConfig catalogConfig, - Properties additionalProperties, - Configuration hadoopConf) - throws IOException { + String name, + String formatName, + String basePath, + String dataPath, + String[] namespace, + CatalogConfig catalogConfig, + Properties additionalProperties, + Configuration hadoopConf) { super( - name, - formatName != null ? formatName : DetectSourceType.detectFormat(basePath, hadoopConf), - basePath, - namespace, - catalogConfig, - additionalProperties); + name, + formatName != null ? formatName : resolveFormatOrThrow(basePath, hadoopConf), + basePath, + namespace, + catalogConfig, + additionalProperties); this.dataPath = dataPath == null ? this.getBasePath() : sanitizeBasePath(dataPath); } -} + + private static String resolveFormatOrThrow(String basePath, Configuration hadoopConf) { + try { + return DetectSourceType.detectFormat(basePath, hadoopConf); + } catch (IOException e) { + throw new UncheckedIOException("Failed to auto-detect source table format", e); + } + } +} \ No newline at end of file From 843a358ec2060c36581d157fffdea49fec934530 Mon Sep 17 00:00:00 2001 From: Selim Soufargi Date: Fri, 19 Jun 2026 09:12:28 +0200 Subject: [PATCH 09/30] fix for hadoop conf in SourceTable --- .../apache/xtable/conversion/SourceTable.java | 20 ++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTable.java b/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTable.java index 5fd1665d0..d2fb61853 100644 --- a/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTable.java +++ b/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTable.java @@ -1,3 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.xtable.conversion; import java.io.IOException; @@ -36,7 +54,7 @@ public SourceTable( additionalProperties); this.dataPath = dataPath == null ? this.getBasePath() : sanitizeBasePath(dataPath); } - + private static String resolveFormatOrThrow(String basePath, Configuration hadoopConf) { try { return DetectSourceType.detectFormat(basePath, hadoopConf); From ba7b1fd0f0ec1c4ecad0c60adf1c0972875a0f6d Mon Sep 17 00:00:00 2001 From: Selim Soufargi Date: Fri, 19 Jun 2026 09:20:12 +0200 Subject: [PATCH 10/30] fix for hadoop conf in SourceTable --- .../apache/xtable/conversion/SourceTable.java | 35 ++++++++++--------- 1 file changed, 19 insertions(+), 16 deletions(-) diff --git a/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTable.java b/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTable.java index d2fb61853..3424a9a8f 100644 --- a/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTable.java +++ b/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTable.java @@ -15,7 +15,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - + package org.apache.xtable.conversion; import java.io.IOException; @@ -35,24 +35,27 @@ public class SourceTable extends ExternalTable { /** The path to the data files, defaults to the basePath */ @NonNull private final String dataPath; + private final transient Configuration hadoopConf; + @Builder(toBuilder = true) public SourceTable( - String name, - String formatName, - String basePath, - String dataPath, - String[] namespace, - CatalogConfig catalogConfig, - Properties additionalProperties, - Configuration hadoopConf) { + String name, + String formatName, + String basePath, + String dataPath, + String[] namespace, + CatalogConfig catalogConfig, + Properties additionalProperties, + Configuration conf) { super( - name, - formatName != null ? formatName : resolveFormatOrThrow(basePath, hadoopConf), - basePath, - namespace, - catalogConfig, - additionalProperties); + name, + formatName != null ? formatName : resolveFormatOrThrow(basePath, conf), + basePath, + namespace, + catalogConfig, + additionalProperties); this.dataPath = dataPath == null ? this.getBasePath() : sanitizeBasePath(dataPath); + this.hadoopConf = conf; } private static String resolveFormatOrThrow(String basePath, Configuration hadoopConf) { @@ -62,4 +65,4 @@ private static String resolveFormatOrThrow(String basePath, Configuration hadoop throw new UncheckedIOException("Failed to auto-detect source table format", e); } } -} \ No newline at end of file +} From c7aee87a546c434a985f57d0566ce04293dd3925 Mon Sep 17 00:00:00 2001 From: Selim Soufargi Date: Fri, 19 Jun 2026 18:22:33 +0200 Subject: [PATCH 11/30] fix for CI --- .../xtable/conversion/ExternalTable.java | 2 +- .../apache/xtable/conversion/SourceTable.java | 19 ++++++++++++++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/xtable-api/src/main/java/org/apache/xtable/conversion/ExternalTable.java b/xtable-api/src/main/java/org/apache/xtable/conversion/ExternalTable.java index 8a4450b79..268b37b32 100644 --- a/xtable-api/src/main/java/org/apache/xtable/conversion/ExternalTable.java +++ b/xtable-api/src/main/java/org/apache/xtable/conversion/ExternalTable.java @@ -65,7 +65,7 @@ class ExternalTable { this.additionalProperties = additionalProperties; } - public static String sanitizeBasePath(String tableBasePath) { + protected static String sanitizeBasePath(String tableBasePath) { Path path = new Path(tableBasePath); Preconditions.checkArgument(path.isAbsolute(), "Table base path must be absolute"); if (path.isAbsoluteAndSchemeAuthorityNull()) { diff --git a/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTable.java b/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTable.java index 3424a9a8f..29adc32a0 100644 --- a/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTable.java +++ b/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTable.java @@ -46,16 +46,29 @@ public SourceTable( String[] namespace, CatalogConfig catalogConfig, Properties additionalProperties, - Configuration conf) { + Configuration hadoopConf) { + super(name, formatName, basePath, namespace, catalogConfig, additionalProperties); + this.dataPath = dataPath == null ? this.getBasePath() : sanitizeBasePath(dataPath); + this.hadoopConf = hadoopConf; + } + + public SourceTable( + @NonNull String name, + @NonNull String basePath, + String dataPath, + String[] namespace, + CatalogConfig catalogConfig, + Properties additionalProperties, + Configuration hadoopConf) { super( name, - formatName != null ? formatName : resolveFormatOrThrow(basePath, conf), + resolveFormatOrThrow(basePath, hadoopConf), basePath, namespace, catalogConfig, additionalProperties); this.dataPath = dataPath == null ? this.getBasePath() : sanitizeBasePath(dataPath); - this.hadoopConf = conf; + this.hadoopConf = hadoopConf; } private static String resolveFormatOrThrow(String basePath, Configuration hadoopConf) { From e025289f5fa0d9f8cbfe2fd3031bb8fbdf0295a5 Mon Sep 17 00:00:00 2001 From: Selim Soufargi Date: Fri, 19 Jun 2026 18:33:32 +0200 Subject: [PATCH 12/30] fix for CI --- .../java/org/apache/xtable/conversion/SourceTable.java | 2 +- ...tSourceType.java => SourceTableFormatDetector.java} | 10 ++++++---- .../main/java/org/apache/xtable/utilities/RunSync.java | 4 ++-- 3 files changed, 9 insertions(+), 7 deletions(-) rename xtable-api/src/main/java/org/apache/xtable/conversion/{DetectSourceType.java => SourceTableFormatDetector.java} (90%) diff --git a/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTable.java b/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTable.java index 29adc32a0..e2dcf9790 100644 --- a/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTable.java +++ b/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTable.java @@ -73,7 +73,7 @@ public SourceTable( private static String resolveFormatOrThrow(String basePath, Configuration hadoopConf) { try { - return DetectSourceType.detectFormat(basePath, hadoopConf); + return SourceTableFormatDetector.detectFormat(basePath, hadoopConf); } catch (IOException e) { throw new UncheckedIOException("Failed to auto-detect source table format", e); } diff --git a/xtable-api/src/main/java/org/apache/xtable/conversion/DetectSourceType.java b/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTableFormatDetector.java similarity index 90% rename from xtable-api/src/main/java/org/apache/xtable/conversion/DetectSourceType.java rename to xtable-api/src/main/java/org/apache/xtable/conversion/SourceTableFormatDetector.java index d552d1b4d..c0b2fd7cb 100644 --- a/xtable-api/src/main/java/org/apache/xtable/conversion/DetectSourceType.java +++ b/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTableFormatDetector.java @@ -27,7 +27,9 @@ import org.apache.iceberg.Table; import org.apache.iceberg.hadoop.HadoopTables; -public class DetectSourceType { +import org.apache.xtable.model.storage.TableFormat; + +public class SourceTableFormatDetector { // helper method to detect input format public static String detectFormat(String pathStr, Configuration conf) throws IOException { @@ -40,11 +42,11 @@ public static String detectFormat(String pathStr, Configuration conf) throws IOE } if (fs.exists(new Path(basePath, "_delta_log"))) { - return "DELTA"; + return TableFormat.DELTA; } if (fs.exists(new Path(basePath, ".hoodie"))) { - return "HUDI"; + return TableFormat.HUDI; } // workaround for: .metadata can be set elsewhere @@ -53,7 +55,7 @@ public static String detectFormat(String pathStr, Configuration conf) throws IOE // if the path points to a valid Iceberg table (even with a custom metadata location), Table table = tables.load(pathStr); if (table != null) { - return "ICEBERG"; + return TableFormat.ICEBERG; } } catch (Exception e) { } diff --git a/xtable-utilities/src/main/java/org/apache/xtable/utilities/RunSync.java b/xtable-utilities/src/main/java/org/apache/xtable/utilities/RunSync.java index 95460bc5a..36c37f733 100644 --- a/xtable-utilities/src/main/java/org/apache/xtable/utilities/RunSync.java +++ b/xtable-utilities/src/main/java/org/apache/xtable/utilities/RunSync.java @@ -56,8 +56,8 @@ import org.apache.xtable.conversion.ConversionConfig; import org.apache.xtable.conversion.ConversionController; import org.apache.xtable.conversion.ConversionSourceProvider; -import org.apache.xtable.conversion.DetectSourceType; import org.apache.xtable.conversion.SourceTable; +import org.apache.xtable.conversion.SourceTableFormatDetector; import org.apache.xtable.conversion.TargetTable; import org.apache.xtable.hudi.HudiSourceConfig; import org.apache.xtable.iceberg.IcebergCatalogConfig; @@ -229,7 +229,7 @@ static ConversionSourceProvider getConversionSourceProvider( if (firstTable.getTableBasePath() != null) { try { String tablePath = firstTable.getTableBasePath(); - sourceFormat = DetectSourceType.detectFormat(tablePath, hadoopConf); + sourceFormat = SourceTableFormatDetector.detectFormat(tablePath, hadoopConf); log.info( "Source format was omitted in config. Auto-detected table format: {}", sourceFormat); } catch (Exception e) { From 416a38c2ffa4be10cee7065da521fc3f355874fc Mon Sep 17 00:00:00 2001 From: Selim Soufargi Date: Fri, 19 Jun 2026 19:03:45 +0200 Subject: [PATCH 13/30] fix for CI --- xtable-api/pom.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/xtable-api/pom.xml b/xtable-api/pom.xml index 08d39da44..5b09ad8de 100644 --- a/xtable-api/pom.xml +++ b/xtable-api/pom.xml @@ -29,6 +29,10 @@ XTable Project API + + org.apache.commons + commons-lang3 + com.fasterxml.jackson.core jackson-annotations From 4cbb1978ef392273902963facea4eb3904292e73 Mon Sep 17 00:00:00 2001 From: Selim Soufargi Date: Fri, 19 Jun 2026 20:40:26 +0200 Subject: [PATCH 14/30] create a second sourceTable constructor to handle null format arg in runsync --- .../xtable/conversion/ExternalTable.java | 7 +++- .../apache/xtable/conversion/SourceTable.java | 5 ++- .../apache/xtable/conversion/TargetTable.java | 7 +++- .../xtable/conversion/TestExternalTable.java | 15 +++++--- .../xtable/conversion/ConversionUtils.java | 3 +- .../org/apache/xtable/utilities/RunSync.java | 37 +++++++++++++------ 6 files changed, 50 insertions(+), 24 deletions(-) diff --git a/xtable-api/src/main/java/org/apache/xtable/conversion/ExternalTable.java b/xtable-api/src/main/java/org/apache/xtable/conversion/ExternalTable.java index 268b37b32..c75eaf522 100644 --- a/xtable-api/src/main/java/org/apache/xtable/conversion/ExternalTable.java +++ b/xtable-api/src/main/java/org/apache/xtable/conversion/ExternalTable.java @@ -24,6 +24,7 @@ import lombok.Getter; import lombok.NonNull; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import com.google.common.base.Preconditions; @@ -50,19 +51,23 @@ class ExternalTable { /** Optional, additional properties that can be used to define interactions with the table */ protected final Properties additionalProperties; + protected final Configuration hadoopConf; + ExternalTable( @NonNull String name, @NonNull String formatName, @NonNull String basePath, String[] namespace, CatalogConfig catalogConfig, - Properties additionalProperties) { + Properties additionalProperties, + Configuration hadoopConf) { this.name = name; this.formatName = formatName; this.basePath = sanitizeBasePath(basePath); this.namespace = namespace; this.catalogConfig = catalogConfig; this.additionalProperties = additionalProperties; + this.hadoopConf = hadoopConf; } protected static String sanitizeBasePath(String tableBasePath) { diff --git a/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTable.java b/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTable.java index e2dcf9790..f287ed5b4 100644 --- a/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTable.java +++ b/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTable.java @@ -47,7 +47,7 @@ public SourceTable( CatalogConfig catalogConfig, Properties additionalProperties, Configuration hadoopConf) { - super(name, formatName, basePath, namespace, catalogConfig, additionalProperties); + super(name, formatName, basePath, namespace, catalogConfig, additionalProperties, hadoopConf); this.dataPath = dataPath == null ? this.getBasePath() : sanitizeBasePath(dataPath); this.hadoopConf = hadoopConf; } @@ -66,7 +66,8 @@ public SourceTable( basePath, namespace, catalogConfig, - additionalProperties); + additionalProperties, + hadoopConf); this.dataPath = dataPath == null ? this.getBasePath() : sanitizeBasePath(dataPath); this.hadoopConf = hadoopConf; } diff --git a/xtable-api/src/main/java/org/apache/xtable/conversion/TargetTable.java b/xtable-api/src/main/java/org/apache/xtable/conversion/TargetTable.java index 6256da2c6..4ac9ede1b 100644 --- a/xtable-api/src/main/java/org/apache/xtable/conversion/TargetTable.java +++ b/xtable-api/src/main/java/org/apache/xtable/conversion/TargetTable.java @@ -26,6 +26,8 @@ import lombok.EqualsAndHashCode; import lombok.Getter; +import org.apache.hadoop.conf.Configuration; + @Getter @EqualsAndHashCode(callSuper = true) public class TargetTable extends ExternalTable { @@ -39,8 +41,9 @@ public TargetTable( String[] namespace, CatalogConfig catalogConfig, Duration metadataRetention, - Properties additionalProperties) { - super(name, formatName, basePath, namespace, catalogConfig, additionalProperties); + Properties additionalProperties, + Configuration hadoopConf) { + super(name, formatName, basePath, namespace, catalogConfig, additionalProperties, hadoopConf); this.metadataRetention = metadataRetention == null ? Duration.of(7, ChronoUnit.DAYS) : metadataRetention; } diff --git a/xtable-api/src/test/java/org/apache/xtable/conversion/TestExternalTable.java b/xtable-api/src/test/java/org/apache/xtable/conversion/TestExternalTable.java index 5422b0a7f..d34fc752a 100644 --- a/xtable-api/src/test/java/org/apache/xtable/conversion/TestExternalTable.java +++ b/xtable-api/src/test/java/org/apache/xtable/conversion/TestExternalTable.java @@ -21,21 +21,24 @@ import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import org.apache.hadoop.conf.Configuration; import org.junit.jupiter.api.Test; public class TestExternalTable { + Configuration hadoopConf = new Configuration(); + @Test void sanitizePath() { ExternalTable tooManySlashes = - new ExternalTable("name", "hudi", "s3://bucket//path", null, null, null); + new ExternalTable("name", "hudi", "s3://bucket//path", null, null, null, hadoopConf); assertEquals("s3://bucket/path", tooManySlashes.getBasePath()); ExternalTable localFilePath = - new ExternalTable("name", "hudi", "/local/data//path", null, null, null); + new ExternalTable("name", "hudi", "/local/data//path", null, null, null, hadoopConf); assertEquals("file:///local/data/path", localFilePath.getBasePath()); ExternalTable properLocalFilePath = - new ExternalTable("name", "hudi", "file:///local/data//path", null, null, null); + new ExternalTable("name", "hudi", "file:///local/data//path", null, null, null, hadoopConf); assertEquals("file:///local/data/path", properLocalFilePath.getBasePath()); } @@ -43,14 +46,14 @@ void sanitizePath() { void errorIfRequiredArgsNotSet() { assertThrows( NullPointerException.class, - () -> new ExternalTable("name", "hudi", null, null, null, null)); + () -> new ExternalTable("name", "hudi", null, null, null, null, hadoopConf)); assertThrows( NullPointerException.class, - () -> new ExternalTable("name", null, "file://bucket/path", null, null, null)); + () -> new ExternalTable("name", null, "file://bucket/path", null, null, null, hadoopConf)); assertThrows( NullPointerException.class, - () -> new ExternalTable(null, "hudi", "file://bucket/path", null, null, null)); + () -> new ExternalTable(null, "hudi", "file://bucket/path", null, null, null, hadoopConf)); } } diff --git a/xtable-core/src/main/java/org/apache/xtable/conversion/ConversionUtils.java b/xtable-core/src/main/java/org/apache/xtable/conversion/ConversionUtils.java index f21be6702..198713f7c 100644 --- a/xtable-core/src/main/java/org/apache/xtable/conversion/ConversionUtils.java +++ b/xtable-core/src/main/java/org/apache/xtable/conversion/ConversionUtils.java @@ -28,6 +28,7 @@ public static SourceTable convertToSourceTable(TargetTable table) { table.getBasePath(), table.getNamespace(), table.getCatalogConfig(), - table.getAdditionalProperties()); + table.getAdditionalProperties(), + table.hadoopConf); } } diff --git a/xtable-utilities/src/main/java/org/apache/xtable/utilities/RunSync.java b/xtable-utilities/src/main/java/org/apache/xtable/utilities/RunSync.java index 36c37f733..a696d3bf2 100644 --- a/xtable-utilities/src/main/java/org/apache/xtable/utilities/RunSync.java +++ b/xtable-utilities/src/main/java/org/apache/xtable/utilities/RunSync.java @@ -125,18 +125,31 @@ static SourceTable sourceTableBuilder( Properties sourceProperties, Configuration hadoopConf) throws IOException { - SourceTable sourceTable = - SourceTable.builder() - .name(table.getTableName()) - .basePath(table.getTableBasePath()) - .namespace(table.getNamespace() == null ? null : table.getNamespace().split("\\.")) - .dataPath(table.getTableDataPath()) - .catalogConfig(catalogConfig) - .additionalProperties(sourceProperties) - .formatName(datasetConfig.sourceFormat) - .hadoopConf(hadoopConf) - .build(); - return sourceTable; + if (datasetConfig.sourceFormat != null) { + SourceTable sourceTable = + SourceTable.builder() + .name(table.getTableName()) + .basePath(table.getTableBasePath()) + .namespace(table.getNamespace() == null ? null : table.getNamespace().split("\\.")) + .dataPath(table.getTableDataPath()) + .catalogConfig(catalogConfig) + .additionalProperties(sourceProperties) + .formatName(datasetConfig.sourceFormat) + .hadoopConf(hadoopConf) + .build(); + return sourceTable; + } else { + SourceTable sourceTable = + new SourceTable( + table.getTableName(), + table.getTableBasePath(), + table.getTableDataPath(), + table.getNamespace() == null ? null : table.getNamespace().split("\\."), + catalogConfig, + sourceProperties, + hadoopConf); + return sourceTable; + } } static List targetTableBuilder( From 3be1921429ec07d091adfe5a3a5172a851e38046 Mon Sep 17 00:00:00 2001 From: Selim Soufargi Date: Fri, 19 Jun 2026 22:47:02 +0200 Subject: [PATCH 15/30] add hadoopConf param to sourceTable in ITConversionController --- .../src/test/java/org/apache/xtable/ITConversionController.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/xtable-core/src/test/java/org/apache/xtable/ITConversionController.java b/xtable-core/src/test/java/org/apache/xtable/ITConversionController.java index 2da3078b6..5af3e5357 100644 --- a/xtable-core/src/test/java/org/apache/xtable/ITConversionController.java +++ b/xtable-core/src/test/java/org/apache/xtable/ITConversionController.java @@ -1126,6 +1126,7 @@ private static ConversionConfig getTableSyncConfig( .basePath(table.getBasePath()) .dataPath(table.getDataPath()) .additionalProperties(sourceProperties) + .hadoopConf(jsc.hadoopConfiguration()) .build(); List targetTables = @@ -1138,6 +1139,7 @@ private static ConversionConfig getTableSyncConfig( // set the metadata path to the data path as the default (required by Hudi) .basePath(table.getDataPath()) .metadataRetention(metadataRetention) + .hadoopConf(jsc.hadoopConfiguration()) .build()) .collect(Collectors.toList()); From 52f14d69d36dc1d317a17161c2d1a3fd420efab0 Mon Sep 17 00:00:00 2001 From: Selim Soufargi Date: Mon, 22 Jun 2026 23:38:06 +0200 Subject: [PATCH 16/30] if many formats detected throw Exception --- .../conversion/SourceTableFormatDetector.java | 29 +++++++++++++------ .../org/apache/xtable/utilities/RunSync.java | 6 ++++ .../apache/xtable/utilities/ITRunSync.java | 29 +++++++++++++++++++ 3 files changed, 55 insertions(+), 9 deletions(-) diff --git a/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTableFormatDetector.java b/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTableFormatDetector.java index c0b2fd7cb..d4853b640 100644 --- a/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTableFormatDetector.java +++ b/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTableFormatDetector.java @@ -19,6 +19,8 @@ package org.apache.xtable.conversion; import java.io.IOException; +import java.util.ArrayList; +import java.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; @@ -37,29 +39,38 @@ public static String detectFormat(String pathStr, Configuration conf) throws IOE Path basePath = new Path(sanitizeBasePath); FileSystem fs = basePath.getFileSystem(conf); - if (!fs.exists(basePath)) { - return "UNKNOWN"; - } + List matches = new ArrayList<>(); if (fs.exists(new Path(basePath, "_delta_log"))) { - return TableFormat.DELTA; + matches.add(TableFormat.DELTA); } if (fs.exists(new Path(basePath, ".hoodie"))) { - return TableFormat.HUDI; + matches.add(TableFormat.HUDI); } - // workaround for: .metadata can be set elsewhere try { HadoopTables tables = new HadoopTables(conf); - // if the path points to a valid Iceberg table (even with a custom metadata location), Table table = tables.load(pathStr); if (table != null) { - return TableFormat.ICEBERG; + matches.add(TableFormat.ICEBERG); } } catch (Exception e) { + throw new IllegalArgumentException("Failed to inspect Iceberg table at " + pathStr, e); + } + + if (matches.size() == 1) { + return matches.get(0); } - return "UNKNOWN"; + if (matches.size() > 1) { + throw new IllegalArgumentException( + "Multiple table formats detected at path '" + + pathStr + + "': " + + matches + + ". Please provide one source format explicitly."); + } + throw new IllegalArgumentException("Unable to detect table format for path: " + pathStr); } } diff --git a/xtable-utilities/src/main/java/org/apache/xtable/utilities/RunSync.java b/xtable-utilities/src/main/java/org/apache/xtable/utilities/RunSync.java index a696d3bf2..5865a0e42 100644 --- a/xtable-utilities/src/main/java/org/apache/xtable/utilities/RunSync.java +++ b/xtable-utilities/src/main/java/org/apache/xtable/utilities/RunSync.java @@ -256,6 +256,12 @@ static ConversionSourceProvider getConversionSourceProvider( TableFormatConverters.ConversionConfig sourceConversionConfig = tableFormatConverters.getTableFormatConverters().get(sourceFormat); + if (sourceConversionConfig == null) { + throw new IllegalArgumentException( + String.format( + "Source format %s is not supported. Known source and target formats are %s", + sourceFormat, tableFormatConverters.getTableFormatConverters().keySet())); + } String sourceProviderClass = sourceConversionConfig.conversionSourceProviderClass; ConversionSourceProvider conversionSourceProvider = ReflectionUtils.createInstanceOfClass(sourceProviderClass); diff --git a/xtable-utilities/src/test/java/org/apache/xtable/utilities/ITRunSync.java b/xtable-utilities/src/test/java/org/apache/xtable/utilities/ITRunSync.java index fbb06fef0..f2693be73 100644 --- a/xtable-utilities/src/test/java/org/apache/xtable/utilities/ITRunSync.java +++ b/xtable-utilities/src/test/java/org/apache/xtable/utilities/ITRunSync.java @@ -19,6 +19,7 @@ package org.apache.xtable.utilities; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; import java.io.File; import java.io.IOException; @@ -108,6 +109,17 @@ void testSingleSyncModeWithoutInputTableFormat(@TempDir Path tempDir) throws IOE } } + @Test + void testSingleSyncModeWithoutInputTableFormatAndEmptyPath(@TempDir Path tempDir) + throws IOException { + String tableName = "test-table"; + + File configFile = writeConfigFileWithoutSourceTableFormatAndEmptyPath(tempDir, tableName); + String[] args = new String[] {"--datasetConfig", configFile.getPath()}; + + assertThrows(IllegalArgumentException.class, () -> RunSync.main(args)); + } + private static File writeConfigFile(Path tempDir, GenericTable table, String tableName) throws IOException { RunSync.DatasetConfig config = @@ -126,6 +138,23 @@ private static File writeConfigFile(Path tempDir, GenericTable table, String tab return configFile; } + private static File writeConfigFileWithoutSourceTableFormatAndEmptyPath( + Path tempDir, String tableName) throws IOException { + RunSync.DatasetConfig config = + RunSync.DatasetConfig.builder() + .targetFormats(Collections.singletonList("ICEBERG")) + .datasets( + Collections.singletonList( + RunSync.DatasetConfig.Table.builder() + .tableBasePath("") + .tableName(tableName) + .build())) + .build(); + File configFile = new File(tempDir + "config.yaml"); + RunSync.YAML_MAPPER.writeValue(configFile, config); + return configFile; + } + private static File writeConfigFileWithoutSourceTableFormat( Path tempDir, GenericTable table, String tableName) throws IOException { RunSync.DatasetConfig config = From 371a9fd36bc7af63f7be8dc92463d265d6494720 Mon Sep 17 00:00:00 2001 From: Selim Soufargi Date: Tue, 23 Jun 2026 00:41:05 +0200 Subject: [PATCH 17/30] if many formats detected return first only --- .../apache/xtable/conversion/SourceTableFormatDetector.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTableFormatDetector.java b/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTableFormatDetector.java index d4853b640..49f56097b 100644 --- a/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTableFormatDetector.java +++ b/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTableFormatDetector.java @@ -59,18 +59,18 @@ public static String detectFormat(String pathStr, Configuration conf) throws IOE throw new IllegalArgumentException("Failed to inspect Iceberg table at " + pathStr, e); } - if (matches.size() == 1) { + if (matches.size() > 1) { return matches.get(0); } - if (matches.size() > 1) { + /*if (matches.size() > 1) { throw new IllegalArgumentException( "Multiple table formats detected at path '" + pathStr + "': " + matches + ". Please provide one source format explicitly."); - } + }*/ throw new IllegalArgumentException("Unable to detect table format for path: " + pathStr); } } From 30a8d71e0808b184a9b827da6084ca39e9031851 Mon Sep 17 00:00:00 2001 From: Selim Soufargi Date: Tue, 23 Jun 2026 00:42:17 +0200 Subject: [PATCH 18/30] if many formats detected return first only --- .../org/apache/xtable/conversion/SourceTableFormatDetector.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTableFormatDetector.java b/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTableFormatDetector.java index 49f56097b..c975cba6b 100644 --- a/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTableFormatDetector.java +++ b/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTableFormatDetector.java @@ -59,7 +59,7 @@ public static String detectFormat(String pathStr, Configuration conf) throws IOE throw new IllegalArgumentException("Failed to inspect Iceberg table at " + pathStr, e); } - if (matches.size() > 1) { + if (matches.size() >= 1) { return matches.get(0); } From 7448e483337569a18492b3ec2f9d93a527ea7ff1 Mon Sep 17 00:00:00 2001 From: Selim Soufargi Date: Tue, 23 Jun 2026 07:31:00 +0200 Subject: [PATCH 19/30] if many formats detected throw exception: tested OK --- .../apache/xtable/conversion/SourceTableFormatDetector.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTableFormatDetector.java b/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTableFormatDetector.java index c975cba6b..d4853b640 100644 --- a/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTableFormatDetector.java +++ b/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTableFormatDetector.java @@ -59,18 +59,18 @@ public static String detectFormat(String pathStr, Configuration conf) throws IOE throw new IllegalArgumentException("Failed to inspect Iceberg table at " + pathStr, e); } - if (matches.size() >= 1) { + if (matches.size() == 1) { return matches.get(0); } - /*if (matches.size() > 1) { + if (matches.size() > 1) { throw new IllegalArgumentException( "Multiple table formats detected at path '" + pathStr + "': " + matches + ". Please provide one source format explicitly."); - }*/ + } throw new IllegalArgumentException("Unable to detect table format for path: " + pathStr); } } From 6c54ceda30d6338b3dfe46759c5b10d47726a385 Mon Sep 17 00:00:00 2001 From: Selim Soufargi Date: Tue, 23 Jun 2026 08:08:05 +0200 Subject: [PATCH 20/30] fix CI --- .../test/java/org/apache/xtable/utilities/ITRunSync.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/xtable-utilities/src/test/java/org/apache/xtable/utilities/ITRunSync.java b/xtable-utilities/src/test/java/org/apache/xtable/utilities/ITRunSync.java index f2693be73..92aaa9700 100644 --- a/xtable-utilities/src/test/java/org/apache/xtable/utilities/ITRunSync.java +++ b/xtable-utilities/src/test/java/org/apache/xtable/utilities/ITRunSync.java @@ -133,7 +133,7 @@ private static File writeConfigFile(Path tempDir, GenericTable table, String tab .tableName(tableName) .build())) .build(); - File configFile = new File(tempDir + "config.yaml"); + File configFile = tempDir.resolve("config.yaml").toFile(); RunSync.YAML_MAPPER.writeValue(configFile, config); return configFile; } @@ -150,7 +150,7 @@ private static File writeConfigFileWithoutSourceTableFormatAndEmptyPath( .tableName(tableName) .build())) .build(); - File configFile = new File(tempDir + "config.yaml"); + File configFile = tempDir.resolve("config.yaml").toFile(); RunSync.YAML_MAPPER.writeValue(configFile, config); return configFile; } @@ -167,7 +167,7 @@ private static File writeConfigFileWithoutSourceTableFormat( .tableName(tableName) .build())) .build(); - File configFile = new File(tempDir + "config.yaml"); + File configFile = tempDir.resolve("config.yaml").toFile(); RunSync.YAML_MAPPER.writeValue(configFile, config); return configFile; } From a2bfbd7992890ca1f0de17ddfe4de7dbe88251c5 Mon Sep 17 00:00:00 2001 From: Selim Soufargi Date: Tue, 23 Jun 2026 08:33:33 +0200 Subject: [PATCH 21/30] run utilities tests only to debug CI --- .github/workflows/mvn-ci-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/mvn-ci-build.yml b/.github/workflows/mvn-ci-build.yml index 6b51ce66e..9e64536d8 100644 --- a/.github/workflows/mvn-ci-build.yml +++ b/.github/workflows/mvn-ci-build.yml @@ -44,4 +44,4 @@ jobs: cache: maven - name: Build all module with Maven - run: ./mvnw clean install -ntp -B + run: ./mvnw verify -pl xtable-utilities -DfailIfNoTests=false From 9f9e7d1f591537ba88cecd15537115d306cbb661 Mon Sep 17 00:00:00 2001 From: Selim Soufargi Date: Tue, 23 Jun 2026 08:35:11 +0200 Subject: [PATCH 22/30] run utilities tests only to debug CI --- .github/workflows/mvn-ci-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/mvn-ci-build.yml b/.github/workflows/mvn-ci-build.yml index 9e64536d8..9010268a6 100644 --- a/.github/workflows/mvn-ci-build.yml +++ b/.github/workflows/mvn-ci-build.yml @@ -44,4 +44,4 @@ jobs: cache: maven - name: Build all module with Maven - run: ./mvnw verify -pl xtable-utilities -DfailIfNoTests=false + run: ./mvnw verify -pl xtable-utilities -am -DfailIfNoTests=false From b870a31e8df5ce4f38a6e9d92ede9d1f37f54008 Mon Sep 17 00:00:00 2001 From: Selim Soufargi Date: Tue, 23 Jun 2026 18:08:54 +0200 Subject: [PATCH 23/30] remove throw when cannot load Iceberg table --- .github/workflows/mvn-ci-build.yml | 2 +- .../org/apache/xtable/conversion/SourceTableFormatDetector.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/mvn-ci-build.yml b/.github/workflows/mvn-ci-build.yml index 9010268a6..6b51ce66e 100644 --- a/.github/workflows/mvn-ci-build.yml +++ b/.github/workflows/mvn-ci-build.yml @@ -44,4 +44,4 @@ jobs: cache: maven - name: Build all module with Maven - run: ./mvnw verify -pl xtable-utilities -am -DfailIfNoTests=false + run: ./mvnw clean install -ntp -B diff --git a/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTableFormatDetector.java b/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTableFormatDetector.java index d4853b640..d30de8402 100644 --- a/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTableFormatDetector.java +++ b/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTableFormatDetector.java @@ -56,7 +56,7 @@ public static String detectFormat(String pathStr, Configuration conf) throws IOE matches.add(TableFormat.ICEBERG); } } catch (Exception e) { - throw new IllegalArgumentException("Failed to inspect Iceberg table at " + pathStr, e); + // throw new IllegalArgumentException("Failed to inspect Iceberg table at " + pathStr, e); } if (matches.size() == 1) { From 0d0f1549a440d67081ad75f20f77132b267bd014 Mon Sep 17 00:00:00 2001 From: Selim Soufargi Date: Tue, 23 Jun 2026 18:58:16 +0200 Subject: [PATCH 24/30] private constructor to prevent instantiation --- .../apache/xtable/conversion/SourceTableFormatDetector.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTableFormatDetector.java b/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTableFormatDetector.java index d30de8402..87613d5a4 100644 --- a/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTableFormatDetector.java +++ b/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTableFormatDetector.java @@ -32,8 +32,11 @@ import org.apache.xtable.model.storage.TableFormat; public class SourceTableFormatDetector { + // private constructor to prevent instantiation + private SourceTableFormatDetector() { + throw new UnsupportedOperationException("This class cannot be instantiated"); + } // helper method to detect input format - public static String detectFormat(String pathStr, Configuration conf) throws IOException { String sanitizeBasePath = ExternalTable.sanitizeBasePath(pathStr); Path basePath = new Path(sanitizeBasePath); From d2ca7501289ecc1ea40d87b834d2fdfd951579dc Mon Sep 17 00:00:00 2001 From: Selim Soufargi Date: Tue, 7 Jul 2026 20:53:11 +0200 Subject: [PATCH 25/30] added support for detection over multi-synced table --- xtable-api/pom.xml | 24 ++++++ .../apache/xtable/conversion/SourceTable.java | 33 ++++---- .../conversion/SourceTableFormatDetector.java | 83 +++++++++++++++++-- .../apache/xtable/conversion/TargetTable.java | 9 +- .../org/apache/xtable/utilities/RunSync.java | 2 +- 5 files changed, 124 insertions(+), 27 deletions(-) diff --git a/xtable-api/pom.xml b/xtable-api/pom.xml index 5b09ad8de..3bc14c9af 100644 --- a/xtable-api/pom.xml +++ b/xtable-api/pom.xml @@ -100,5 +100,29 @@ org.apache.iceberg iceberg-api + + io.delta + delta-core_2.12 + + + io.delta + delta-core_2.12 + + + io.delta + delta-core_2.12 + + + io.delta + delta-core_2.12 + + + io.delta + delta-kernel-api + + + io.delta + delta-kernel-defaults + diff --git a/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTable.java b/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTable.java index f287ed5b4..cdd9c8111 100644 --- a/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTable.java +++ b/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTable.java @@ -35,8 +35,6 @@ public class SourceTable extends ExternalTable { /** The path to the data files, defaults to the basePath */ @NonNull private final String dataPath; - private final transient Configuration hadoopConf; - @Builder(toBuilder = true) public SourceTable( String name, @@ -49,27 +47,30 @@ public SourceTable( Configuration hadoopConf) { super(name, formatName, basePath, namespace, catalogConfig, additionalProperties, hadoopConf); this.dataPath = dataPath == null ? this.getBasePath() : sanitizeBasePath(dataPath); - this.hadoopConf = hadoopConf; } - public SourceTable( - @NonNull String name, - @NonNull String basePath, + public static SourceTable withDetectedFormat( + String name, + String basePath, String dataPath, String[] namespace, CatalogConfig catalogConfig, Properties additionalProperties, Configuration hadoopConf) { - super( - name, - resolveFormatOrThrow(basePath, hadoopConf), - basePath, - namespace, - catalogConfig, - additionalProperties, - hadoopConf); - this.dataPath = dataPath == null ? this.getBasePath() : sanitizeBasePath(dataPath); - this.hadoopConf = hadoopConf; + + Configuration resolvedConf = hadoopConf != null ? hadoopConf : new Configuration(); + String detectedFormat = resolveFormatOrThrow(basePath, resolvedConf); + + return SourceTable.builder() + .name(name) + .formatName(detectedFormat) + .basePath(basePath) + .dataPath(dataPath) + .namespace(namespace) + .catalogConfig(catalogConfig) + .additionalProperties(additionalProperties) + .hadoopConf(resolvedConf) + .build(); } private static String resolveFormatOrThrow(String basePath, Configuration hadoopConf) { diff --git a/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTableFormatDetector.java b/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTableFormatDetector.java index 87613d5a4..af9143d1b 100644 --- a/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTableFormatDetector.java +++ b/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTableFormatDetector.java @@ -21,16 +21,25 @@ import java.io.IOException; import java.util.ArrayList; import java.util.List; +import java.util.Map; + +import lombok.extern.log4j.Log4j2; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.iceberg.Table; +import org.apache.iceberg.exceptions.NoSuchTableException; import org.apache.iceberg.hadoop.HadoopTables; +import io.delta.kernel.Snapshot; +import io.delta.kernel.defaults.engine.DefaultEngine; +import io.delta.kernel.engine.Engine; +import io.delta.kernel.internal.SnapshotImpl; + import org.apache.xtable.model.storage.TableFormat; +@Log4j2 public class SourceTableFormatDetector { // private constructor to prevent instantiation private SourceTableFormatDetector() { @@ -54,12 +63,14 @@ public static String detectFormat(String pathStr, Configuration conf) throws IOE try { HadoopTables tables = new HadoopTables(conf); - Table table = tables.load(pathStr); + org.apache.iceberg.Table table = tables.load(pathStr); if (table != null) { matches.add(TableFormat.ICEBERG); } + } catch (NoSuchTableException e) { + log.debug("No Iceberg table found at path: {}", pathStr); } catch (Exception e) { - // throw new IllegalArgumentException("Failed to inspect Iceberg table at " + pathStr, e); + log.debug("Unexpected error while probing for Iceberg table at path: {}", pathStr, e); } if (matches.size() == 1) { @@ -67,13 +78,67 @@ public static String detectFormat(String pathStr, Configuration conf) throws IOE } if (matches.size() > 1) { - throw new IllegalArgumentException( - "Multiple table formats detected at path '" - + pathStr - + "': " - + matches - + ". Please provide one source format explicitly."); + log.info( + "Multiple formats detected: {}. Resolving target sync vs original source...", matches); + return inferSourceFromSyncMetadata(basePath, fs, matches, conf); } throw new IllegalArgumentException("Unable to detect table format for path: " + pathStr); } + + // checks source format from a XTable target + private static String inferSourceFromSyncMetadata( + Path basePath, FileSystem fs, List detectedFormats, Configuration conf) { + try { + // check Hudi metadata if present + if (detectedFormats.contains(TableFormat.HUDI)) { + // check XTable target for hudi property file + Path hoodieMeta = new Path(basePath, ".hoodie"); + if (fs.exists(new Path(hoodieMeta, "hoodie.properties"))) { + return TableFormat.HUDI; + } + } + + if (detectedFormats.contains(TableFormat.ICEBERG)) { + HadoopTables tables = new HadoopTables(conf); + org.apache.iceberg.Table table = tables.load(basePath.toString()); + // check for target property tag during a sync run + if (table.properties().containsKey("xtable.conversion.target")) { + detectedFormats.remove(TableFormat.ICEBERG); + if (detectedFormats.size() == 1) { + return detectedFormats.get(0); + } + } + } + if (detectedFormats.contains(TableFormat.DELTA)) { + try { + + Engine kernelEngine = DefaultEngine.create(conf); + + io.delta.kernel.Table table = + io.delta.kernel.Table.forPath(kernelEngine, basePath.toString()); + Snapshot snapshot = table.getLatestSnapshot(kernelEngine); + + if (snapshot instanceof SnapshotImpl) { + Map config = ((SnapshotImpl) snapshot).getMetadata().getConfiguration(); + + if (config != null && config.containsKey("xtable.conversion.target")) { + detectedFormats.remove(TableFormat.DELTA); + if (detectedFormats.size() == 1) { + return detectedFormats.get(0); + } + } + } + } catch (Exception e) { + log.debug( + "Failed to verify Delta log metadata via internal XTable engine at path: {}", + basePath, + e); + } + } + } catch (Exception e) { + log.warn("Failed to parse table metadata properties during conflict resolution step", e); + } + + return null; + } } diff --git a/xtable-api/src/main/java/org/apache/xtable/conversion/TargetTable.java b/xtable-api/src/main/java/org/apache/xtable/conversion/TargetTable.java index 4ac9ede1b..360add146 100644 --- a/xtable-api/src/main/java/org/apache/xtable/conversion/TargetTable.java +++ b/xtable-api/src/main/java/org/apache/xtable/conversion/TargetTable.java @@ -43,7 +43,14 @@ public TargetTable( Duration metadataRetention, Properties additionalProperties, Configuration hadoopConf) { - super(name, formatName, basePath, namespace, catalogConfig, additionalProperties, hadoopConf); + super( + name, + formatName, + basePath, + namespace, + catalogConfig, + additionalProperties, + hadoopConf != null ? hadoopConf : new Configuration()); this.metadataRetention = metadataRetention == null ? Duration.of(7, ChronoUnit.DAYS) : metadataRetention; } diff --git a/xtable-utilities/src/main/java/org/apache/xtable/utilities/RunSync.java b/xtable-utilities/src/main/java/org/apache/xtable/utilities/RunSync.java index 5865a0e42..0a7fe6793 100644 --- a/xtable-utilities/src/main/java/org/apache/xtable/utilities/RunSync.java +++ b/xtable-utilities/src/main/java/org/apache/xtable/utilities/RunSync.java @@ -140,7 +140,7 @@ static SourceTable sourceTableBuilder( return sourceTable; } else { SourceTable sourceTable = - new SourceTable( + SourceTable.withDetectedFormat( table.getTableName(), table.getTableBasePath(), table.getTableDataPath(), From c74aecf5b696b7f95a8656b1bc732b20f57b3a25 Mon Sep 17 00:00:00 2001 From: Selim Soufargi Date: Wed, 8 Jul 2026 17:38:01 +0200 Subject: [PATCH 26/30] fix CI --- .../test/java/org/apache/xtable/ITConversionController.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/xtable-core/src/test/java/org/apache/xtable/ITConversionController.java b/xtable-core/src/test/java/org/apache/xtable/ITConversionController.java index b2f665503..c07c8d839 100644 --- a/xtable-core/src/test/java/org/apache/xtable/ITConversionController.java +++ b/xtable-core/src/test/java/org/apache/xtable/ITConversionController.java @@ -76,7 +76,6 @@ import org.apache.hudi.client.HoodieReadClient; import org.apache.hudi.common.config.HoodieMetadataConfig; -import org.apache.hudi.common.config.TypedProperties; import org.apache.hudi.common.model.HoodieAvroPayload; import org.apache.hudi.common.model.HoodieRecord; import org.apache.hudi.common.model.HoodieTableType; @@ -1218,9 +1217,7 @@ private static ConversionConfig getTableSyncConfig( // set the metadata path to the data path as the default (required by Hudi) .basePath(table.getDataPath()) .metadataRetention(metadataRetention) - .hadoopConf(jsc.hadoopConfiguration()) - .build()) .collect(Collectors.toList()); From 56fb7cd1d384737b50112bb4bda6af650c54eca1 Mon Sep 17 00:00:00 2001 From: Selim Soufargi Date: Wed, 8 Jul 2026 18:19:17 +0200 Subject: [PATCH 27/30] fix CI --- .../src/test/java/org/apache/xtable/utilities/ITRunSync.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xtable-utilities/src/test/java/org/apache/xtable/utilities/ITRunSync.java b/xtable-utilities/src/test/java/org/apache/xtable/utilities/ITRunSync.java index 92aaa9700..75ebca1cd 100644 --- a/xtable-utilities/src/test/java/org/apache/xtable/utilities/ITRunSync.java +++ b/xtable-utilities/src/test/java/org/apache/xtable/utilities/ITRunSync.java @@ -84,7 +84,7 @@ void testContinuousSyncMode(@TempDir Path tempDir) throws IOException { try (GenericTable table = TestJavaHudiTable.withAdditionalColumnsAndFieldIds( tableName, tempDir, null, HoodieTableType.COPY_ON_WRITE)) { - // write more data now that table is initialized and data is synced + // write more data now that the table is initialized and data is synced table.insertRows(20); Path icebergMetadataPath = Paths.get(URI.create(table.getBasePath() + "/metadata")); waitForNumIcebergCommits(icebergMetadataPath, 6); From 0a131d93c97d94b0d39c66a3ca6080ba4b5729a0 Mon Sep 17 00:00:00 2001 From: Selim Soufargi Date: Tue, 14 Jul 2026 22:26:10 +0200 Subject: [PATCH 28/30] test for detector with two successive syncs --- .../conversion/SourceTableFormatDetector.java | 12 +++-- .../apache/xtable/utilities/ITRunSync.java | 46 +++++++++++++++++-- 2 files changed, 52 insertions(+), 6 deletions(-) diff --git a/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTableFormatDetector.java b/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTableFormatDetector.java index af9143d1b..469e87045 100644 --- a/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTableFormatDetector.java +++ b/xtable-api/src/main/java/org/apache/xtable/conversion/SourceTableFormatDetector.java @@ -60,7 +60,6 @@ public static String detectFormat(String pathStr, Configuration conf) throws IOE if (fs.exists(new Path(basePath, ".hoodie"))) { matches.add(TableFormat.HUDI); } - try { HadoopTables tables = new HadoopTables(conf); org.apache.iceberg.Table table = tables.load(pathStr); @@ -72,7 +71,6 @@ public static String detectFormat(String pathStr, Configuration conf) throws IOE } catch (Exception e) { log.debug("Unexpected error while probing for Iceberg table at path: {}", pathStr, e); } - if (matches.size() == 1) { return matches.get(0); } @@ -101,9 +99,13 @@ private static String inferSourceFromSyncMetadata( if (detectedFormats.contains(TableFormat.ICEBERG)) { HadoopTables tables = new HadoopTables(conf); org.apache.iceberg.Table table = tables.load(basePath.toString()); - // check for target property tag during a sync run + // check for target property tag during a sync run if (table.properties().containsKey("xtable.conversion.target")) { detectedFormats.remove(TableFormat.ICEBERG); + // remove Hudi if it was detected as potential source, as it is not the source + if (detectedFormats.contains(TableFormat.HUDI)) { + detectedFormats.remove(TableFormat.HUDI); + } if (detectedFormats.size() == 1) { return detectedFormats.get(0); } @@ -123,6 +125,10 @@ private static String inferSourceFromSyncMetadata( if (config != null && config.containsKey("xtable.conversion.target")) { detectedFormats.remove(TableFormat.DELTA); + // remove Hudi if it was detected as potential source, as it is not the source + if (detectedFormats.contains(TableFormat.HUDI)) { + detectedFormats.remove(TableFormat.HUDI); + } if (detectedFormats.size() == 1) { return detectedFormats.get(0); } diff --git a/xtable-utilities/src/test/java/org/apache/xtable/utilities/ITRunSync.java b/xtable-utilities/src/test/java/org/apache/xtable/utilities/ITRunSync.java index 75ebca1cd..7a3d8b8f1 100644 --- a/xtable-utilities/src/test/java/org/apache/xtable/utilities/ITRunSync.java +++ b/xtable-utilities/src/test/java/org/apache/xtable/utilities/ITRunSync.java @@ -28,7 +28,9 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.Arrays; import java.util.Collections; +import java.util.List; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; @@ -97,15 +99,34 @@ void testContinuousSyncMode(@TempDir Path tempDir) throws IOException { @Test void testSingleSyncModeWithoutInputTableFormat(@TempDir Path tempDir) throws IOException { String tableName = "test-table"; + try (GenericTable table = TestJavaHudiTable.forStandardSchema( tableName, tempDir, null, HoodieTableType.COPY_ON_WRITE)) { table.insertRows(10); - File configFile = writeConfigFileWithoutSourceTableFormat(tempDir, table, tableName); + + File configFile = + writeConfigFileWithoutSourceTableFormat( + tempDir, table, tableName, Arrays.asList("ICEBERG")); String[] args = new String[] {"--datasetConfig", configFile.getPath()}; RunSync.main(args); + Path icebergMetadataPath = Paths.get(URI.create(table.getBasePath() + "/metadata")); - waitForNumIcebergCommits(icebergMetadataPath, 3); + waitForNumIcebergCommits(icebergMetadataPath, 1); + } + + try (GenericTable table = + TestJavaHudiTable.forStandardSchema( + tableName, tempDir, null, HoodieTableType.COPY_ON_WRITE)) { + + table.insertRows(5); + + File configFile_2 = + writeConfigFileWithoutSourceTableFormat( + tempDir, table, tableName, Arrays.asList("DELTA")); + String[] args_2 = new String[] {"--datasetConfig", configFile_2.getPath()}; + + RunSync.main(args_2); } } @@ -155,7 +176,7 @@ private static File writeConfigFileWithoutSourceTableFormatAndEmptyPath( return configFile; } - private static File writeConfigFileWithoutSourceTableFormat( + /*private static File writeConfigFileWithoutSourceTableFormat( Path tempDir, GenericTable table, String tableName) throws IOException { RunSync.DatasetConfig config = RunSync.DatasetConfig.builder() @@ -170,6 +191,25 @@ private static File writeConfigFileWithoutSourceTableFormat( File configFile = tempDir.resolve("config.yaml").toFile(); RunSync.YAML_MAPPER.writeValue(configFile, config); return configFile; + }*/ + private static File writeConfigFileWithoutSourceTableFormat( + Path tempDir, GenericTable table, String tableName, List targetFormats) + throws IOException { + + RunSync.DatasetConfig config = + RunSync.DatasetConfig.builder() + .targetFormats(targetFormats) + .datasets( + Collections.singletonList( + RunSync.DatasetConfig.Table.builder() + .tableBasePath(table.getBasePath()) + .tableName(tableName) + .build())) + .build(); + + File configFile = tempDir.resolve("config.yaml").toFile(); + RunSync.YAML_MAPPER.writeValue(configFile, config); + return configFile; } @SneakyThrows From be62bf5c456f569ed490eeb8a5f136d9b2dd6294 Mon Sep 17 00:00:00 2001 From: Selim Soufargi Date: Tue, 14 Jul 2026 23:41:40 +0200 Subject: [PATCH 29/30] test for detector with two successive syncs:add comments for the test --- .../java/org/apache/xtable/hudi/BaseFileUpdatesExtractor.java | 2 +- .../src/test/java/org/apache/xtable/utilities/ITRunSync.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/xtable-core/src/main/java/org/apache/xtable/hudi/BaseFileUpdatesExtractor.java b/xtable-core/src/main/java/org/apache/xtable/hudi/BaseFileUpdatesExtractor.java index 44acd089b..9417f1cbe 100644 --- a/xtable-core/src/main/java/org/apache/xtable/hudi/BaseFileUpdatesExtractor.java +++ b/xtable-core/src/main/java/org/apache/xtable/hudi/BaseFileUpdatesExtractor.java @@ -315,7 +315,7 @@ private Map> convertColStats( -1L, valueMetadata); }) - .collect(Collectors.toMap(HoodieColumnRangeMetadata::getColumnName, Function.identity())); + .collect(Collectors.toMap(HoodieColumnRangeMetadata::getColumnName, metadata -> metadata)); } /** Holds the information needed to create a "replace" commit in the Hudi table. */ diff --git a/xtable-utilities/src/test/java/org/apache/xtable/utilities/ITRunSync.java b/xtable-utilities/src/test/java/org/apache/xtable/utilities/ITRunSync.java index 7a3d8b8f1..3ff447654 100644 --- a/xtable-utilities/src/test/java/org/apache/xtable/utilities/ITRunSync.java +++ b/xtable-utilities/src/test/java/org/apache/xtable/utilities/ITRunSync.java @@ -99,7 +99,7 @@ void testContinuousSyncMode(@TempDir Path tempDir) throws IOException { @Test void testSingleSyncModeWithoutInputTableFormat(@TempDir Path tempDir) throws IOException { String tableName = "test-table"; - + // first sync: Hudi to Iceberg try (GenericTable table = TestJavaHudiTable.forStandardSchema( tableName, tempDir, null, HoodieTableType.COPY_ON_WRITE)) { @@ -114,7 +114,7 @@ void testSingleSyncModeWithoutInputTableFormat(@TempDir Path tempDir) throws IOE Path icebergMetadataPath = Paths.get(URI.create(table.getBasePath() + "/metadata")); waitForNumIcebergCommits(icebergMetadataPath, 1); } - + // second sync: Hudi to Delta try (GenericTable table = TestJavaHudiTable.forStandardSchema( tableName, tempDir, null, HoodieTableType.COPY_ON_WRITE)) { From 4934f78001c92fda5afa85c5c6d3647af142e71c Mon Sep 17 00:00:00 2001 From: Selim Soufargi Date: Tue, 14 Jul 2026 23:43:05 +0200 Subject: [PATCH 30/30] test for detector with two successive syncs:add comments for the test --- .../java/org/apache/xtable/hudi/BaseFileUpdatesExtractor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/xtable-core/src/main/java/org/apache/xtable/hudi/BaseFileUpdatesExtractor.java b/xtable-core/src/main/java/org/apache/xtable/hudi/BaseFileUpdatesExtractor.java index 9417f1cbe..44acd089b 100644 --- a/xtable-core/src/main/java/org/apache/xtable/hudi/BaseFileUpdatesExtractor.java +++ b/xtable-core/src/main/java/org/apache/xtable/hudi/BaseFileUpdatesExtractor.java @@ -315,7 +315,7 @@ private Map> convertColStats( -1L, valueMetadata); }) - .collect(Collectors.toMap(HoodieColumnRangeMetadata::getColumnName, metadata -> metadata)); + .collect(Collectors.toMap(HoodieColumnRangeMetadata::getColumnName, Function.identity())); } /** Holds the information needed to create a "replace" commit in the Hudi table. */