From ef8be52c28615026aaf807061c97ba4dbc8dfaac Mon Sep 17 00:00:00 2001 From: Eunbin Son Date: Thu, 30 Jul 2026 15:35:43 +0900 Subject: [PATCH] [gs] Map gs.* config entries to fs.gs.* Hadoop config keys GSFileIO accepted both gs. and fs.gs. prefixed options but copied matching keys into the Hadoop configuration unchanged, so gs.* entries were silently ignored by the gcs-connector, which only reads fs.gs.* keys. Translate the accepted prefixes to fs.gs.* like S3FileIO and AzureFileIO do for their prefixes, and log the mapping at debug level without values. Generated-by: Claude Code --- .../java/org/apache/paimon/gs/GSFileIO.java | 23 ++++--- .../org/apache/paimon/gs/GSFileIOTest.java | 66 +++++++++++++++++++ 2 files changed, 81 insertions(+), 8 deletions(-) create mode 100644 paimon-filesystems/paimon-gs-impl/src/test/java/org/apache/paimon/gs/GSFileIOTest.java diff --git a/paimon-filesystems/paimon-gs-impl/src/main/java/org/apache/paimon/gs/GSFileIO.java b/paimon-filesystems/paimon-gs-impl/src/main/java/org/apache/paimon/gs/GSFileIO.java index d2efcad70c7b..ccfb92409c7f 100644 --- a/paimon-filesystems/paimon-gs-impl/src/main/java/org/apache/paimon/gs/GSFileIO.java +++ b/paimon-filesystems/paimon-gs-impl/src/main/java/org/apache/paimon/gs/GSFileIO.java @@ -18,10 +18,10 @@ package org.apache.paimon.gs; +import org.apache.paimon.annotation.VisibleForTesting; import org.apache.paimon.catalog.CatalogContext; import org.apache.paimon.fs.FileIO; import org.apache.paimon.options.Options; -import org.apache.paimon.utils.SensitiveConfigUtils; import com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem; import org.apache.hadoop.conf.Configuration; @@ -46,6 +46,8 @@ public class GSFileIO extends HadoopCompliantFileIO { private static final String[] CONFIG_PREFIXES = {"gs.", "fs.gs."}; + private static final String HADOOP_CONFIG_PREFIX = "fs.gs."; + /** * Cache GSFileSystem, at present, there is no good mechanism to ensure that the file system * will be shut down, so here the fs cache is used to avoid resource leakage. @@ -61,20 +63,25 @@ public boolean isObjectStore() { @Override public void configure(CatalogContext context) { - hadoopOptions = new Options(); - // read all configuration with prefix 'CONFIG_PREFIXES' + this.hadoopOptions = loadHadoopConfigFromContext(context); + } + + // add additional config entries from the IO config to the Hadoop config + @VisibleForTesting + Options loadHadoopConfigFromContext(CatalogContext context) { + Options hadoopConfig = new Options(); for (String key : context.options().keySet()) { for (String prefix : CONFIG_PREFIXES) { if (key.startsWith(prefix)) { + String newKey = HADOOP_CONFIG_PREFIX + key.substring(prefix.length()); String value = context.options().get(key); - hadoopOptions.set(key, value); - LOG.warn( - "Adding config entry for {} as {} to Hadoop config", - key, - SensitiveConfigUtils.redactValue(key, hadoopOptions.get(key))); + hadoopConfig.set(newKey, value); + + LOG.debug("Adding config entry for {} as {} to Hadoop config", key, newKey); } } } + return hadoopConfig; } @Override diff --git a/paimon-filesystems/paimon-gs-impl/src/test/java/org/apache/paimon/gs/GSFileIOTest.java b/paimon-filesystems/paimon-gs-impl/src/test/java/org/apache/paimon/gs/GSFileIOTest.java new file mode 100644 index 000000000000..c0e2dc9293dd --- /dev/null +++ b/paimon-filesystems/paimon-gs-impl/src/test/java/org/apache/paimon/gs/GSFileIOTest.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.gs; + +import org.apache.paimon.catalog.CatalogContext; +import org.apache.paimon.options.Options; + +import org.junit.jupiter.api.Test; + +import static org.assertj.core.api.Assertions.assertThat; + +/** Tests for {@link GSFileIO}. */ +public class GSFileIOTest { + + @Test + public void testGsPrefixedKeyIsMappedToHadoopKey() { + Options options = new Options(); + options.set("gs.auth.type", "SERVICE_ACCOUNT_JSON_KEYFILE"); + + Options hadoopConfig = loadHadoopConfig(options); + + assertThat(hadoopConfig.get("fs.gs.auth.type")).isEqualTo("SERVICE_ACCOUNT_JSON_KEYFILE"); + assertThat(hadoopConfig.containsKey("gs.auth.type")).isFalse(); + } + + @Test + public void testHadoopPrefixedKeyIsKept() { + Options options = new Options(); + options.set("fs.gs.project.id", "my-project"); + + Options hadoopConfig = loadHadoopConfig(options); + + assertThat(hadoopConfig.get("fs.gs.project.id")).isEqualTo("my-project"); + } + + @Test + public void testUnrelatedKeysAreIgnored() { + Options options = new Options(); + options.set("warehouse", "gs://bucket/path"); + options.set("s3.endpoint", "http://localhost:9000"); + + Options hadoopConfig = loadHadoopConfig(options); + + assertThat(hadoopConfig.toMap()).isEmpty(); + } + + private static Options loadHadoopConfig(Options options) { + return new GSFileIO().loadHadoopConfigFromContext(CatalogContext.create(options)); + } +}