From 311304d48871f8709daef81d141eca8f609378f3 Mon Sep 17 00:00:00 2001 From: Anton Borisov Date: Sat, 4 Apr 2026 09:41:13 +0100 Subject: [PATCH] [client] Fix client.fs.* config prefix not stripped before passing to FileSystem --- .../java/org/apache/fluss/client/FlussConnection.java | 4 ++-- .../apache/fluss/client/table/FlussTableITCase.java | 2 +- .../apache/fluss/fs/azure/AzureFileSystemPlugin.java | 2 +- .../org/apache/fluss/fs/obs/OBSFileSystemPlugin.java | 11 ++++++----- .../org/apache/fluss/fs/oss/OSSFileSystemPlugin.java | 11 ++++++----- 5 files changed, 16 insertions(+), 14 deletions(-) diff --git a/fluss-client/src/main/java/org/apache/fluss/client/FlussConnection.java b/fluss-client/src/main/java/org/apache/fluss/client/FlussConnection.java index d98e45869e..e5eaa41c3e 100644 --- a/fluss-client/src/main/java/org/apache/fluss/client/FlussConnection.java +++ b/fluss-client/src/main/java/org/apache/fluss/client/FlussConnection.java @@ -48,7 +48,7 @@ import static org.apache.fluss.client.utils.MetadataUtils.getOneAvailableTabletServerNode; import static org.apache.fluss.config.FlussConfigUtils.CLIENT_PREFIX; -import static org.apache.fluss.utils.PropertiesUtils.extractPrefix; +import static org.apache.fluss.utils.PropertiesUtils.extractAndRemovePrefix; /** A connection to Fluss cluster, and holds the client session resources. */ public final class FlussConnection implements Connection { @@ -74,7 +74,7 @@ public final class FlussConnection implements Connection { // only pass options with 'client.fs.' prefix FileSystem.initialize( Configuration.fromMap( - extractPrefix(new HashMap<>(conf.toMap()), CLIENT_PREFIX + "fs.")), + extractAndRemovePrefix(new HashMap<>(conf.toMap()), CLIENT_PREFIX + "fs.")), null); // for client metrics. setupClientMetricsConfiguration(); diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/FlussTableITCase.java b/fluss-client/src/test/java/org/apache/fluss/client/table/FlussTableITCase.java index 8fd0c138de..1d3865a08f 100644 --- a/fluss-client/src/test/java/org/apache/fluss/client/table/FlussTableITCase.java +++ b/fluss-client/src/test/java/org/apache/fluss/client/table/FlussTableITCase.java @@ -1740,7 +1740,7 @@ void testFileSystemRecognizeConnectionConf() throws Exception { Configuration filesystemConf = testFileSystem.getConfiguration(); assertThat(filesystemConf.toMap()) .containsExactlyEntriesOf( - Collections.singletonMap("client.fs.test.key", "fs_test_value")); + Collections.singletonMap("test.key", "fs_test_value")); } } diff --git a/fluss-filesystems/fluss-fs-azure/src/main/java/org/apache/fluss/fs/azure/AzureFileSystemPlugin.java b/fluss-filesystems/fluss-fs-azure/src/main/java/org/apache/fluss/fs/azure/AzureFileSystemPlugin.java index 318198fe3b..d57994631f 100644 --- a/fluss-filesystems/fluss-fs-azure/src/main/java/org/apache/fluss/fs/azure/AzureFileSystemPlugin.java +++ b/fluss-filesystems/fluss-fs-azure/src/main/java/org/apache/fluss/fs/azure/AzureFileSystemPlugin.java @@ -43,7 +43,7 @@ * (abfs, abfss, wasb, wasbs) based on Azure HDFS support in the hadoop-azure module. */ abstract class AzureFileSystemPlugin implements FileSystemPlugin { - private static final String[] FLUSS_CONFIG_PREFIXES = {"fs.azure."}; + private static final String[] FLUSS_CONFIG_PREFIXES = {"azure.", "fs.azure."}; private static final String HADOOP_CONFIG_PREFIX = "fs.azure."; diff --git a/fluss-filesystems/fluss-fs-obs/src/main/java/org/apache/fluss/fs/obs/OBSFileSystemPlugin.java b/fluss-filesystems/fluss-fs-obs/src/main/java/org/apache/fluss/fs/obs/OBSFileSystemPlugin.java index 815b36ab01..0aaa3df94a 100644 --- a/fluss-filesystems/fluss-fs-obs/src/main/java/org/apache/fluss/fs/obs/OBSFileSystemPlugin.java +++ b/fluss-filesystems/fluss-fs-obs/src/main/java/org/apache/fluss/fs/obs/OBSFileSystemPlugin.java @@ -41,7 +41,9 @@ public class OBSFileSystemPlugin implements FileSystemPlugin { * In order to simplify, we make fluss obs configuration keys same with hadoop obs module. So, * we add all configuration key with prefix `fs.obs` in fluss conf to hadoop conf */ - private static final String[] FLUSS_CONFIG_PREFIXES = {"fs.obs."}; + private static final String[] FLUSS_CONFIG_PREFIXES = {"obs.", "fs.obs."}; + + private static final String HADOOP_CONFIG_PREFIX = "fs.obs."; private static final String ACCESS_KEY_ID = "fs.obs.access.key"; public static final String CREDENTIALS_PROVIDER = "fs.obs.security.provider"; @@ -116,15 +118,14 @@ org.apache.hadoop.conf.Configuration getHadoopConfiguration(Configuration flussC for (String key : flussConfig.keySet()) { for (String prefix : FLUSS_CONFIG_PREFIXES) { if (key.startsWith(prefix)) { + String newKey = HADOOP_CONFIG_PREFIX + key.substring(prefix.length()); String value = flussConfig.getString( ConfigBuilder.key(key).stringType().noDefaultValue(), null); - conf.set(key, value); + conf.set(newKey, value); LOG.debug( - "Adding Fluss config entry for {} as {} to Hadoop config", - key, - conf.get(key)); + "Adding Fluss config entry for {} as {} to Hadoop config", key, newKey); } } } diff --git a/fluss-filesystems/fluss-fs-oss/src/main/java/org/apache/fluss/fs/oss/OSSFileSystemPlugin.java b/fluss-filesystems/fluss-fs-oss/src/main/java/org/apache/fluss/fs/oss/OSSFileSystemPlugin.java index e3d3f88b58..2714280ef4 100644 --- a/fluss-filesystems/fluss-fs-oss/src/main/java/org/apache/fluss/fs/oss/OSSFileSystemPlugin.java +++ b/fluss-filesystems/fluss-fs-oss/src/main/java/org/apache/fluss/fs/oss/OSSFileSystemPlugin.java @@ -54,7 +54,9 @@ public class OSSFileSystemPlugin implements FileSystemPlugin { * In order to simplify, we make fluss oss configuration keys same with hadoop oss module. So, * we add all configuration key with prefix `fs.oss` in fluss conf to hadoop conf */ - private static final String[] FLUSS_CONFIG_PREFIXES = {"fs.oss."}; + private static final String[] FLUSS_CONFIG_PREFIXES = {"oss.", "fs.oss."}; + + private static final String HADOOP_CONFIG_PREFIX = "fs.oss."; public static final String REGION_KEY = "fs.oss.region"; @@ -127,15 +129,14 @@ org.apache.hadoop.conf.Configuration getHadoopConfiguration(Configuration flussC for (String key : flussConfig.keySet()) { for (String prefix : FLUSS_CONFIG_PREFIXES) { if (key.startsWith(prefix)) { + String newKey = HADOOP_CONFIG_PREFIX + key.substring(prefix.length()); String value = flussConfig.getString( ConfigBuilder.key(key).stringType().noDefaultValue(), null); - conf.set(key, value); + conf.set(newKey, value); LOG.debug( - "Adding Fluss config entry for {} as {} to Hadoop config", - key, - conf.get(key)); + "Adding Fluss config entry for {} as {} to Hadoop config", key, newKey); } } }