From e1e20606ebf6817c95bfa52cf18842ece9794836 Mon Sep 17 00:00:00 2001 From: maobaolong <307499405@qq.com> Date: Thu, 28 Oct 2021 09:33:44 +0800 Subject: [PATCH 1/4] Support use default block size to replace the HdfsUnderFileSystem block size. --- .../common/src/main/java/alluxio/conf/PropertyKey.java | 10 ++++++++++ .../java/alluxio/underfs/hdfs/HdfsUnderFileSystem.java | 6 +++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/core/common/src/main/java/alluxio/conf/PropertyKey.java b/core/common/src/main/java/alluxio/conf/PropertyKey.java index 1b1949ddf1c6..7bc816162150 100755 --- a/core/common/src/main/java/alluxio/conf/PropertyKey.java +++ b/core/common/src/main/java/alluxio/conf/PropertyKey.java @@ -3927,6 +3927,14 @@ public String toString() { .setDescription("Preferred media type while storing file's blocks.") .setScope(Scope.CLIENT) .build(); + + public static final PropertyKey USER_BLOCK_SIZE_ENABLED = + new Builder(Name.USER_BLOCK_SIZE_ENABLED) + .setDefaultValue(false) + .setDescription("Use Default block size for Alluxio files.") + .setConsistencyCheckLevel(ConsistencyCheckLevel.WARN) + .setScope(Scope.ALL) + .build(); public static final PropertyKey USER_BLOCK_SIZE_BYTES_DEFAULT = new Builder(Name.USER_BLOCK_SIZE_BYTES_DEFAULT) .setDefaultValue("64MB") @@ -6281,6 +6289,8 @@ public static final class Name { "alluxio.user.block.read.metrics.enabled"; public static final String USER_BLOCK_REMOTE_READ_BUFFER_SIZE_BYTES = "alluxio.user.block.remote.read.buffer.size.bytes"; + public static final String USER_BLOCK_SIZE_ENABLED = + "alluxio.user.block.size.enabled"; public static final String USER_BLOCK_SIZE_BYTES_DEFAULT = "alluxio.user.block.size.bytes.default"; public static final String USER_BLOCK_READ_RETRY_SLEEP_MIN = diff --git a/underfs/hdfs/src/main/java/alluxio/underfs/hdfs/HdfsUnderFileSystem.java b/underfs/hdfs/src/main/java/alluxio/underfs/hdfs/HdfsUnderFileSystem.java index 1ee2e3f6eca9..14b0880aab5f 100755 --- a/underfs/hdfs/src/main/java/alluxio/underfs/hdfs/HdfsUnderFileSystem.java +++ b/underfs/hdfs/src/main/java/alluxio/underfs/hdfs/HdfsUnderFileSystem.java @@ -395,8 +395,12 @@ public UfsFileStatus getFileStatus(String path) throws IOException { FileStatus fs = hdfs.getFileStatus(tPath); String contentHash = UnderFileSystemUtils.approximateContentHash(fs.getLen(), fs.getModificationTime()); + long blockSize = fs.getBlockSize(); + if (mUfsConf.getBoolean(PropertyKey.USER_BLOCK_SIZE_ENABLED)) { + blockSize = mUfsConf.getBytes(PropertyKey.USER_BLOCK_SIZE_BYTES_DEFAULT); + } return new UfsFileStatus(path, contentHash, fs.getLen(), fs.getModificationTime(), - fs.getOwner(), fs.getGroup(), fs.getPermission().toShort(), fs.getBlockSize()); + fs.getOwner(), fs.getGroup(), fs.getPermission().toShort(), blockSize); } @Override From 06c39f9f0aa6611c6ab4bfdbaf4c57c34c391b1b Mon Sep 17 00:00:00 2001 From: maobaolong <307499405@qq.com> Date: Wed, 22 Dec 2021 11:01:29 +0800 Subject: [PATCH 2/4] Move the customize block size logic to high level --- .../src/main/java/alluxio/master/file/InodeSyncStream.java | 3 +++ .../main/java/alluxio/underfs/hdfs/HdfsUnderFileSystem.java | 6 +----- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/core/server/master/src/main/java/alluxio/master/file/InodeSyncStream.java b/core/server/master/src/main/java/alluxio/master/file/InodeSyncStream.java index ba7a34178f96..9cf7e00cc282 100644 --- a/core/server/master/src/main/java/alluxio/master/file/InodeSyncStream.java +++ b/core/server/master/src/main/java/alluxio/master/file/InodeSyncStream.java @@ -880,6 +880,9 @@ static void loadFileMetadataInternal(RpcContext rpcContext, LockedInodePath inod } ufsLength = ((UfsFileStatus) context.getUfsStatus()).getContentLength(); long blockSize = ((UfsFileStatus) context.getUfsStatus()).getBlockSize(); + if (ServerConfiguration.getBoolean(PropertyKey.USER_BLOCK_SIZE_ENABLED)) { + blockSize = ServerConfiguration.getBytes(PropertyKey.USER_BLOCK_SIZE_BYTES_DEFAULT); + } ufsBlockSizeByte = blockSize != UfsFileStatus.UNKNOWN_BLOCK_SIZE ? blockSize : ufs.getBlockSizeByte(ufsUri.toString()); diff --git a/underfs/hdfs/src/main/java/alluxio/underfs/hdfs/HdfsUnderFileSystem.java b/underfs/hdfs/src/main/java/alluxio/underfs/hdfs/HdfsUnderFileSystem.java index 14b0880aab5f..1ee2e3f6eca9 100755 --- a/underfs/hdfs/src/main/java/alluxio/underfs/hdfs/HdfsUnderFileSystem.java +++ b/underfs/hdfs/src/main/java/alluxio/underfs/hdfs/HdfsUnderFileSystem.java @@ -395,12 +395,8 @@ public UfsFileStatus getFileStatus(String path) throws IOException { FileStatus fs = hdfs.getFileStatus(tPath); String contentHash = UnderFileSystemUtils.approximateContentHash(fs.getLen(), fs.getModificationTime()); - long blockSize = fs.getBlockSize(); - if (mUfsConf.getBoolean(PropertyKey.USER_BLOCK_SIZE_ENABLED)) { - blockSize = mUfsConf.getBytes(PropertyKey.USER_BLOCK_SIZE_BYTES_DEFAULT); - } return new UfsFileStatus(path, contentHash, fs.getLen(), fs.getModificationTime(), - fs.getOwner(), fs.getGroup(), fs.getPermission().toShort(), blockSize); + fs.getOwner(), fs.getGroup(), fs.getPermission().toShort(), fs.getBlockSize()); } @Override From 5584b61365b8e69de52bd164ed35e6e2a3a07e33 Mon Sep 17 00:00:00 2001 From: maobaolong <307499405@qq.com> Date: Wed, 12 Jun 2024 18:41:14 +0800 Subject: [PATCH 3/4] Update core/common/src/main/java/alluxio/conf/PropertyKey.java Co-authored-by: Jiacheng Liu --- core/common/src/main/java/alluxio/conf/PropertyKey.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/common/src/main/java/alluxio/conf/PropertyKey.java b/core/common/src/main/java/alluxio/conf/PropertyKey.java index 7bc816162150..d17f99b3d032 100755 --- a/core/common/src/main/java/alluxio/conf/PropertyKey.java +++ b/core/common/src/main/java/alluxio/conf/PropertyKey.java @@ -3928,7 +3928,7 @@ public String toString() { .setScope(Scope.CLIENT) .build(); - public static final PropertyKey USER_BLOCK_SIZE_ENABLED = + public static final PropertyKey USER_BLOCK_SIZE_OVERRIDE_UFS_ENABLED = new Builder(Name.USER_BLOCK_SIZE_ENABLED) .setDefaultValue(false) .setDescription("Use Default block size for Alluxio files.") From 5e8583d4b9abd26da64710af4115c945e944ea47 Mon Sep 17 00:00:00 2001 From: maobaolong <307499405@qq.com> Date: Wed, 12 Jun 2024 18:41:21 +0800 Subject: [PATCH 4/4] Update core/common/src/main/java/alluxio/conf/PropertyKey.java Co-authored-by: Jiacheng Liu --- core/common/src/main/java/alluxio/conf/PropertyKey.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/common/src/main/java/alluxio/conf/PropertyKey.java b/core/common/src/main/java/alluxio/conf/PropertyKey.java index d17f99b3d032..55b31590e97a 100755 --- a/core/common/src/main/java/alluxio/conf/PropertyKey.java +++ b/core/common/src/main/java/alluxio/conf/PropertyKey.java @@ -3931,7 +3931,7 @@ public String toString() { public static final PropertyKey USER_BLOCK_SIZE_OVERRIDE_UFS_ENABLED = new Builder(Name.USER_BLOCK_SIZE_ENABLED) .setDefaultValue(false) - .setDescription("Use Default block size for Alluxio files.") + .setDescription("When reading a file from UFS, whether Alluxio should ignore the UFS block size and rely on the Alluxio configuration.") .setConsistencyCheckLevel(ConsistencyCheckLevel.WARN) .setScope(Scope.ALL) .build();