From 303b8f5bd6b8c8700cbc7f2a3f22dd624b03934f Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Wed, 23 Oct 2024 10:52:50 -0700 Subject: [PATCH 1/2] HDDS-11697. Implement ListStatus Light API implement and integrate it with Ozone Filesystem API Change-Id: I7695087b00fc9f16bfb68f1aaae4bc1e7e5323f8 --- .../hadoop/ozone/client/OzoneBucket.java | 19 ++++- .../ozone/client/protocol/ClientProtocol.java | 16 +++++ .../hadoop/ozone/client/rpc/RpcClient.java | 16 +++++ .../ozone/om/helpers/BasicOmKeyInfo.java | 4 ++ .../om/protocol/OzoneManagerProtocol.java | 14 ++++ ...ManagerProtocolClientSideTranslatorPB.java | 6 ++ .../fs/ozone/AbstractOzoneFileSystemTest.java | 35 +++++++++ .../AbstractRootedOzoneFileSystemTest.java | 2 +- ...estOzoneRpcClientWithKeyLatestVersion.java | 7 ++ .../hadoop/ozone/om/OmMetadataReader.java | 4 +- .../apache/hadoop/ozone/om/OzoneManager.java | 6 ++ .../fs/ozone/BasicOzoneClientAdapterImpl.java | 46 ++++++++++-- .../hadoop/fs/ozone/BasicOzoneFileSystem.java | 18 ++--- .../BasicRootedOzoneClientAdapterImpl.java | 71 ++++++++++++++----- .../fs/ozone/BasicRootedOzoneFileSystem.java | 26 +++---- .../hadoop/fs/ozone/OzoneClientAdapter.java | 3 +- .../hadoop/fs/ozone/OzoneClientUtils.java | 6 ++ .../ozone/client/ClientProtocolStub.java | 7 ++ 18 files changed, 258 insertions(+), 48 deletions(-) diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java index 216b51b8e86..ebd708ffec9 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java @@ -985,8 +985,23 @@ public OzoneDataStreamOutput createStreamFile(String keyName, long size, */ public List listStatus(String keyName, boolean recursive, String startKey, long numEntries) throws IOException { - return proxy - .listStatus(volumeName, name, keyName, recursive, startKey, numEntries); + return proxy.listStatus(volumeName, name, keyName, recursive, startKey, numEntries); + } + + /** + * List the lightweight status for a file or a directory and its contents. + * + * @param keyName Absolute path of the entry to be listed + * @param recursive For a directory if true all the descendants of a + * particular directory are listed + * @param startKey Key from which listing needs to start. If startKey exists + * its status is included in the final list. + * @param numEntries Number of entries to list from the start key + * @return list of file status + */ + public List listStatusLight(String keyName, boolean recursive, + String startKey, long numEntries) throws IOException { + return proxy.listStatusLight(volumeName, name, keyName, recursive, startKey, numEntries); } /** diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java index 8d9614b554a..b9623efd570 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/protocol/ClientProtocol.java @@ -989,6 +989,22 @@ List listStatus(String volumeName, String bucketName, String keyName, boolean recursive, String startKey, long numEntries, boolean allowPartialPrefixes) throws IOException; + /** + * Lightweight listStatus API. + * + * @param volumeName Volume name + * @param bucketName Bucket name + * @param keyName Absolute path of the entry to be listed + * @param recursive For a directory if true all the descendants of a + * particular directory are listed + * @param startKey Key from which listing needs to start. If startKey exists + * its status is included in the final list. + * @param numEntries Number of entries to list from the start key + * @return list of file status + */ + List listStatusLight(String volumeName, + String bucketName, String keyName, boolean recursive, String startKey, long numEntries) throws IOException; + /** * Lightweight listStatus API. * diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index fe986640176..727164e908f 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@ -2311,6 +2311,22 @@ public List listStatus(String volumeName, String bucketName, allowPartialPrefixes); } + @Override + public List listStatusLight(String volumeName, String bucketName, String keyName, + boolean recursive, String startKey, long numEntries) throws IOException { + OmKeyArgs keyArgs = prepareOmKeyArgs(volumeName, bucketName, keyName); + if (omVersion.compareTo(OzoneManagerVersion.LIGHTWEIGHT_LIST_STATUS) >= 0) { + return ozoneManagerClient.listStatusLight(keyArgs, recursive, startKey, + numEntries); + } else { + return ozoneManagerClient.listStatus(keyArgs, recursive, startKey, + numEntries) + .stream() + .map(OzoneFileStatusLight::fromOzoneFileStatus) + .collect(Collectors.toList()); + } + } + @Override public List listStatusLight(String volumeName, String bucketName, String keyName, boolean recursive, String startKey, diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BasicOmKeyInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BasicOmKeyInfo.java index a9fa742a108..82b9d8cccfb 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BasicOmKeyInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BasicOmKeyInfo.java @@ -110,6 +110,10 @@ public String getOwnerName() { return ownerName; } + public long getReplicatedSize() { + return QuotaUtil.getReplicatedSize(getDataSize(), replicationConfig); + } + /** * Builder of BasicOmKeyInfo. */ diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java index 94822630f8e..1f95efee464 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java @@ -986,6 +986,20 @@ List listStatus(OmKeyArgs keyArgs, boolean recursive, boolean allowPartialPrefixes) throws IOException; + /** + * Lightweight listStatus API. + * + * @param keyArgs Key args + * @param recursive For a directory if true all the descendants of a + * particular directory are listed + * @param startKey Key from which listing needs to start. If startKey exists + * its status is included in the final list. + * @param numEntries Number of entries to list from the start key + * @return list of file status + */ + List listStatusLight(OmKeyArgs keyArgs, boolean recursive, String startKey, + long numEntries) throws IOException; + /** * Lightweight listStatus API. * diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java index b140cf95e69..9f8b244a821 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java @@ -2374,6 +2374,12 @@ public List listStatus(OmKeyArgs args, boolean recursive, return statusList; } + @Override + public List listStatusLight(OmKeyArgs args, + boolean recursive, String startKey, long numEntries) throws IOException { + return listStatusLight(args, recursive, startKey, numEntries, false); + } + @Override public List listStatusLight(OmKeyArgs args, boolean recursive, String startKey, long numEntries, diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java index e7c4cbee1d5..5dd8f7db862 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java @@ -2112,6 +2112,41 @@ void testListStatus2() throws IOException { } } + @Test + public void testOzoneManagerListLocatedStatusAndListStatus() throws IOException { + String data = RandomStringUtils.randomAlphanumeric(20); + String directory = RandomStringUtils.randomAlphanumeric(5); + String filePath = RandomStringUtils.randomAlphanumeric(5); + Path path = createPath("/" + directory + "/" + filePath); + try (FSDataOutputStream stream = fs.create(path)) { + stream.writeBytes(data); + } + RemoteIterator listLocatedStatus = fs.listLocatedStatus(path); + int count = 0; + while (listLocatedStatus.hasNext()) { + LocatedFileStatus locatedFileStatus = listLocatedStatus.next(); + assertTrue(locatedFileStatus.getBlockLocations().length >= 1); + + for (BlockLocation blockLocation : locatedFileStatus.getBlockLocations()) { + assertTrue(blockLocation.getNames().length >= 1); + assertTrue(blockLocation.getHosts().length >= 1); + } + count++; + } + assertEquals(1, count); + count = 0; + RemoteIterator listStatus = fs.listStatusIterator(path); + while (listStatus.hasNext()) { + FileStatus fileStatus = listStatus.next(); + assertFalse(fileStatus instanceof LocatedFileStatus); + count++; + } + assertEquals(1, count); + FileStatus[] fileStatuses = fs.listStatus(path.getParent()); + assertEquals(1, fileStatuses.length); + assertFalse(fileStatuses[0] instanceof LocatedFileStatus); + } + @Test void testOzoneManagerFileSystemInterface() throws IOException { String dirPath = RandomStringUtils.randomAlphanumeric(5); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java index cfc9029019a..8b71a216003 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java @@ -1071,7 +1071,7 @@ private void listStatusRecursiveHelper(Path curPath, List result) private List callAdapterListStatus(String pathStr, boolean recursive, String startPath, long numEntries) throws IOException { return adapter.listStatus(pathStr, recursive, startPath, numEntries, - ofs.getUri(), ofs.getWorkingDirectory(), ofs.getUsername()) + ofs.getUri(), ofs.getWorkingDirectory(), ofs.getUsername(), false) .stream().map(ofs::convertFileStatus).collect(Collectors.toList()); } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientWithKeyLatestVersion.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientWithKeyLatestVersion.java index fd32698eec2..62429368690 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientWithKeyLatestVersion.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientWithKeyLatestVersion.java @@ -31,6 +31,7 @@ import org.apache.hadoop.ozone.client.OzoneClientFactory; import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; +import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.TestInstance; @@ -151,5 +152,11 @@ private void assertListStatus(OzoneBucket bucket, String keyName, List versions = files.get(0).getKeyInfo().getKeyLocationVersions(); assertEquals(expectedVersionCount, versions.size()); + + List lightFiles = bucket.listStatusLight(keyName, false, "", 1); + + assertNotNull(lightFiles); + assertEquals(1, lightFiles.size()); + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java index fdee1b71287..20ef5ac7bd1 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataReader.java @@ -264,7 +264,7 @@ public List listStatusLight(OmKeyArgs args, .map(OzoneFileStatusLight::fromOzoneFileStatus) .collect(Collectors.toList()); } - + @Override public OzoneFileStatus getFileStatus(OmKeyArgs args) throws IOException { ResolvedBucket bucket = ozoneManager.resolveBucketLink(args); @@ -501,7 +501,7 @@ void checkAcls(ResourceType resType, StoreType store, ozoneManager.getOmRpcServerAddr().getHostName()); } - + /** * CheckAcls for the ozone object. * diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 2bb8c915b3c..dc672790114 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -3802,6 +3802,12 @@ public List listStatus(OmKeyArgs args, boolean recursive, } } + @Override + public List listStatusLight(OmKeyArgs args, + boolean recursive, String startKey, long numEntries) throws IOException { + return listStatusLight(args, recursive, startKey, numEntries, false); + } + @Override public List listStatusLight(OmKeyArgs args, boolean recursive, String startKey, long numEntries, diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java index bc7c0120446..689e340ff5d 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java @@ -69,6 +69,7 @@ import org.apache.hadoop.ozone.client.rpc.RpcClient; import org.apache.hadoop.ozone.container.common.helpers.BlockData; import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.om.helpers.BasicOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.LeaseKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; @@ -77,6 +78,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; +import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight; import org.apache.hadoop.ozone.security.OzoneTokenIdentifier; import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone; import org.apache.hadoop.ozone.snapshot.SnapshotDiffResponse; @@ -436,15 +438,22 @@ public Iterator listKeys(String pathKey) throws IOException { @Override public List listStatus(String keyName, boolean recursive, String startKey, long numEntries, URI uri, - Path workingDir, String username) throws IOException { + Path workingDir, String username, boolean lite) throws IOException { try { incrementCounter(Statistic.OBJECTS_LIST, 1); - List statuses = bucket - .listStatus(keyName, recursive, startKey, numEntries); - List result = new ArrayList<>(); - for (OzoneFileStatus status : statuses) { - result.add(toFileStatusAdapter(status, username, uri, workingDir)); + if (lite) { + List statuses = bucket + .listStatusLight(keyName, recursive, startKey, numEntries); + for (OzoneFileStatusLight status : statuses) { + result.add(toFileStatusAdapter(status, username, uri, workingDir)); + } + } else { + List statuses = bucket + .listStatus(keyName, recursive, startKey, numEntries); + for (OzoneFileStatus status : statuses) { + result.add(toFileStatusAdapter(status, username, uri, workingDir)); + } } return result; } catch (OMException e) { @@ -549,6 +558,31 @@ private FileStatusAdapter toFileStatusAdapter(OzoneFileStatus status, ); } + private FileStatusAdapter toFileStatusAdapter(OzoneFileStatusLight status, + String owner, URI defaultUri, Path workingDir) { + BasicOmKeyInfo keyInfo = status.getKeyInfo(); + short replication = (short) keyInfo.getReplicationConfig() + .getRequiredNodes(); + return new FileStatusAdapter( + keyInfo.getDataSize(), + keyInfo.getReplicatedSize(), + new Path(OZONE_URI_DELIMITER + keyInfo.getKeyName()) + .makeQualified(defaultUri, workingDir), + status.isDirectory(), + replication, + status.getBlockSize(), + keyInfo.getModificationTime(), + keyInfo.getModificationTime(), + status.isDirectory() ? (short) 00777 : (short) 00666, + StringUtils.defaultIfEmpty(keyInfo.getOwnerName(), owner), + owner, + null, + getBlockLocations(null), + false, + OzoneClientUtils.isKeyErasureCode(keyInfo) + ); + } + /** * Helper method to get List of BlockLocation from OM Key info. * @param fileStatus Ozone key file status. diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java index ed5574af32b..d2c26ecf3cd 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java @@ -688,7 +688,7 @@ public FileStatus[] listStatus(Path f) throws IOException { do { tmpStatusList = adapter.listStatus(pathToKey(f), false, startKey, numEntries, uri, - workingDir, getUsername()) + workingDir, getUsername(), true) .stream() .map(this::convertFileStatus) .collect(Collectors.toList()); @@ -947,13 +947,13 @@ public RemoteIterator listFiles(Path f, boolean recursive) public RemoteIterator listLocatedStatus(Path f) throws IOException { incrementCounter(Statistic.INVOCATION_LIST_LOCATED_STATUS); - return super.listLocatedStatus(f); + return new OzoneFileStatusIterator<>(f, false); } @Override public RemoteIterator listStatusIterator(Path f) throws IOException { - return new OzoneFileStatusIterator<>(f); + return new OzoneFileStatusIterator<>(f, true); } @Override @@ -999,6 +999,7 @@ private final class OzoneFileStatusIterator private Path p; private T curStat = null; private String startPath = ""; + private boolean lite; /** * Constructor to initialize OzoneFileStatusIterator. @@ -1007,10 +1008,11 @@ private final class OzoneFileStatusIterator * @param p path to file/directory. * @throws IOException */ - private OzoneFileStatusIterator(Path p) throws IOException { + private OzoneFileStatusIterator(Path p, boolean lite) throws IOException { this.p = p; + this.lite = lite; // fetch the first batch of entries in the directory - thisListing = listFileStatus(p, startPath); + thisListing = listFileStatus(p, startPath, lite); if (thisListing != null && !thisListing.isEmpty()) { startPath = pathToKey( thisListing.get(thisListing.size() - 1).getPath()); @@ -1050,7 +1052,7 @@ private boolean hasNextNoFilter() throws IOException { if (startPath != null && (thisListing.size() == listingPageSize || thisListing.size() == listingPageSize - 1)) { // current listing is exhausted & fetch a new listing - thisListing = listFileStatus(p, startPath); + thisListing = listFileStatus(p, startPath, lite); if (thisListing != null && !thisListing.isEmpty()) { startPath = pathToKey( thisListing.get(thisListing.size() - 1).getPath()); @@ -1088,7 +1090,7 @@ public T next() throws IOException { * @return list of file status. * @throws IOException */ - private List listFileStatus(Path f, String startPath) + private List listFileStatus(Path f, String startPath, boolean lite) throws IOException { incrementCounter(Statistic.INVOCATION_LIST_STATUS, 1); statistics.incrementReadOps(1); @@ -1096,7 +1098,7 @@ private List listFileStatus(Path f, String startPath) List statusList; statusList = adapter.listStatus(pathToKey(f), false, startPath, - listingPageSize, uri, workingDir, getUsername()) + listingPageSize, uri, workingDir, getUsername(), lite) .stream() .map(this::convertFileStatus) .collect(Collectors.toList()); diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java index 46602068ccd..039c4ad898f 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java @@ -79,6 +79,7 @@ import org.apache.hadoop.ozone.client.rpc.RpcClient; import org.apache.hadoop.ozone.container.common.helpers.BlockData; import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.om.helpers.BasicOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.LeaseKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; @@ -87,6 +88,7 @@ import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; +import org.apache.hadoop.ozone.om.helpers.OzoneFileStatusLight; import org.apache.hadoop.ozone.om.helpers.SnapshotInfo; import org.apache.hadoop.ozone.security.OzoneTokenIdentifier; import org.apache.hadoop.ozone.snapshot.SnapshotDiffReportOzone; @@ -784,7 +786,7 @@ public Iterator listKeys(String pathStr) throws IOException { */ private List listStatusRoot( boolean recursive, String startPath, long numEntries, - URI uri, Path workingDir, String username) throws IOException { + URI uri, Path workingDir, String username, boolean lite) throws IOException { OFSPath ofsStartPath = new OFSPath(startPath, config); // list volumes @@ -797,7 +799,7 @@ private List listStatusRoot( if (recursive) { String pathStrNextVolume = volume.getName(); res.addAll(listStatus(pathStrNextVolume, recursive, startPath, - numEntries - res.size(), uri, workingDir, username)); + numEntries - res.size(), uri, workingDir, username, lite)); } } return res; @@ -806,9 +808,10 @@ private List listStatusRoot( /** * Helper for OFS listStatus on a volume. */ + @SuppressWarnings("checkstyle:ParameterNumber") private List listStatusVolume(String volumeStr, boolean recursive, String startPath, long numEntries, - URI uri, Path workingDir, String username) throws IOException { + URI uri, Path workingDir, String username, boolean lite) throws IOException { OFSPath ofsStartPath = new OFSPath(startPath, config); // list buckets in the volume @@ -822,7 +825,7 @@ private List listStatusVolume(String volumeStr, if (recursive) { String pathStrNext = volumeStr + OZONE_URI_DELIMITER + bucket.getName(); res.addAll(listStatus(pathStrNext, recursive, startPath, - numEntries - res.size(), uri, workingDir, username)); + numEntries - res.size(), uri, workingDir, username, lite)); } } return res; @@ -874,13 +877,15 @@ private List listStatusBucketSnapshot( * Used in making the return path qualified. * @param username User name. * Used in making the return path qualified. + * @param lite true if lightweight response needs to be returned otherwise false. * @return A list of FileStatusAdapter. * @throws IOException Bucket exception or FileNotFoundException. */ + @SuppressWarnings("checkstyle:ParameterNumber") @Override public List listStatus(String pathStr, boolean recursive, String startPath, long numEntries, URI uri, - Path workingDir, String username) throws IOException { + Path workingDir, String username, boolean lite) throws IOException { incrementCounter(Statistic.OBJECTS_LIST, 1); // Remove authority from startPath if it exists @@ -899,44 +904,53 @@ public List listStatus(String pathStr, boolean recursive, OFSPath ofsPath = new OFSPath(pathStr, config); if (ofsPath.isRoot()) { return listStatusRoot( - recursive, startPath, numEntries, uri, workingDir, username); + recursive, startPath, numEntries, uri, workingDir, username, lite); } OFSPath ofsStartPath = new OFSPath(startPath, config); if (ofsPath.isVolume()) { String startBucketPath = ofsStartPath.getNonKeyPath(); return listStatusVolume(ofsPath.getVolumeName(), - recursive, startBucketPath, numEntries, uri, workingDir, username); + recursive, startBucketPath, numEntries, uri, workingDir, username, lite); } if (ofsPath.isSnapshotPath()) { return listStatusBucketSnapshot(ofsPath.getVolumeName(), ofsPath.getBucketName(), uri); } - + List result = new ArrayList<>(); String keyName = ofsPath.getKeyName(); // Internally we need startKey to be passed into bucket.listStatus String startKey = ofsStartPath.getKeyName(); try { OzoneBucket bucket = getBucket(ofsPath, false); - List statuses; + List statuses = Collections.emptyList(); + List lightStatuses = Collections.emptyList(); if (bucket.isSourcePathExist()) { - statuses = bucket - .listStatus(keyName, recursive, startKey, numEntries); + if (lite) { + lightStatuses = bucket.listStatusLight(keyName, recursive, startKey, numEntries); + } else { + statuses = bucket.listStatus(keyName, recursive, startKey, numEntries); + } + } else { LOG.warn("Source Bucket does not exist, link bucket {} is orphan " + "and returning empty list of files inside it", bucket.getName()); - statuses = Collections.emptyList(); } // Note: result in statuses above doesn't have volume/bucket path since // they are from the server. String ofsPathPrefix = ofsPath.getNonKeyPath(); - List result = new ArrayList<>(); - for (OzoneFileStatus status : statuses) { - result.add(toFileStatusAdapter(status, username, uri, workingDir, - ofsPathPrefix)); + if (lite) { + for (OzoneFileStatusLight status : lightStatuses) { + result.add(toFileStatusAdapter(status, username, uri, workingDir, ofsPathPrefix)); + } + } else { + for (OzoneFileStatus status : statuses) { + result.add(toFileStatusAdapter(status, username, uri, workingDir, ofsPathPrefix)); + } } + return result; } catch (OMException e) { if (e.getResult() == OMException.ResultCodes.FILE_NOT_FOUND) { @@ -1039,6 +1053,31 @@ private FileStatusAdapter toFileStatusAdapter(OzoneFileStatus status, ); } + private FileStatusAdapter toFileStatusAdapter(OzoneFileStatusLight status, + String owner, URI defaultUri, Path workingDir, String ofsPathPrefix) { + BasicOmKeyInfo keyInfo = status.getKeyInfo(); + short replication = (short) keyInfo.getReplicationConfig() + .getRequiredNodes(); + return new FileStatusAdapter( + keyInfo.getDataSize(), + keyInfo.getReplicatedSize(), + new Path(ofsPathPrefix + OZONE_URI_DELIMITER + keyInfo.getKeyName()) + .makeQualified(defaultUri, workingDir), + status.isDirectory(), + replication, + status.getBlockSize(), + keyInfo.getModificationTime(), + keyInfo.getModificationTime(), + status.isDirectory() ? (short) 00777 : (short) 00666, + StringUtils.defaultIfEmpty(keyInfo.getOwnerName(), owner), + owner, + null, + getBlockLocations(null), + false, + OzoneClientUtils.isKeyErasureCode(keyInfo) + ); + } + /** * Helper method to get List of BlockLocation from OM Key info. * @param fileStatus Ozone key file status. diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java index 3e0a3730627..592678204cf 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java @@ -915,7 +915,7 @@ private boolean o3Exists(final Path f) throws IOException { @Override public FileStatus[] listStatus(Path f) throws IOException { return TracingUtil.executeInNewSpan("ofs listStatus", - () -> convertFileStatusArr(listStatusAdapter(f))); + () -> convertFileStatusArr(listStatusAdapter(f, true))); } private FileStatus[] convertFileStatusArr( @@ -929,7 +929,7 @@ private FileStatus[] convertFileStatusArr( } - public List listStatusAdapter(Path f) throws IOException { + private List listStatusAdapter(Path f, boolean lite) throws IOException { incrementCounter(Statistic.INVOCATION_LIST_STATUS, 1); statistics.incrementReadOps(1); LOG.trace("listStatus() path:{}", f); @@ -941,7 +941,7 @@ public List listStatusAdapter(Path f) throws IOException { do { tmpStatusList = adapter.listStatus(pathToKey(f), false, startPath, - numEntries, uri, workingDir, getUsername()); + numEntries, uri, workingDir, getUsername(), lite); if (!tmpStatusList.isEmpty()) { if (startPath.isEmpty() || !statuses.getLast().getPath().toString() @@ -1178,7 +1178,7 @@ public RemoteIterator listFiles(Path f, boolean recursive) public RemoteIterator listLocatedStatus(Path f) throws IOException { incrementCounter(Statistic.INVOCATION_LIST_LOCATED_STATUS); - return super.listLocatedStatus(f); + return new OzoneFileStatusIterator<>(f, false); } @Override @@ -1193,7 +1193,7 @@ public RemoteIterator listStatusIterator(Path f) "Instead use 'ozone sh key list " + "' command"); } - return new OzoneFileStatusIterator<>(f); + return new OzoneFileStatusIterator<>(f, true); } /** @@ -1208,6 +1208,7 @@ private final class OzoneFileStatusIterator private Path p; private T curStat = null; private String startPath = ""; + private boolean lite; /** * Constructor to initialize OzoneFileStatusIterator. @@ -1216,10 +1217,11 @@ private final class OzoneFileStatusIterator * @param p path to file/directory. * @throws IOException */ - private OzoneFileStatusIterator(Path p) throws IOException { + private OzoneFileStatusIterator(Path p, boolean lite) throws IOException { this.p = p; + this.lite = lite; // fetch the first batch of entries in the directory - thisListing = listFileStatus(p, startPath); + thisListing = listFileStatus(p, startPath, lite); if (thisListing != null && !thisListing.isEmpty()) { startPath = pathToKey( thisListing.get(thisListing.size() - 1).getPath()); @@ -1259,7 +1261,7 @@ private boolean hasNextNoFilter() throws IOException { if (startPath != null && (thisListing.size() == listingPageSize || thisListing.size() == listingPageSize - 1)) { // current listing is exhausted & fetch a new listing - thisListing = listFileStatus(p, startPath); + thisListing = listFileStatus(p, startPath, lite); if (thisListing != null && !thisListing.isEmpty()) { startPath = pathToKey( thisListing.get(thisListing.size() - 1).getPath()); @@ -1297,7 +1299,7 @@ public T next() throws IOException { * @return list of file status. * @throws IOException */ - private List listFileStatus(Path f, String startPath) + private List listFileStatus(Path f, String startPath, boolean lite) throws IOException { incrementCounter(Statistic.INVOCATION_LIST_STATUS, 1); statistics.incrementReadOps(1); @@ -1305,7 +1307,7 @@ private List listFileStatus(Path f, String startPath) List statusList; statusList = adapter.listStatus(pathToKey(f), false, startPath, - listingPageSize, uri, workingDir, getUsername()) + listingPageSize, uri, workingDir, getUsername(), lite) .stream() .map(this::convertFileStatus) .collect(Collectors.toList()); @@ -1442,7 +1444,7 @@ boolean iterate() throws IOException { ofsPath.getNonKeyPathNoPrefixDelim() + OZONE_URI_DELIMITER; if (isFSO) { List fileStatuses; - fileStatuses = listStatusAdapter(path); + fileStatuses = listStatusAdapter(path, true); for (FileStatusAdapter fileStatus : fileStatuses) { String keyName = new OFSPath(fileStatus.getPath().toString(), @@ -1567,7 +1569,7 @@ private ContentSummary getContentSummaryInSpan(Path f) throws IOException { // f is a directory long[] summary = {0, 0, 0, 1}; int i = 0; - for (FileStatusAdapter s : listStatusAdapter(f)) { + for (FileStatusAdapter s : listStatusAdapter(f, true)) { long length = s.getLength(); long spaceConsumed = s.getDiskConsumed(); ContentSummary c = s.isDir() ? getContentSummary(s.getPath()) : diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapter.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapter.java index e468ac498c4..24ff692e1b4 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapter.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapter.java @@ -69,9 +69,10 @@ OzoneFSDataStreamOutput createStreamFile(String key, short replication, Iterator listKeys(String pathKey) throws IOException; + @SuppressWarnings("checkstyle:ParameterNumber") List listStatus(String keyName, boolean recursive, String startKey, long numEntries, URI uri, - Path workingDir, String username) throws IOException; + Path workingDir, String username, boolean lite) throws IOException; Token getDelegationToken(String renewer) throws IOException; diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientUtils.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientUtils.java index 383ad6db495..6c9fb3ccc7b 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientUtils.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientUtils.java @@ -32,6 +32,7 @@ import org.apache.hadoop.ozone.client.checksum.ChecksumHelperFactory; import org.apache.hadoop.ozone.client.protocol.ClientProtocol; import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.om.helpers.BasicOmKeyInfo; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmKeyArgs; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; @@ -246,6 +247,11 @@ public static boolean isKeyErasureCode(OmKeyInfo keyInfo) { HddsProtos.ReplicationType.EC; } + public static boolean isKeyErasureCode(BasicOmKeyInfo keyInfo) { + return keyInfo.getReplicationConfig().getReplicationType() == + HddsProtos.ReplicationType.EC; + } + public static boolean isKeyEncrypted(OmKeyInfo keyInfo) { return !Objects.isNull(keyInfo.getFileEncryptionInfo()); } diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java index e3e3537b1c3..7225216b702 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java @@ -564,6 +564,13 @@ public List listStatus(String volumeName, String bucketName, return null; } + @Override + public List listStatusLight(String volumeName, String bucketName, String keyName, + boolean recursive, String startKey, long numEntries) + throws IOException { + return null; + } + @Override public List listStatusLight(String volumeName, String bucketName, String keyName, boolean recursive, String startKey, From bd264b5d415466743cf25825f9d4d75d1af26143 Mon Sep 17 00:00:00 2001 From: Swaminathan Balachandran Date: Fri, 15 Nov 2024 23:30:24 -0800 Subject: [PATCH 2/2] HDDS-11697. Fix list located status api Change-Id: I5eafb02822ea5e8bfe1adbe58d9fc8a601880249 --- .../hadoop/fs/ozone/BasicOzoneFileSystem.java | 14 +++++++++----- .../fs/ozone/BasicRootedOzoneFileSystem.java | 13 +++++++++---- 2 files changed, 18 insertions(+), 9 deletions(-) diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java index d2c26ecf3cd..c1f00b6d46e 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java @@ -18,6 +18,7 @@ package org.apache.hadoop.fs.ozone; +import com.google.common.base.Function; import com.google.common.base.Preconditions; import org.apache.hadoop.conf.Configuration; @@ -947,13 +948,15 @@ public RemoteIterator listFiles(Path f, boolean recursive) public RemoteIterator listLocatedStatus(Path f) throws IOException { incrementCounter(Statistic.INVOCATION_LIST_LOCATED_STATUS); - return new OzoneFileStatusIterator<>(f, false); + return new OzoneFileStatusIterator<>(f, + (stat) -> stat instanceof LocatedFileStatus ? (LocatedFileStatus) stat : new LocatedFileStatus(stat, null), + false); } @Override public RemoteIterator listStatusIterator(Path f) throws IOException { - return new OzoneFileStatusIterator<>(f, true); + return new OzoneFileStatusIterator<>(f, stat -> stat, true); } @Override @@ -986,7 +989,6 @@ public void setTimes(Path f, long mtime, long atime) throws IOException { String key = pathToKey(qualifiedPath); adapter.setTimes(key, mtime, atime); } - /** * A private class implementation for iterating list of file status. * @@ -1000,6 +1002,7 @@ private final class OzoneFileStatusIterator private T curStat = null; private String startPath = ""; private boolean lite; + private Function transformFunc; /** * Constructor to initialize OzoneFileStatusIterator. @@ -1008,9 +1011,10 @@ private final class OzoneFileStatusIterator * @param p path to file/directory. * @throws IOException */ - private OzoneFileStatusIterator(Path p, boolean lite) throws IOException { + private OzoneFileStatusIterator(Path p, Function transformFunc, boolean lite) throws IOException { this.p = p; this.lite = lite; + this.transformFunc = transformFunc; // fetch the first batch of entries in the directory thisListing = listFileStatus(p, startPath, lite); if (thisListing != null && !thisListing.isEmpty()) { @@ -1031,7 +1035,7 @@ public boolean hasNext() throws IOException { while (curStat == null && hasNextNoFilter()) { T next; FileStatus fileStat = thisListing.get(i++); - next = (T) (fileStat); + next = this.transformFunc.apply(fileStat); curStat = next; } return curStat != null; diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java index 592678204cf..de18c454b2f 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.fs.ozone; +import com.google.common.base.Function; import com.google.common.base.Preconditions; import io.opentracing.Span; import io.opentracing.util.GlobalTracer; @@ -1178,7 +1179,9 @@ public RemoteIterator listFiles(Path f, boolean recursive) public RemoteIterator listLocatedStatus(Path f) throws IOException { incrementCounter(Statistic.INVOCATION_LIST_LOCATED_STATUS); - return new OzoneFileStatusIterator<>(f, false); + return new OzoneFileStatusIterator<>(f, + (stat) -> stat instanceof LocatedFileStatus ? (LocatedFileStatus) stat : new LocatedFileStatus(stat, null), + false); } @Override @@ -1193,7 +1196,7 @@ public RemoteIterator listStatusIterator(Path f) "Instead use 'ozone sh key list " + "' command"); } - return new OzoneFileStatusIterator<>(f, true); + return new OzoneFileStatusIterator<>(f, stat -> stat, true); } /** @@ -1203,6 +1206,7 @@ public RemoteIterator listStatusIterator(Path f) */ private final class OzoneFileStatusIterator implements RemoteIterator { + private final Function transformFunc; private List thisListing; private int i; private Path p; @@ -1217,9 +1221,10 @@ private final class OzoneFileStatusIterator * @param p path to file/directory. * @throws IOException */ - private OzoneFileStatusIterator(Path p, boolean lite) throws IOException { + private OzoneFileStatusIterator(Path p, Function transformFunc, boolean lite) throws IOException { this.p = p; this.lite = lite; + this.transformFunc = transformFunc; // fetch the first batch of entries in the directory thisListing = listFileStatus(p, startPath, lite); if (thisListing != null && !thisListing.isEmpty()) { @@ -1240,7 +1245,7 @@ public boolean hasNext() throws IOException { while (curStat == null && hasNextNoFilter()) { T next; FileStatus fileStat = thisListing.get(i++); - next = (T) (fileStat); + next = transformFunc.apply(fileStat); curStat = next; } return curStat != null;