-
Notifications
You must be signed in to change notification settings - Fork 2.9k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Add regx pattern file filter for distributed load
Add regx pattern file filter for distributed load. **Example:** The following request allows us to load the files under `/test-load` directory with "hello" prefix: `curl -X GET http://localhost:28080/v1/load?path=s3a://jiamingmai-test/test-load&opType=submit&verbose=true&fileFilterRegx=^hello.*` pr-link: #18311 change-id: cid-4ec2bfe58bfba413f6d2925f5b3937bd6f5c2eb1
- Loading branch information
1 parent
53c49f7
commit d0ad98f
Showing
19 changed files
with
316 additions
and
31 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
122 changes: 122 additions & 0 deletions
122
dora/core/server/master/src/main/java/alluxio/master/predicate/FileNamePatternPredicate.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,122 @@ | ||
| /* | ||
| * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 | ||
| * (the "License"). You may not use this work except in compliance with the License, which is | ||
| * available at www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, | ||
| * either express or implied, as more fully set forth in the License. | ||
| * | ||
| * See the NOTICE file distributed with this work for information regarding copyright ownership. | ||
| */ | ||
|
|
||
| package alluxio.master.predicate; | ||
|
|
||
| import alluxio.proto.journal.Job.FileFilter; | ||
| import alluxio.underfs.UfsStatus; | ||
| import alluxio.wire.FileInfo; | ||
|
|
||
| import org.slf4j.Logger; | ||
| import org.slf4j.LoggerFactory; | ||
|
|
||
| import java.util.function.Predicate; | ||
| import java.util.regex.Pattern; | ||
|
|
||
| /** | ||
| * A predicate related to date of the file. | ||
| */ | ||
| public class FileNamePatternPredicate implements FilePredicate { | ||
| private static final Logger LOG = LoggerFactory.getLogger(FileNamePatternPredicate.class); | ||
| private final String mFilterName; | ||
| private String mRegexPatternStr; | ||
|
|
||
| /** | ||
| * Factory for modification time predicate. | ||
| */ | ||
| public static class FileNamePatternFactory extends Factory { | ||
| @Override | ||
| public String getFilterName() { | ||
| return "fileNamePattern"; | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Factory for creating instances. | ||
| */ | ||
| public abstract static class Factory implements FilePredicateFactory { | ||
| /** | ||
| * @return filter name for the predicate | ||
| */ | ||
| public abstract String getFilterName(); | ||
|
|
||
| /** | ||
| * Creates a {@link FilePredicate} from the string value. | ||
| * | ||
| * @param regexPatternStr the regex pattern string from the filter | ||
| * @return the created predicate | ||
| */ | ||
| public FilePredicate createFileNamePatternPredicate(String regexPatternStr) { | ||
| return new FileNamePatternPredicate(getFilterName(), regexPatternStr); | ||
| } | ||
|
|
||
| @Override | ||
| public FilePredicate create(FileFilter filter) { | ||
| try { | ||
| if (filter.hasName() && filter.getName().equals(getFilterName())) { | ||
| if (filter.hasValue()) { | ||
| return createFileNamePatternPredicate(filter.getValue()); | ||
| } | ||
| } | ||
| } catch (Exception e) { | ||
| // fall through | ||
| } | ||
| return null; | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Creates an instance. | ||
| * | ||
| * @param filterName the filter name | ||
| * @param regexPatternStr the regex pattern string for file filtering | ||
| */ | ||
| public FileNamePatternPredicate(String filterName, String regexPatternStr) { | ||
| mFilterName = filterName; | ||
| mRegexPatternStr = regexPatternStr; | ||
| } | ||
|
|
||
| @Override | ||
| public Predicate<FileInfo> get() { | ||
| return FileInfo -> { | ||
| try { | ||
| String fileName = FileInfo.getName(); | ||
| return Pattern.matches(mRegexPatternStr, fileName); | ||
| } catch (RuntimeException e) { | ||
| LOG.debug("Failed to filter: ", e); | ||
| return false; | ||
| } | ||
| }; | ||
| } | ||
|
|
||
| @Override | ||
| public Predicate<UfsStatus> getUfsStatusPredicate() { | ||
| return UfsStatus -> { | ||
| try { | ||
| String fileName = getFileName(UfsStatus); | ||
| return Pattern.matches(mRegexPatternStr, fileName); | ||
| } catch (RuntimeException e) { | ||
| LOG.debug("Failed to filter: ", e); | ||
| return false; | ||
| } | ||
| }; | ||
| } | ||
|
|
||
| private String getFileName(UfsStatus ufsStatus) { | ||
| String name = ufsStatus.getName(); | ||
| int index = name.lastIndexOf("/"); | ||
| if (index == -1) { | ||
| return name; | ||
| } | ||
| name = name.substring(index + 1); | ||
| return name; | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.