Skip to content

Commit

Permalink
Further improvement
Browse files Browse the repository at this point in the history
  • Loading branch information
@apc999
apc999 committed Sep 6, 2023
1 parent b42ddb8 commit e2371ae
Show file tree
Hide file tree
Showing 15 changed files with 63 additions and 217 deletions.
9 changes: 9 additions & 0 deletions dora/core/common/src/main/java/alluxio/conf/PropertyKey.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6261,6 +6261,13 @@ public String toString() {
.setConsistencyCheckLevel(ConsistencyCheckLevel.IGNORE)
.setScope(Scope.CLIENT)
.build();
public static final PropertyKey FUSE_UPDATE_CHECK_ENABLED =
booleanBuilder(Name.FUSE_UPDATE_CHECK_ENABLED)
.setDefaultValue(Boolean.parseBoolean(ProjectConstants.UPDATE_CHECK_ENABLED))
.setDescription("Whether to check for update availability for alluxio-fuse")
.setConsistencyCheckLevel(ConsistencyCheckLevel.ENFORCE)
.setIsHidden(true)
.build();
public static final PropertyKey FUSE_USER_GROUP_TRANSLATION_ENABLED =
booleanBuilder(Name.FUSE_USER_GROUP_TRANSLATION_ENABLED)
.setDefaultValue(false)
Expand Down Expand Up @@ -8259,6 +8266,8 @@ public static final class Name {
"alluxio.fuse.stat.cache.refresh.interval";
public static final String FUSE_UMOUNT_TIMEOUT =
"alluxio.fuse.umount.timeout";
public static final String FUSE_UPDATE_CHECK_ENABLED =
"alluxio.fuse.update.check.enabled";
public static final String FUSE_USER_GROUP_TRANSLATION_ENABLED =
"alluxio.fuse.user.group.translation.enabled";
public static final String FUSE_SPECIAL_COMMAND_ENABLED =
Expand Down
13 changes: 0 additions & 13 deletions dora/core/common/src/main/java/alluxio/util/UpdateCheckUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -140,19 +140,6 @@ public static void addUserAgentEnvironments(List<String> info) {
}
}

/**
* Add feature name if condition is true.
*
* @param valid true, if condition is valid
* @param features feature list
* @param featureName feature name
*/
public static void addIfTrue(boolean valid, List<String> features, String featureName) {
if (valid) {
features.add(featureName);
}
}

/**
* Adds the information of EC2 environment to given list.
*
Expand Down
1 change: 0 additions & 1 deletion dora/core/common/src/test/java/alluxio/util/UpdateCheckUtilsTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -159,5 +159,4 @@ public void userAgent() {
Matcher matcher = pattern.matcher(userAgentString);
Assert.assertTrue(matcher.matches());
}

}
3 changes: 2 additions & 1 deletion dora/core/server/master/src/main/java/alluxio/master/meta/MasterUpdateChecker.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@
public final class MasterUpdateChecker implements HeartbeatExecutor {
private static final Logger LOG = LoggerFactory.getLogger(MasterUpdateChecker.class);
private static final String NUM_WORKER_INFO_FORMAT = "numWorkers:%s";

private static final String TYPE_MASTER = "master";
private final MetaMaster mMetaMaster;

/**
Expand All @@ -53,6 +53,7 @@ public void heartbeat(long timeLimitMs) {
// TODO(lu) use -1 here since we cannot distinguish
// no worker vs cluster not ready (still registering) cases
clusterSize > 0 ? clusterSize : -1));
additionalInfo.add(TYPE_MASTER);
String latestVersion =
UpdateCheckUtils.getLatestVersion(mMetaMaster.getClusterID(), additionalInfo);
if (!ProjectConstants.VERSION.equals(latestVersion)) {
Expand Down
15 changes: 7 additions & 8 deletions dora/integration/fuse/src/main/java/alluxio/fuse/AlluxioFuse.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,6 @@

package alluxio.fuse;

import static alluxio.fuse.options.FuseOptions.FUSE_UPDATE_CHECK_ENABLED;

import alluxio.Constants;
import alluxio.ProjectConstants;
import alluxio.RuntimeConstants;
Expand Down Expand Up @@ -97,10 +95,15 @@ public void start(AlluxioConfiguration conf) throws ParseException {
}
startJvmMonitorProcess();
ExecutorService executor = null;
if (fuseOptions.updateCheckEnabled()) {
// updateCheck is false only if configurable and not enabled
boolean updateCheck = true;
if (Boolean.parseBoolean(ProjectConstants.UPDATE_CHECK_CONFIGURABLE)) {
updateCheck = Configuration.getBoolean(PropertyKey.FUSE_UPDATE_CHECK_ENABLED);
}
if (updateCheck && !Configuration.getBoolean(PropertyKey.TEST_MODE)) {
executor = Executors.newSingleThreadExecutor();
executor.submit(new HeartbeatThread(HeartbeatContext.FUSE_UPDATE_CHECK,
FuseUpdateChecker.create(fuseOptions), () -> new FixedIntervalSupplier(Constants.DAY_MS),
new FuseUpdateChecker(fuseOptions), () -> new FixedIntervalSupplier(Constants.DAY_MS),
Configuration.global(), UserState.Factory.create(conf)));
}
try (FileSystem fs = createBaseFileSystem(fsContext, fuseOptions)) {
Expand Down Expand Up @@ -242,10 +245,6 @@ protected static InstancedConfiguration parseCliOptionsAsConfig(FuseCliOptions c
InstancedConfiguration conf = new InstancedConfiguration(new AlluxioProperties());
cli.getMountPoint()
.ifPresent(mp -> conf.set(PropertyKey.FUSE_MOUNT_POINT, mp, Source.RUNTIME));
cli.getUpdateCheck()
.ifPresent(updateCheckEnabled -> {
conf.set(FUSE_UPDATE_CHECK_ENABLED, updateCheckEnabled, Source.RUNTIME);
});
cli.getRootUfsUri()
.ifPresent(ufsRootUri -> {
conf.set(FuseOptions.FUSE_UFS_ROOT, ufsRootUri.toString(), Source.RUNTIME);
Expand Down
99 changes: 24 additions & 75 deletions dora/integration/fuse/src/main/java/alluxio/fuse/meta/FuseUpdateChecker.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,23 +12,18 @@
package alluxio.fuse.meta;

import alluxio.ProjectConstants;
import alluxio.util.UpdateCheckUtils;
import alluxio.fuse.FuseConstants;
import alluxio.fuse.options.FuseOptions;
import alluxio.heartbeat.HeartbeatExecutor;
import alluxio.metrics.MetricsSystem;
import alluxio.util.URIUtils;
import alluxio.util.UpdateCheckUtils;

import com.google.common.annotations.VisibleForTesting;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.UUID;
import java.util.stream.Collectors;
import javax.annotation.concurrent.NotThreadSafe;

/**
Expand All @@ -37,42 +32,32 @@
@NotThreadSafe
public final class FuseUpdateChecker implements HeartbeatExecutor {
private static final Logger LOG = LoggerFactory.getLogger(FuseUpdateChecker.class);
static final String UNDERLYING_FS_FORMAT = "UnderlyingFileSystem:%s";
static final String LOCAL_ALLUXIO_DATA_CACHE = "localAlluxioDataCache";
static final String LOCAL_ALLUXIO_METADATA_CACHE = "localAlluxioMetadataCache";
static final String LOCAL_KERNEL_DATA_CACHE = "localKernelDataCache";

static final String ALLUXIO_FS = "alluxio";
static final String LOCAL_FS = "local";

static final String TYPE_FUSE = "fuse";
private final String mInstanceId = UUID.randomUUID().toString();

private final Map<String, Long> mFuseOpsCounter;
private final List<String> mUnchangeableFuseInfo;
private final List<String> mFuseInfo = new ArrayList<>();

/**
* Creates a {@link FuseUpdateChecker}.
*
* @param fuseOptions the fuse options
* @return the update checker
*/
public static FuseUpdateChecker create(FuseOptions fuseOptions) {
List<String> fuseInfo = new ArrayList<>();
UpdateCheckUtils.addIfTrue(isLocalAlluxioDataCacheEnabled(fuseOptions),
fuseInfo, LOCAL_ALLUXIO_DATA_CACHE);
UpdateCheckUtils.addIfTrue(isLocalAlluxioMetadataCacheEnabled(fuseOptions), fuseInfo,
LOCAL_ALLUXIO_METADATA_CACHE);
UpdateCheckUtils.addIfTrue(isLocalKernelDataCacheEnabled(fuseOptions),
fuseInfo, LOCAL_KERNEL_DATA_CACHE);
fuseInfo.add(String.format("UnderlyingFileSystem:%s", getUnderlyingFileSystem(fuseOptions)));
return new FuseUpdateChecker(Collections.unmodifiableList(fuseInfo),
FuseConstants.getFuseMethodNames().stream()
.collect(Collectors.toMap(methodName -> methodName, methodName -> 0L)));
}

private FuseUpdateChecker(List<String> unchangeableFuseInfo, Map<String, Long> fuseOpsCounter) {
mUnchangeableFuseInfo = unchangeableFuseInfo;
mFuseOpsCounter = fuseOpsCounter;
public FuseUpdateChecker(FuseOptions fuseOptions) {
if (fuseOptions.getFileSystemOptions().isDataCacheEnabled()) {
mFuseInfo.add(LOCAL_ALLUXIO_DATA_CACHE);
}
if (fuseOptions.getFileSystemOptions().isMetadataCacheEnabled()) {
mFuseInfo.add(LOCAL_ALLUXIO_METADATA_CACHE);
}
if (!fuseOptions.getFuseMountOptions().contains("direct_io")) {
mFuseInfo.add(LOCAL_KERNEL_DATA_CACHE);
}
mFuseInfo.add(String.format("UnderlyingFileSystem:%s", getUnderlyingFileSystem(fuseOptions)));
mFuseInfo.add(TYPE_FUSE);
}

/**
Expand All @@ -82,7 +67,7 @@ private FuseUpdateChecker(List<String> unchangeableFuseInfo, Map<String, Long> f
public void heartbeat(long timeLimitMs) {
try {
String latestVersion =
UpdateCheckUtils.getLatestVersion(mInstanceId, getFuseCheckInfo());
UpdateCheckUtils.getLatestVersion(mInstanceId, mFuseInfo);
if (!ProjectConstants.VERSION.equals(latestVersion)) {
LOG.info("The latest version (" + latestVersion + ") is not the same "
+ "as the current version (" + ProjectConstants.VERSION + "). To upgrade "
Expand All @@ -96,50 +81,6 @@ public void heartbeat(long timeLimitMs) {
@Override
public void close() {}

@VisibleForTesting
List<String> getFuseCheckInfo() {
List<String> info = new ArrayList<>(mUnchangeableFuseInfo);
for (String fuseOps : mFuseOpsCounter.keySet()) {
mFuseOpsCounter.computeIfPresent(fuseOps, (key, value) -> {
long newCount = MetricsSystem.timer(key).getCount();
if (newCount > value) {
info.add(fuseOps);
}
return newCount;
});
}
return info;
}

/**
* @return
*/
@VisibleForTesting
List<String> getUnchangeableFuseInfo() {
return mUnchangeableFuseInfo;
}

/**
* @return true, if local Alluxio data cache is enabled
*/
private static boolean isLocalAlluxioDataCacheEnabled(FuseOptions fuseOptions) {
return fuseOptions.getFileSystemOptions().isDataCacheEnabled();
}

/**
* @return true, if local Alluxio metadata cache is enabled
*/
private static boolean isLocalAlluxioMetadataCacheEnabled(FuseOptions fuseOptions) {
return fuseOptions.getFileSystemOptions().isMetadataCacheEnabled();
}

/**
* @return true, if local kernel data cache is enabled
*/
private static boolean isLocalKernelDataCacheEnabled(FuseOptions fuseOptions) {
return !fuseOptions.getFuseMountOptions().contains("direct_io");
}

/**
* @return the underlying file system type
*/
Expand All @@ -158,4 +99,12 @@ private static String getUnderlyingFileSystem(FuseOptions fuseOptions) {
}
return components[0];
}

/**
* @return
*/
@VisibleForTesting
List<String> getFuseInfo() {
return mFuseInfo;
}
}
24 changes: 2 additions & 22 deletions dora/integration/fuse/src/main/java/alluxio/fuse/options/FuseCliOptions.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,18 +57,6 @@ public class FuseCliOptions {
@Nullable
protected MountCliOptions mMountCliOptions = new MountCliOptions();

@Parameter(
names = {"--update-check"},
description = "Enables or disables the FUSE version update check. "
+ "Disabled by default when connecting to Alluxio system cache or Dora cache. "
+ "Enabled by default when connecting an under storage directly.",
arity = 0,
required = false,
hidden = true
)
@Nullable
protected Boolean mUpdateCheck = null;

@Parameter(
names = {"-h", "--help"},
description = "Display this help message",
Expand Down Expand Up @@ -112,13 +100,6 @@ public Optional<AlluxioURI> getRootUfsUri() {
return Optional.ofNullable(mRootUfsUri);
}

/**
* @return if update check is enabled
*/
public Optional<Boolean> getUpdateCheck() {
return Optional.ofNullable(mUpdateCheck);
}

/**
* @return if user specified {@code --help}
*/
Expand Down Expand Up @@ -156,12 +137,11 @@ public boolean equals(Object o) {
return mHelp == that.mHelp
&& Objects.equals(mMountPoint, that.mMountPoint)
&& Objects.equals(mRootUfsUri, that.mRootUfsUri)
&& Objects.equals(mMountCliOptions, that.mMountCliOptions)
&& Objects.equals(mUpdateCheck, that.mUpdateCheck);
&& Objects.equals(mMountCliOptions, that.mMountCliOptions);
}

@Override
public int hashCode() {
return Objects.hash(mMountPoint, mRootUfsUri, mMountCliOptions, mUpdateCheck, mHelp);
return Objects.hash(mMountPoint, mRootUfsUri, mMountCliOptions, mHelp);
}
}
Loading

0 comments on commit e2371ae

Please sign in to comment.