-
Notifications
You must be signed in to change notification settings - Fork 2.9k
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
- Loading branch information
Showing
38 changed files
with
618 additions
and
207 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
54 changes: 54 additions & 0 deletions
54
dora/core/client/fs/src/main/java/alluxio/client/file/AdaptivePrefetchCachePolicy.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,54 @@ | ||
| /* | ||
| * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 | ||
| * (the "License"). You may not use this work except in compliance with the License, which is | ||
| * available at www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, | ||
| * either express or implied, as more fully set forth in the License. | ||
| * | ||
| * See the NOTICE file distributed with this work for information regarding copyright ownership. | ||
| */ | ||
|
|
||
| package alluxio.client.file; | ||
|
|
||
| import alluxio.conf.Configuration; | ||
| import alluxio.conf.PropertyKey; | ||
|
|
||
| /** | ||
| * An improved implementation of the prefetch cache policy that only halves the prefetch size, | ||
| * on cache miss. | ||
| */ | ||
| public class AdaptivePrefetchCachePolicy implements PrefetchCachePolicy { | ||
| private int mPrefetchSize = 0; | ||
| private long mLastCallEndPos = -1; | ||
| private final int mMaxPrefetchSize = | ||
| (int) Configuration.getBytes(PropertyKey.USER_POSITION_READER_STREAMING_PREFETCH_MAX_SIZE); | ||
|
|
||
| @Override | ||
| public void addTrace(long pos, int size) { | ||
| if (pos == mLastCallEndPos) { | ||
| // increase the prefetch size by the size of cumulative, consecutive reads | ||
| mPrefetchSize = Math.min(mMaxPrefetchSize, mPrefetchSize + size); | ||
| } | ||
| mLastCallEndPos = pos + size; | ||
| } | ||
|
|
||
| @Override | ||
| public void onCacheHitRead() { | ||
| // Noop | ||
| } | ||
|
|
||
| @Override | ||
| public void onCacheMissRead() { | ||
| // on prefetch cache miss, there may be a chance that the read position is | ||
| // not consecutive, e.g. the reader seeks to a position far away from the | ||
| // previous position | ||
| // halve the prefetch size to be conservative | ||
| mPrefetchSize /= 2; | ||
| } | ||
|
|
||
| @Override | ||
| public int getPrefetchSize() { | ||
| return mPrefetchSize; | ||
| } | ||
| } |
69 changes: 69 additions & 0 deletions
69
dora/core/client/fs/src/main/java/alluxio/client/file/BasePrefetchCachePolicy.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,69 @@ | ||
| /* | ||
| * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 | ||
| * (the "License"). You may not use this work except in compliance with the License, which is | ||
| * available at www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, | ||
| * either express or implied, as more fully set forth in the License. | ||
| * | ||
| * See the NOTICE file distributed with this work for information regarding copyright ownership. | ||
| */ | ||
|
|
||
| package alluxio.client.file; | ||
|
|
||
| import alluxio.conf.Configuration; | ||
| import alluxio.conf.PropertyKey; | ||
|
|
||
| import com.google.common.collect.EvictingQueue; | ||
|
|
||
| /** | ||
| * A base prefetch cache policy that increases the prefetch window if the read pattern is | ||
| * contiguous and reduce the window down to the read size if it is not. | ||
| */ | ||
| public class BasePrefetchCachePolicy implements PrefetchCachePolicy { | ||
| private int mPrefetchSize = 0; | ||
| private final EvictingQueue<CallTrace> mCallHistory = EvictingQueue.create( | ||
| Configuration.getInt(PropertyKey.USER_POSITION_READER_STREAMING_MULTIPLIER)); | ||
|
|
||
| @Override | ||
| public void addTrace(long pos, int size) { | ||
| mCallHistory.add(new CallTrace(pos, size)); | ||
| int consecutiveReadLength = 0; | ||
| long lastReadEnd = -1; | ||
| for (CallTrace trace : mCallHistory) { | ||
| if (trace.mPosition == lastReadEnd) { | ||
| lastReadEnd += trace.mLength; | ||
| consecutiveReadLength += trace.mLength; | ||
| } else { | ||
| lastReadEnd = trace.mPosition + trace.mLength; | ||
| consecutiveReadLength = trace.mLength; | ||
| } | ||
| } | ||
| mPrefetchSize = consecutiveReadLength; | ||
| } | ||
|
|
||
| @Override | ||
| public void onCacheHitRead() { | ||
| // Noop | ||
| } | ||
|
|
||
| @Override | ||
| public void onCacheMissRead() { | ||
| // Noop | ||
| } | ||
|
|
||
| @Override | ||
| public int getPrefetchSize() { | ||
| return mPrefetchSize; | ||
| } | ||
|
|
||
| private static class CallTrace { | ||
| final long mPosition; | ||
| final int mLength; | ||
|
|
||
| private CallTrace(long pos, int length) { | ||
| mPosition = pos; | ||
| mLength = length; | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
61 changes: 61 additions & 0 deletions
61
dora/core/client/fs/src/main/java/alluxio/client/file/PrefetchCachePolicy.java
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,61 @@ | ||
| /* | ||
| * The Alluxio Open Foundation licenses this work under the Apache License, version 2.0 | ||
| * (the "License"). You may not use this work except in compliance with the License, which is | ||
| * available at www.apache.org/licenses/LICENSE-2.0 | ||
| * | ||
| * This software is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, | ||
| * either express or implied, as more fully set forth in the License. | ||
| * | ||
| * See the NOTICE file distributed with this work for information regarding copyright ownership. | ||
| */ | ||
|
|
||
| package alluxio.client.file; | ||
|
|
||
| import alluxio.conf.Configuration; | ||
| import alluxio.conf.PropertyKey; | ||
|
|
||
| import com.amazonaws.annotation.NotThreadSafe; | ||
|
|
||
| /** | ||
| * The prefetch cache policy to determine the prefetch size. | ||
| */ | ||
| @NotThreadSafe | ||
| public interface PrefetchCachePolicy { | ||
| /** | ||
| * Adds the trace of a read request. | ||
| * @param pos the position | ||
| * @param size the size | ||
| */ | ||
| void addTrace(long pos, int size); | ||
|
|
||
| /** | ||
| * Called when a read hits the cache. | ||
| */ | ||
| void onCacheHitRead(); | ||
|
|
||
| /** | ||
| * Called when a read does not hit the cache. | ||
| */ | ||
| void onCacheMissRead(); | ||
|
|
||
| /** | ||
| * @return the expected prefetch size | ||
| */ | ||
| int getPrefetchSize(); | ||
|
|
||
| /** | ||
| * The factory class. | ||
| */ | ||
| class Factory { | ||
| /** | ||
| * @return a prefetch cache policy | ||
| */ | ||
| public static PrefetchCachePolicy create() { | ||
| if (Configuration.getBoolean( | ||
| PropertyKey.USER_POSITION_READER_STREAMING_ADAPTIVE_POLICY_ENABLED)) { | ||
| return new AdaptivePrefetchCachePolicy(); | ||
| } | ||
| return new BasePrefetchCachePolicy(); | ||
| } | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.