From 21ce86b6e119a4938ff35723773686d7668e94f5 Mon Sep 17 00:00:00 2001
From: Ankit Saurabh <sauraank@amazon.co.uk>
Date: Fri, 6 Oct 2023 04:01:53 +0100
Subject: [PATCH 1/8] Improved the instructions to add benchmark files manually
in doc/BENCHMARKING.md
Signed-off-by: Ankit Saurabh <sauraank@amazon.co.uk>
---
doc/BENCHMARKING.md | 13 +++++++++----
1 file changed, 9 insertions(+), 4 deletions(-)
diff --git a/doc/BENCHMARKING.md b/doc/BENCHMARKING.md
index ae63ff1f4..70a99cfd9 100644
--- a/doc/BENCHMARKING.md
+++ b/doc/BENCHMARKING.md
@@ -26,7 +26,7 @@ We keep the records of benchmarking results in `gh-pages` branch and the perform
While our benchmark script is written for CI testing only, it is possible to run manually.
You can use the following steps.
-1. Install dependencies and configure FUSE by running the following script in the repository:
+1. Install dependencies and configure FUSE by running the following script in the `mountpoint-s3` repository:
bash .github/actions/install-dependencies/install.sh \
--fuse-version 2 \
@@ -39,10 +39,15 @@ You can use the following steps.
export S3_BUCKET_BENCH_FILE=bench_file_name
export S3_BUCKET_SMALL_BENCH_FILE=small_bench_file_name
-3. Create the bench files manually in your bucket. The size of the files must be exactly the same as the size defined in fio configuration files. The easiest way to do this is running fio against your local file system first to let fio create the files for you, and then upload them to your S3 bucket using the AWS CLI. For example:
+3. Create the bench files manually in your bucket. The size of the files must be exactly the same as the size defined in fio configuration files. The easiest way to do this is mounting your bucket on local file system using mountpoint-s3.Then, running fio against your mount directory to let fio create the files for you. For example:
+* To create small benchmark file for read workload use:
- fio --directory=your_local_dir --filename=your_file_name mountpoint-s3/scripts/fio/read/seq_read_small.fio
- aws s3 cp your_local_dir/your_file_name s3://${S3_BUCKET_NAME}/${S3_BUCKET_TEST_PREFIX}
+ fio --directory=your_mount_dir --filename=your_file_name mountpoint-s3/scripts/fio/read/seq_read_small.fio
+
+* To create directory with 10000 files for readdir workload use:
+
+ mkdir bench_dir_10000
+ fio --directory=your_mount_dir/bench_dir_10000 --filename=your_file_name mountpoint-s3/scripts/fio/create/create_files_10000.fio
4. Run the benchmark script for [throughput](../mountpoint-s3/scripts/fs_bench.sh) or [latency](../mountpoint-s3/scripts/fs_latency_bench.sh).
From d1183b514aa8d4e3e7937c307efe943175e4ea86 Mon Sep 17 00:00:00 2001
From: Ankit Saurabh <sauraank@amazon.co.uk>
Date: Fri, 6 Oct 2023 13:29:03 +0100
Subject: [PATCH 2/8] Added all the steps for creating benchmark files
Signed-off-by: Ankit Saurabh <sauraank@amazon.co.uk>
---
doc/BENCHMARKING.md | 18 ++++++++++--------
1 file changed, 10 insertions(+), 8 deletions(-)
diff --git a/doc/BENCHMARKING.md b/doc/BENCHMARKING.md
index 70a99cfd9..f52688c62 100644
--- a/doc/BENCHMARKING.md
+++ b/doc/BENCHMARKING.md
@@ -39,15 +39,17 @@ You can use the following steps.
export S3_BUCKET_BENCH_FILE=bench_file_name
export S3_BUCKET_SMALL_BENCH_FILE=small_bench_file_name
-3. Create the bench files manually in your bucket. The size of the files must be exactly the same as the size defined in fio configuration files. The easiest way to do this is mounting your bucket on local file system using mountpoint-s3.Then, running fio against your mount directory to let fio create the files for you. For example:
-* To create small benchmark file for read workload use:
-
- fio --directory=your_mount_dir --filename=your_file_name mountpoint-s3/scripts/fio/read/seq_read_small.fio
-
-* To create directory with 10000 files for readdir workload use:
-
- mkdir bench_dir_10000
+3. Create the bench files manually in your bucket(create a `benchmark` directory in it if not present). The size of the files must be exactly the same as the size defined in fio configuration files. The easiest way to do this is mounting your bucket on local file system using mountpoint-s3. Then, running fio against your mount directory to let fio create the files for you.
+
+ mount-s3 your-bucket-name your_mount_dir/ --allow-delete --prefix benchmark/
+ cd your_mount_dir/
+ fio --directory=your_mount_dir --filename=bench5MB.bin mountpoint-s3/scripts/fio/read/seq_read_small.fio
+ fio --directory=your_mount_dir --filename=bench100GB.bin mountpoint-s3/scripts/fio/read/seq_read_small.fio
+ mkdir bench_dir_100/ bench_dir_1000/ bench_dir_10000/ bench_dir_100000/
+ fio --directory=your_mount_dir/bench_dir_100 --filename=your_file_name mountpoint-s3/scripts/fio/create/create_files_100.fio
+ fio --directory=your_mount_dir/bench_dir_1000 --filename=your_file_name mountpoint-s3/scripts/fio/create/create_files_1000.fio
fio --directory=your_mount_dir/bench_dir_10000 --filename=your_file_name mountpoint-s3/scripts/fio/create/create_files_10000.fio
+ fio --directory=your_mount_dir/bench_dir_100000 --filename=your_file_name mountpoint-s3/scripts/fio/create/create_files_100000.fio
4. Run the benchmark script for [throughput](../mountpoint-s3/scripts/fs_bench.sh) or [latency](../mountpoint-s3/scripts/fs_latency_bench.sh).
From cc0c27bfe1b1ce8ce548090a90a63637e173696d Mon Sep 17 00:00:00 2001
From: Ankit Saurabh <sauraank@amazon.co.uk>
Date: Fri, 6 Oct 2023 13:30:40 +0100
Subject: [PATCH 3/8] Corrected job name
Signed-off-by: Ankit Saurabh <sauraank@amazon.co.uk>
---
doc/BENCHMARKING.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/doc/BENCHMARKING.md b/doc/BENCHMARKING.md
index f52688c62..2b7acd29b 100644
--- a/doc/BENCHMARKING.md
+++ b/doc/BENCHMARKING.md
@@ -44,7 +44,7 @@ You can use the following steps.
mount-s3 your-bucket-name your_mount_dir/ --allow-delete --prefix benchmark/
cd your_mount_dir/
fio --directory=your_mount_dir --filename=bench5MB.bin mountpoint-s3/scripts/fio/read/seq_read_small.fio
- fio --directory=your_mount_dir --filename=bench100GB.bin mountpoint-s3/scripts/fio/read/seq_read_small.fio
+ fio --directory=your_mount_dir --filename=bench100GB.bin mountpoint-s3/scripts/fio/read/seq_read.fio
mkdir bench_dir_100/ bench_dir_1000/ bench_dir_10000/ bench_dir_100000/
fio --directory=your_mount_dir/bench_dir_100 --filename=your_file_name mountpoint-s3/scripts/fio/create/create_files_100.fio
fio --directory=your_mount_dir/bench_dir_1000 --filename=your_file_name mountpoint-s3/scripts/fio/create/create_files_1000.fio
From 415fd3442c02ebca655738fad62d1c6d40adc8bc Mon Sep 17 00:00:00 2001
From: Ankit Saurabh <sauraank@amazon.co.uk>
Date: Fri, 6 Oct 2023 13:34:32 +0100
Subject: [PATCH 4/8] Removed filename argument to create directory
Signed-off-by: Ankit Saurabh <sauraank@amazon.co.uk>
---
doc/BENCHMARKING.md | 8 ++++----
1 file changed, 4 insertions(+), 4 deletions(-)
diff --git a/doc/BENCHMARKING.md b/doc/BENCHMARKING.md
index 2b7acd29b..59bd8ab34 100644
--- a/doc/BENCHMARKING.md
+++ b/doc/BENCHMARKING.md
@@ -46,10 +46,10 @@ You can use the following steps.
fio --directory=your_mount_dir --filename=bench5MB.bin mountpoint-s3/scripts/fio/read/seq_read_small.fio
fio --directory=your_mount_dir --filename=bench100GB.bin mountpoint-s3/scripts/fio/read/seq_read.fio
mkdir bench_dir_100/ bench_dir_1000/ bench_dir_10000/ bench_dir_100000/
- fio --directory=your_mount_dir/bench_dir_100 --filename=your_file_name mountpoint-s3/scripts/fio/create/create_files_100.fio
- fio --directory=your_mount_dir/bench_dir_1000 --filename=your_file_name mountpoint-s3/scripts/fio/create/create_files_1000.fio
- fio --directory=your_mount_dir/bench_dir_10000 --filename=your_file_name mountpoint-s3/scripts/fio/create/create_files_10000.fio
- fio --directory=your_mount_dir/bench_dir_100000 --filename=your_file_name mountpoint-s3/scripts/fio/create/create_files_100000.fio
+ fio --directory=your_mount_dir/bench_dir_100 mountpoint-s3/scripts/fio/create/create_files_100.fio
+ fio --directory=your_mount_dir/bench_dir_1000 mountpoint-s3/scripts/fio/create/create_files_1000.fio
+ fio --directory=your_mount_dir/bench_dir_10000 mountpoint-s3/scripts/fio/create/create_files_10000.fio
+ fio --directory=your_mount_dir/bench_dir_100000 mountpoint-s3/scripts/fio/create/create_files_100000.fio
4. Run the benchmark script for [throughput](../mountpoint-s3/scripts/fs_bench.sh) or [latency](../mountpoint-s3/scripts/fs_latency_bench.sh).
From a60451a37d1912eaa646b9b9aad089235f6e53e0 Mon Sep 17 00:00:00 2001
From: Ankit Saurabh <sauraank@amazon.co.uk>
Date: Mon, 9 Oct 2023 11:48:23 +0100
Subject: [PATCH 5/8] corrected the recommendations
Signed-off-by: Ankit Saurabh <sauraank@amazon.co.uk>
---
doc/BENCHMARKING.md | 22 +++++++++++-----------
1 file changed, 11 insertions(+), 11 deletions(-)
diff --git a/doc/BENCHMARKING.md b/doc/BENCHMARKING.md
index 59bd8ab34..ef90f7bac 100644
--- a/doc/BENCHMARKING.md
+++ b/doc/BENCHMARKING.md
@@ -39,17 +39,17 @@ You can use the following steps.
export S3_BUCKET_BENCH_FILE=bench_file_name
export S3_BUCKET_SMALL_BENCH_FILE=small_bench_file_name
-3. Create the bench files manually in your bucket(create a `benchmark` directory in it if not present). The size of the files must be exactly the same as the size defined in fio configuration files. The easiest way to do this is mounting your bucket on local file system using mountpoint-s3. Then, running fio against your mount directory to let fio create the files for you.
-
- mount-s3 your-bucket-name your_mount_dir/ --allow-delete --prefix benchmark/
- cd your_mount_dir/
- fio --directory=your_mount_dir --filename=bench5MB.bin mountpoint-s3/scripts/fio/read/seq_read_small.fio
- fio --directory=your_mount_dir --filename=bench100GB.bin mountpoint-s3/scripts/fio/read/seq_read.fio
- mkdir bench_dir_100/ bench_dir_1000/ bench_dir_10000/ bench_dir_100000/
- fio --directory=your_mount_dir/bench_dir_100 mountpoint-s3/scripts/fio/create/create_files_100.fio
- fio --directory=your_mount_dir/bench_dir_1000 mountpoint-s3/scripts/fio/create/create_files_1000.fio
- fio --directory=your_mount_dir/bench_dir_10000 mountpoint-s3/scripts/fio/create/create_files_10000.fio
- fio --directory=your_mount_dir/bench_dir_100000 mountpoint-s3/scripts/fio/create/create_files_100000.fio
+3. Create the bench files manually in your S3 bucket. You can do that by mounting the bucket on your machine using Mountpoint. Then, running fio jobs against your mount directory to let fio create the files for you.
+
+ mount-s3 DOC-EXAMPLE-BUCKET path/to/mount/ --prefix benchmark/ --part-size=16777216
+ cd mountpoint-s3/scripts/fio/
+ fio --directory=path/to/mount/ --filename=bench5MB.bin read/seq_read_small.fio --create-only=1
+ fio --directory=path/to/mount/ --filename=bench100GB.bin read/seq_read.fio --create-only=1
+ mkdir path/to/mount/bench_dir_100/ path/to/mount/bench_dir_1000/ path/to/mount/bench_dir_10000/ path/to/mount/bench_dir_100000/
+ fio --directory=path/to/mount/bench_dir_100 create/create_files_100.fio --create-only=1
+ fio --directory=path/to/mount/bench_dir_1000 create/create_files_1000.fio --create-only=1
+ fio --directory=path/to/mount/bench_dir_10000 create/create_files_10000.fio --create-only=1
+ fio --directory=path/to/mount/bench_dir_100000 create/create_files_100000.fio --create-only=1
4. Run the benchmark script for [throughput](../mountpoint-s3/scripts/fs_bench.sh) or [latency](../mountpoint-s3/scripts/fs_latency_bench.sh).
From acac3acf405156d6143facb0187705be16003afc Mon Sep 17 00:00:00 2001
From: Ankit Saurabh <sauraank@amazon.co.uk>
Date: Mon, 9 Oct 2023 18:24:52 +0100
Subject: [PATCH 6/8] Added mount-dir variable
Signed-off-by: Ankit Saurabh <sauraank@amazon.co.uk>
---
doc/BENCHMARKING.md | 15 ++++++++-------
1 file changed, 8 insertions(+), 7 deletions(-)
diff --git a/doc/BENCHMARKING.md b/doc/BENCHMARKING.md
index ef90f7bac..7441671c4 100644
--- a/doc/BENCHMARKING.md
+++ b/doc/BENCHMARKING.md
@@ -43,13 +43,14 @@ You can use the following steps.
mount-s3 DOC-EXAMPLE-BUCKET path/to/mount/ --prefix benchmark/ --part-size=16777216
cd mountpoint-s3/scripts/fio/
- fio --directory=path/to/mount/ --filename=bench5MB.bin read/seq_read_small.fio --create-only=1
- fio --directory=path/to/mount/ --filename=bench100GB.bin read/seq_read.fio --create-only=1
- mkdir path/to/mount/bench_dir_100/ path/to/mount/bench_dir_1000/ path/to/mount/bench_dir_10000/ path/to/mount/bench_dir_100000/
- fio --directory=path/to/mount/bench_dir_100 create/create_files_100.fio --create-only=1
- fio --directory=path/to/mount/bench_dir_1000 create/create_files_1000.fio --create-only=1
- fio --directory=path/to/mount/bench_dir_10000 create/create_files_10000.fio --create-only=1
- fio --directory=path/to/mount/bench_dir_100000 create/create_files_100000.fio --create-only=1
+ MOUNT_DIR = path/to/mount
+ fio --directory=$MOUNT_DIR/ --filename=bench5MB.bin --create_only=1 read/seq_read_small.fio
+ fio --directory=$MOUNT_DIR/ --filename=bench100GB.bin --create_only=1 read/seq_read.fio
+ mkdir $MOUNT_DIR/bench_dir_100/ $MOUNT_DIR/bench_dir_1000/ $MOUNT_DIR/bench_dir_10000/ $MOUNT_DIR/bench_dir_100000/
+ fio --directory=$MOUNT_DIR/bench_dir_100 create/create_files_100.fio
+ fio --directory=$MOUNT_DIR/bench_dir_1000 create/create_files_1000.fio
+ fio --directory=$MOUNT_DIR/bench_dir_10000 create/create_files_10000.fio
+ fio --directory=$MOUNT_DIR/bench_dir_100000 create/create_files_100000.fio
4. Run the benchmark script for [throughput](../mountpoint-s3/scripts/fs_bench.sh) or [latency](../mountpoint-s3/scripts/fs_latency_bench.sh).
From 58faac46be4e60c960107b7fd3a2f6ccafcb4bc1 Mon Sep 17 00:00:00 2001
From: Ankit Saurabh <sauraank@amazon.co.uk>
Date: Mon, 9 Oct 2023 18:29:34 +0100
Subject: [PATCH 7/8] Corrected the ordering
Signed-off-by: Ankit Saurabh <sauraank@amazon.co.uk>
---
doc/BENCHMARKING.md | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/doc/BENCHMARKING.md b/doc/BENCHMARKING.md
index 7441671c4..5777ad7bb 100644
--- a/doc/BENCHMARKING.md
+++ b/doc/BENCHMARKING.md
@@ -41,9 +41,9 @@ You can use the following steps.
3. Create the bench files manually in your S3 bucket. You can do that by mounting the bucket on your machine using Mountpoint. Then, running fio jobs against your mount directory to let fio create the files for you.
- mount-s3 DOC-EXAMPLE-BUCKET path/to/mount/ --prefix benchmark/ --part-size=16777216
- cd mountpoint-s3/scripts/fio/
MOUNT_DIR = path/to/mount
+ mount-s3 DOC-EXAMPLE-BUCKET $MOUNT_DIR/ --prefix benchmark/ --part-size=16777216
+ cd mountpoint-s3/scripts/fio/
fio --directory=$MOUNT_DIR/ --filename=bench5MB.bin --create_only=1 read/seq_read_small.fio
fio --directory=$MOUNT_DIR/ --filename=bench100GB.bin --create_only=1 read/seq_read.fio
mkdir $MOUNT_DIR/bench_dir_100/ $MOUNT_DIR/bench_dir_1000/ $MOUNT_DIR/bench_dir_10000/ $MOUNT_DIR/bench_dir_100000/
From 469ea1cd7b325baa7667269619c08557300a71c7 Mon Sep 17 00:00:00 2001
From: Ankit Saurabh <sauraank@amazon.co.uk>
Date: Tue, 10 Oct 2023 10:52:29 +0100
Subject: [PATCH 8/8] Removed unnecessary space
Signed-off-by: Ankit Saurabh <sauraank@amazon.co.uk>
---
doc/BENCHMARKING.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/doc/BENCHMARKING.md b/doc/BENCHMARKING.md
index 5777ad7bb..8ba0db9b0 100644
--- a/doc/BENCHMARKING.md
+++ b/doc/BENCHMARKING.md
@@ -41,7 +41,7 @@ You can use the following steps.
3. Create the bench files manually in your S3 bucket. You can do that by mounting the bucket on your machine using Mountpoint. Then, running fio jobs against your mount directory to let fio create the files for you.
- MOUNT_DIR = path/to/mount
+ MOUNT_DIR=path/to/mount
mount-s3 DOC-EXAMPLE-BUCKET $MOUNT_DIR/ --prefix benchmark/ --part-size=16777216
cd mountpoint-s3/scripts/fio/
fio --directory=$MOUNT_DIR/ --filename=bench5MB.bin --create_only=1 read/seq_read_small.fio