From 6cd7d3285906c54ca2974bc5ed22b4e113adf88b Mon Sep 17 00:00:00 2001
From: James Bornholt <bornholt@amazon.com>
Date: Wed, 7 Feb 2024 11:56:32 -0600
Subject: [PATCH] Add a `mock-mount-s3` binary that uses the mock client
 backend (#730)

This lets us test the entire mount-s3 binary but without a real S3
backend attached, so we can see the entire data path locally.

Because we're building a mount-s3-like binary, we should think about the
risk of confusing it with the real binary during our release process or
elsewhere. There are two safety measures in this commit that protect
against that confusion:

1. At compile time, the `mountpoint-s3-client/mock` feature must be
   enabled for this binary to compile.
2. At run time, this binary only accepts bucket names that begin with
   `sthree-`, which is documented as an invalid prefix for real S3
   bucket names.

Signed-off-by: James Bornholt <bornholt@amazon.com>
---
 mountpoint-s3/Cargo.toml               |  5 ++
 mountpoint-s3/src/bin/mock-mount-s3.rs | 77 ++++++++++++++++++++++++++
 mountpoint-s3/src/cli.rs               |  2 +-
 3 files changed, 83 insertions(+), 1 deletion(-)
 create mode 100644 mountpoint-s3/src/bin/mock-mount-s3.rs

diff --git a/mountpoint-s3/Cargo.toml b/mountpoint-s3/Cargo.toml
index 14a02f21b..7a0950276 100644
--- a/mountpoint-s3/Cargo.toml
+++ b/mountpoint-s3/Cargo.toml
@@ -87,3 +87,8 @@ shuttle = []
 [[bin]]
 name = "mount-s3"
 path = "src/main.rs"
+
+[[bin]]
+name = "mock-mount-s3"
+path = "src/bin/mock-mount-s3.rs"
+required-features = ["mountpoint-s3-client/mock"]
\ No newline at end of file
diff --git a/mountpoint-s3/src/bin/mock-mount-s3.rs b/mountpoint-s3/src/bin/mock-mount-s3.rs
new file mode 100644
index 000000000..479ccbebf
--- /dev/null
+++ b/mountpoint-s3/src/bin/mock-mount-s3.rs
@@ -0,0 +1,77 @@
+//! A version of `mount-s3` that targets an in-memory mock S3 backend rather than the real service.
+//!
+//! The mock S3 backend supports simulating a target network throughput. The
+//! --maximum-throughput-gbps command-line argument can be used to set the target throughput, which
+//! defaults to 10Gbps.
+//!
+//! As a safety measure, this binary works only if the bucket name begins with "sthree-". This makes
+//! sure we can't accidentally confuse this binary with a real `mount-s3` in any of our testing or
+//! release workflows, since real bucket names cannot start with this prefix.
+//!
+//! This binary is intended only for use in testing and development of Mountpoint.
+
+use futures::executor::ThreadPool;
+use mountpoint_s3::cli::{CliArgs, S3PersonalityArg};
+use mountpoint_s3::fs::S3Personality;
+use mountpoint_s3_client::mock_client::throughput_client::ThroughputMockClient;
+use mountpoint_s3_client::mock_client::{MockClientConfig, MockObject};
+use mountpoint_s3_client::types::ETag;
+
+fn main() -> anyhow::Result<()> {
+    mountpoint_s3::cli::main(create_mock_client)
+}
+
+fn create_mock_client(args: &CliArgs) -> anyhow::Result<(ThroughputMockClient, ThreadPool, S3Personality)> {
+    // An extra little safety thing to make sure we can distinguish the real mount-s3 binary and
+    // this one. Buckets starting with "sthree-" are always invalid against real S3:
+    // https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html
+    anyhow::ensure!(
+        args.bucket_name.starts_with("sthree-"),
+        "mock-mount-s3 bucket names must start with `sthree-`"
+    );
+
+    tracing::warn!("using mock client");
+
+    let max_throughput_gbps = args.maximum_throughput_gbps.unwrap_or(10) as f64;
+    tracing::info!("mock client target network throughput {max_throughput_gbps} Gbps");
+
+    let config = MockClientConfig {
+        bucket: args.bucket_name.clone(),
+        part_size: args.part_size as usize,
+        unordered_list_seed: None,
+    };
+    let client = ThroughputMockClient::new(config, max_throughput_gbps);
+
+    let runtime = ThreadPool::builder().name_prefix("runtime").create()?;
+
+    let s3_personality = if let Some(S3PersonalityArg(personality)) = args.bucket_type {
+        personality
+    } else {
+        S3Personality::Standard
+    };
+
+    // Pre-populate the bucket with some interesting file sizes and a little structure
+    for expt in 0..10 {
+        let size = 2000 * 10u64.pow(expt);
+        let key = if size > 10u64.pow(12) {
+            format!("test-{}TB", size / 10u64.pow(12))
+        } else if size > 10u64.pow(9) {
+            format!("test-{}GB", size / 10u64.pow(9))
+        } else if size > 10u64.pow(6) {
+            format!("test-{}MB", size / 10u64.pow(6))
+        } else if size > 10u64.pow(3) {
+            format!("test-{}kB", size / 10u64.pow(3))
+        } else {
+            format!("test-{}B", size)
+        };
+        client.add_object(&key, MockObject::ramp(0x11, size as usize, ETag::for_tests()));
+    }
+    client.add_object("hello.txt", MockObject::from_bytes(b"hello world", ETag::for_tests()));
+    client.add_object("empty", MockObject::from_bytes(b"", ETag::for_tests()));
+    client.add_object(
+        "dir/hello.txt",
+        MockObject::from_bytes(b"hello world", ETag::for_tests()),
+    );
+
+    Ok((client, runtime, s3_personality))
+}
diff --git a/mountpoint-s3/src/cli.rs b/mountpoint-s3/src/cli.rs
index 43bc6969c..dbaee9413 100644
--- a/mountpoint-s3/src/cli.rs
+++ b/mountpoint-s3/src/cli.rs
@@ -271,7 +271,7 @@ pub struct CliArgs {
 }
 
 #[derive(Debug, Clone)]
-pub struct S3PersonalityArg(S3Personality);
+pub struct S3PersonalityArg(pub S3Personality);
 
 impl ValueEnum for S3PersonalityArg {
     fn value_variants<'a>() -> &'a [Self] {