Support running integration test against custom endpoints (#1094)

## Description of change This allows contributors to run mountpoint integration tests against custom S3 compatible endpoints. ## Does this change impact existing behavior? No, test only ## Does this change need a changelog entry in any of the crates? No --- By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license and I agree to the terms of the [Developer Certificate of Origin (DCO)](https://developercertificate.org/). Signed-off-by: Monthon Klongklaew <[email protected]>
awslabs · Nov 5, 2024 · 2a95d14 · 2a95d14
1 parent a54596b
commit 2a95d14
Show file tree

Hide file tree

Showing 13 changed files with 226 additions and 150 deletions.
diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml
@@ -23,6 +23,8 @@ env:
   S3_BUCKET_TEST_PREFIX: ${{ vars.S3_BUCKET_BENCH_PREFIX || 'mountpoint-benchmark/' }}
   S3_BUCKET_BENCH_FILE: ${{ vars.BENCH_FILE_NAME || 'bench100GB.bin' }}
   S3_BUCKET_SMALL_BENCH_FILE: ${{ vars.SMALL_BENCH_FILE_NAME || 'bench5MB.bin' }}
+  # Optional endpoint url, can be empty
+  S3_ENDPOINT_URL: ${{ vars.S3_ENDPOINT_URL }}
   S3_REGION: ${{ vars.S3_BENCH_REGION }}
 
 jobs:

diff --git a/.github/workflows/bench_s3express.yml b/.github/workflows/bench_s3express.yml
@@ -23,6 +23,8 @@ env:
   S3_BUCKET_TEST_PREFIX: ${{ vars.S3_BUCKET_BENCH_PREFIX || 'mountpoint-benchmark/' }}
   S3_BUCKET_BENCH_FILE: ${{ vars.BENCH_FILE_NAME || 'bench100GB.bin' }}
   S3_BUCKET_SMALL_BENCH_FILE: ${{ vars.SMALL_BENCH_FILE_NAME || 'bench5MB.bin' }}
+  # Optional endpoint url, can be empty
+  S3_ENDPOINT_URL: ${{ vars.S3_EXPRESS_ONE_ZONE_ENDPOINT_URL }}
   S3_REGION: ${{ vars.S3_BENCH_REGION }}
 
 jobs:

diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml
@@ -17,6 +17,9 @@ env:
   CARGO_INCREMENTAL: 0
   S3_BUCKET_NAME: ${{ vars.S3_BUCKET_NAME }}
   S3_EXPRESS_ONE_ZONE_BUCKET_NAME: ${{ vars.S3_EXPRESS_ONE_ZONE_BUCKET_NAME }}
+  # Optional endpoint urls, can be empty
+  S3_ENDPOINT_URL: ${{ vars.S3_ENDPOINT_URL }}
+  S3_EXPRESS_ONE_ZONE_ENDPOINT_URL: ${{ vars.S3_EXPRESS_ONE_ZONE_ENDPOINT_URL }}
   S3_REGION: ${{ vars.S3_REGION }}
   S3_BUCKET_TEST_PREFIX: ${{ vars.S3_BUCKET_TEST_PREFIX || 'github-actions-tmp/' }}run-${{ github.run_id }}/attempt-${{ github.run_attempt }}/
   # A bucket our IAM role has no access to, but is in the right region, for permissions tests

diff --git a/mountpoint-s3-client/tests/auth.rs b/mountpoint-s3-client/tests/auth.rs
@@ -13,7 +13,7 @@ use common::creds::get_scoped_down_credentials;
 use common::creds::{get_sdk_default_chain_creds, get_subsession_iam_role};
 use common::*;
 use futures::StreamExt;
-use mountpoint_s3_client::config::{EndpointConfig, S3ClientAuthConfig, S3ClientConfig};
+use mountpoint_s3_client::config::{S3ClientAuthConfig, S3ClientConfig};
 use mountpoint_s3_client::error::ObjectClientError;
 #[cfg(not(feature = "s3express_tests"))]
 use mountpoint_s3_client::S3RequestError;
@@ -52,7 +52,7 @@ async fn test_static_provider() {
     let provider = CredentialsProvider::new_static(&Allocator::default(), config).unwrap();
     let config = S3ClientConfig::new()
         .auth_config(S3ClientAuthConfig::Provider(provider))
-        .endpoint_config(EndpointConfig::new(&get_test_region()));
+        .endpoint_config(get_test_endpoint_config());
     let client = S3CrtClient::new(config).unwrap();
 
     let result = client
@@ -72,7 +72,7 @@ async fn test_static_provider() {
     let provider = CredentialsProvider::new_static(&Allocator::default(), config).unwrap();
     let config = S3ClientConfig::new()
         .auth_config(S3ClientAuthConfig::Provider(provider))
-        .endpoint_config(EndpointConfig::new(&get_test_region()));
+        .endpoint_config(get_test_endpoint_config());
     let client = S3CrtClient::new(config).unwrap();
 
     let mut request = client
@@ -134,7 +134,7 @@ async fn test_profile_provider_static_async() {
     // Build a S3CrtClient that uses the config file
     let config = S3ClientConfig::new()
         .auth_config(S3ClientAuthConfig::Profile(profile_name.to_owned()))
-        .endpoint_config(EndpointConfig::new(&get_test_region()));
+        .endpoint_config(get_test_endpoint_config());
     let client = S3CrtClient::new(config).unwrap();
 
     let result = client
@@ -150,7 +150,7 @@ async fn test_profile_provider_static_async() {
 
     let config = S3ClientConfig::new()
         .auth_config(S3ClientAuthConfig::Profile(profile_name.to_owned()))
-        .endpoint_config(EndpointConfig::new(&get_test_region()));
+        .endpoint_config(get_test_endpoint_config());
     let client = S3CrtClient::new(config).unwrap();
 
     let mut request = client
@@ -169,7 +169,7 @@ async fn test_profile_provider_static_async() {
     // be constructed.
     let config = S3ClientConfig::new()
         .auth_config(S3ClientAuthConfig::Profile("not-the-right-profile-name".to_owned()))
-        .endpoint_config(EndpointConfig::new(&get_test_region()));
+        .endpoint_config(get_test_endpoint_config());
     let _result = S3CrtClient::new(config).expect_err("profile doesn't exist");
 }
 
@@ -215,7 +215,7 @@ async fn test_profile_provider_assume_role_async() {
     // we did not configure which arn to assume yet.
     let config = S3ClientConfig::new()
         .auth_config(S3ClientAuthConfig::Profile(profile_name.to_owned()))
-        .endpoint_config(EndpointConfig::new(&get_test_region()));
+        .endpoint_config(get_test_endpoint_config());
     let client = S3CrtClient::new(config).unwrap();
 
     let mut request = client
@@ -230,7 +230,7 @@ async fn test_profile_provider_assume_role_async() {
     writeln!(config_file, "source_profile = {}", source_profile).unwrap();
     let config = S3ClientConfig::new()
         .auth_config(S3ClientAuthConfig::Profile(profile_name.to_owned()))
-        .endpoint_config(EndpointConfig::new(&get_test_region()));
+        .endpoint_config(get_test_endpoint_config());
     let client = S3CrtClient::new(config).unwrap();
 
     let result = client
@@ -344,7 +344,7 @@ async fn test_credential_process_behind_source_profile_async() {
     // With correct profile, things should be fine
     let config = S3ClientConfig::new()
         .auth_config(S3ClientAuthConfig::Profile(correct_profile.to_owned()))
-        .endpoint_config(EndpointConfig::new(&get_test_region()));
+        .endpoint_config(get_test_endpoint_config());
     let client = S3CrtClient::new(config).unwrap();
     let _result = client
         .list_objects(&bucket, None, "/", 10, &format!("{prefix}foo/"))
@@ -354,7 +354,7 @@ async fn test_credential_process_behind_source_profile_async() {
     // With incorrect profile, requests should fail with a client error
     let config = S3ClientConfig::new()
         .auth_config(S3ClientAuthConfig::Profile(incorrect_profile.to_owned()))
-        .endpoint_config(EndpointConfig::new(&get_test_region()));
+        .endpoint_config(get_test_endpoint_config());
     let client = S3CrtClient::new(config).unwrap();
     let err = client
         .list_objects(&bucket, None, "/", 10, &format!("{prefix}/"))
@@ -425,7 +425,7 @@ async fn test_scoped_credentials() {
     let provider = CredentialsProvider::new_static(&Allocator::default(), config).unwrap();
     let config = S3ClientConfig::new()
         .auth_config(S3ClientAuthConfig::Provider(provider))
-        .endpoint_config(EndpointConfig::new(&get_test_region()));
+        .endpoint_config(get_test_endpoint_config());
     let client = S3CrtClient::new(config).unwrap();
 
     // Inside the prefix, things should be fine

diff --git a/mountpoint-s3-client/tests/common/mod.rs b/mountpoint-s3-client/tests/common/mod.rs
@@ -12,7 +12,9 @@ use futures::{pin_mut, Stream, StreamExt};
 use mountpoint_s3_client::config::{EndpointConfig, S3ClientConfig};
 use mountpoint_s3_client::types::GetObjectRequest;
 use mountpoint_s3_client::S3CrtClient;
+use mountpoint_s3_crt::common::allocator::Allocator;
 use mountpoint_s3_crt::common::rust_log_adapter::RustLogAdapter;
+use mountpoint_s3_crt::common::uri::Uri;
 use rand::rngs::OsRng;
 use rand::RngCore;
 use std::ops::Range;
@@ -70,14 +72,13 @@ pub fn get_test_kms_key_id() -> String {
 }
 
 pub fn get_test_client() -> S3CrtClient {
-    let endpoint_config = EndpointConfig::new(&get_test_region());
-    S3CrtClient::new(S3ClientConfig::new().endpoint_config(endpoint_config)).expect("could not create test client")
+    S3CrtClient::new(S3ClientConfig::new().endpoint_config(get_test_endpoint_config()))
+        .expect("could not create test client")
 }
 
 pub fn get_test_backpressure_client(initial_read_window: usize, part_size: Option<usize>) -> S3CrtClient {
-    let endpoint_config = EndpointConfig::new(&get_test_region());
     let mut config = S3ClientConfig::new()
-        .endpoint_config(endpoint_config)
+        .endpoint_config(get_test_endpoint_config())
         .read_backpressure(true)
         .initial_read_window(initial_read_window);
     if let Some(part_size) = part_size {
@@ -123,6 +124,26 @@ pub fn get_test_region() -> String {
     std::env::var("S3_REGION").expect("Set S3_REGION to run integration tests")
 }
 
+/// Optional config for testing against a custom endpoint url
+fn get_test_endpoint_url() -> Option<String> {
+    if cfg!(feature = "s3express_tests") {
+        std::env::var("S3_EXPRESS_ONE_ZONE_ENDPOINT_URL")
+            .ok()
+            .filter(|str| !str.is_empty())
+    } else {
+        std::env::var("S3_ENDPOINT_URL").ok().filter(|str| !str.is_empty())
+    }
+}
+
+pub fn get_test_endpoint_config() -> EndpointConfig {
+    let mut endpoint_config = EndpointConfig::new(&get_test_region());
+    if let Some(endpoint_url) = get_test_endpoint_url() {
+        let endpoint = Uri::new_from_str(&Allocator::default(), endpoint_url.clone()).expect("invalid endpoint url");
+        endpoint_config = endpoint_config.endpoint(endpoint);
+    }
+    endpoint_config
+}
+
 pub fn get_test_bucket_without_permissions() -> String {
     std::env::var("S3_FORBIDDEN_BUCKET_NAME").expect("Set S3_FORBIDDEN_BUCKET_NAME to run integration tests")
 }
@@ -136,11 +157,11 @@ pub fn get_test_domain() -> String {
 }
 
 pub async fn get_test_sdk_client() -> s3::Client {
-    let sdk_config = aws_config::defaults(BehaviorVersion::latest())
-        .region(Region::new(get_test_region()))
-        .load()
-        .await;
-    s3::Client::new(&sdk_config)
+    let mut sdk_config = aws_config::defaults(BehaviorVersion::latest()).region(Region::new(get_test_region()));
+    if let Some(endpoint_url) = get_test_endpoint_url() {
+        sdk_config = sdk_config.endpoint_url(endpoint_url);
+    }
+    s3::Client::new(&sdk_config.load().await)
 }
 
 /// Create some objects in a prefix for testing.

diff --git a/mountpoint-s3-client/tests/endpoint_config.rs b/mountpoint-s3-client/tests/endpoint_config.rs
@@ -9,7 +9,7 @@ use mountpoint_s3_client::config::{AddressingStyle, EndpointConfig, S3ClientConf
 use mountpoint_s3_client::{ObjectClient, S3CrtClient};
 use test_case::test_case;
 
-async fn run_test<F: FnOnce(&str) -> EndpointConfig>(f: F, prefix: &str, bucket: String) {
+async fn run_test(endpoint_config: EndpointConfig, prefix: &str, bucket: String) {
     let sdk_client = get_test_sdk_client().await;
 
     // Create one object named "hello"
@@ -24,8 +24,6 @@ async fn run_test<F: FnOnce(&str) -> EndpointConfig>(f: F, prefix: &str, bucket:
         .await
         .unwrap();
 
-    let region = get_test_region();
-    let endpoint_config = f(&region);
     let config = S3ClientConfig::new().endpoint_config(endpoint_config.clone());
     let client = S3CrtClient::new(config).expect("could not create test client");
 
@@ -41,7 +39,7 @@ async fn run_test<F: FnOnce(&str) -> EndpointConfig>(f: F, prefix: &str, bucket:
 #[tokio::test]
 async fn test_addressing_style(addressing_style: AddressingStyle, prefix: &str) {
     run_test(
-        |region| EndpointConfig::new(region).addressing_style(addressing_style),
+        get_test_endpoint_config().addressing_style(addressing_style),
         &get_unique_test_prefix(prefix),
         get_test_bucket(),
     )
@@ -54,12 +52,7 @@ async fn test_addressing_style(addressing_style: AddressingStyle, prefix: &str)
 #[tokio::test]
 async fn test_use_fips() {
     let prefix = get_unique_test_prefix("test_fips");
-    run_test(
-        |region| EndpointConfig::new(region).use_fips(true),
-        &prefix,
-        get_test_bucket(),
-    )
-    .await;
+    run_test(get_test_endpoint_config().use_fips(true), &prefix, get_test_bucket()).await;
 }
 
 // S3 Express One Zone does not support S3 Transfer Acceleration
@@ -69,7 +62,7 @@ async fn test_use_fips() {
 async fn test_use_accelerate() {
     let prefix = get_unique_test_prefix("test_transfer_acceleration");
     run_test(
-        |region| EndpointConfig::new(region).use_accelerate(true),
+        get_test_endpoint_config().use_accelerate(true),
         &prefix,
         get_test_bucket(),
     )
@@ -84,11 +77,9 @@ async fn test_use_accelerate() {
 async fn test_addressing_style_dualstack_option(addressing_style: AddressingStyle, prefix: &str) {
     let prefix = get_unique_test_prefix(prefix);
     run_test(
-        |region| {
-            EndpointConfig::new(region)
-                .addressing_style(addressing_style)
-                .use_dual_stack(true)
-        },
+        get_test_endpoint_config()
+            .addressing_style(addressing_style)
+            .use_dual_stack(true),
         &prefix,
         get_test_bucket(),
     )
@@ -102,7 +93,7 @@ async fn test_addressing_style_dualstack_option(addressing_style: AddressingStyl
 async fn test_fips_dual_stack_mount_option() {
     let prefix = get_unique_test_prefix("test_fips_dual_stack");
     run_test(
-        |region| EndpointConfig::new(region).use_fips(true).use_dual_stack(true),
+        get_test_endpoint_config().use_fips(true).use_dual_stack(true),
         &prefix,
         get_test_bucket(),
     )
@@ -119,7 +110,7 @@ async fn test_fips_dual_stack_mount_option() {
 #[tokio::test]
 async fn test_single_region_access_point(addressing_style: AddressingStyle, arn: bool, prefix: &str) {
     run_test(
-        |region| EndpointConfig::new(region).addressing_style(addressing_style),
+        get_test_endpoint_config().addressing_style(addressing_style),
         &get_unique_test_prefix(prefix),
         get_test_access_point(arn, AccessPointType::SingleRegion),
     )
@@ -129,9 +120,7 @@ async fn test_single_region_access_point(addressing_style: AddressingStyle, arn:
 #[cfg(not(feature = "s3express_tests"))]
 // For Object Lambda Access Point, PutObject is not supported,
 // For multi region access points, Rust SDK is not supported. Hence different helper method for these tests.
-async fn run_list_objects_test<F: FnOnce(&str) -> EndpointConfig>(f: F, prefix: &str, bucket: &str) {
-    let region = get_test_region();
-    let endpoint_config = f(&region);
+async fn run_list_objects_test(endpoint_config: EndpointConfig, prefix: &str, bucket: &str) {
     let config = S3ClientConfig::new().endpoint_config(endpoint_config.clone());
     let client = S3CrtClient::new(config).expect("could not create test client");
 
@@ -149,7 +138,7 @@ async fn run_list_objects_test<F: FnOnce(&str) -> EndpointConfig>(f: F, prefix:
 #[tokio::test]
 async fn test_object_lambda_access_point(arn: bool, prefix: &str) {
     run_list_objects_test(
-        EndpointConfig::new,
+        get_test_endpoint_config(),
         &get_unique_test_prefix(prefix),
         &get_test_access_point(arn, AccessPointType::ObjectLambda),
     )
@@ -164,7 +153,7 @@ async fn test_object_lambda_access_point(arn: bool, prefix: &str) {
 async fn test_multi_region_access_point() {
     let prefix = "test_MRAP";
     run_list_objects_test(
-        EndpointConfig::new,
+        get_test_endpoint_config(),
         &get_unique_test_prefix(prefix),
         &get_test_access_point(true, AccessPointType::MultiRegion),
     )

diff --git a/mountpoint-s3-client/tests/network_interface_config.rs b/mountpoint-s3-client/tests/network_interface_config.rs
@@ -5,7 +5,7 @@ pub mod common;
 use test_case::test_case;
 
 use common::*;
-use mountpoint_s3_client::config::{EndpointConfig, S3ClientConfig};
+use mountpoint_s3_client::config::S3ClientConfig;
 use mountpoint_s3_client::error::{HeadObjectError, ObjectClientError::ServiceError};
 use mountpoint_s3_client::types::HeadObjectParams;
 use mountpoint_s3_client::{ObjectClient, S3CrtClient};
@@ -17,7 +17,7 @@ async fn test_empty_list() {
 
     let interface_names = Vec::new();
     let config = S3ClientConfig::new()
-        .endpoint_config(EndpointConfig::new(&get_test_region()))
+        .endpoint_config(get_test_endpoint_config())
         .network_interface_names(interface_names);
     let client = S3CrtClient::new(config).expect("client should create OK");
 
@@ -39,7 +39,7 @@ async fn test_one_interface_ok() {
     let primary_interface = get_primary_interface_name();
     let interface_names = vec![primary_interface];
     let config = S3ClientConfig::new()
-        .endpoint_config(EndpointConfig::new(&get_test_region()))
+        .endpoint_config(get_test_endpoint_config())
         .network_interface_names(interface_names);
     let client = S3CrtClient::new(config).expect("client should create OK");
 
@@ -66,7 +66,7 @@ async fn test_nonexistent(with_valid_interface: bool) {
         vec![non_existent_interface]
     };
     let config = S3ClientConfig::new()
-        .endpoint_config(EndpointConfig::new(&get_test_region()))
+        .endpoint_config(get_test_endpoint_config())
         .network_interface_names(interface_names);
     S3CrtClient::new(config).expect_err(
         "CRT should return an error during client creation if provided with non-existent network interface",