diff --git a/pom.xml b/pom.xml index b68db37..0ebc495 100644 --- a/pom.xml +++ b/pom.xml @@ -11,7 +11,7 @@ org.janelia.saalfeldlab n5-aws-s3 - 4.0.3-SNAPSHOT + 4.1.0-SNAPSHOT N5 AWS S3 N5 library implementation using Amazon Web Services S3 backend. @@ -120,7 +120,7 @@ 0.2.5 2.2.2 - 3.1.3 + 3.2.0 @@ -132,6 +132,11 @@ com.amazonaws aws-java-sdk-s3 + + com.google.code.findbugs + jsr305 + 3.0.2 + @@ -172,23 +177,18 @@ - - - - org.apache.maven.plugins - maven-surefire-plugin - - - org.janelia.saalfeldlab.n5.s3.backend.N5AmazonS3BucketRootBackendTest - org.janelia.saalfeldlab.n5.s3.backend.N5AmazonS3ContainerPathBackendTest - org.janelia.saalfeldlab.n5.s3.backend.CachedN5AmazonS3BucketRootBackendTest - org.janelia.saalfeldlab.n5.s3.backend.CachedN5AmazonS3ContainerPathBackendTest - org.janelia.saalfeldlab.n5.s3.backend.BackendUriTest - - - - - + + + org.apache.maven.plugins + maven-surefire-plugin + + + org.janelia.saalfeldlab.n5.s3.backend.*.java + org.janelia.saalfeldlab.n5.s3.N5AmazonS3Tests.java + + + + @@ -201,5 +201,19 @@ ${scijava.jvm.version} + + run-backend-tests + + + + org.apache.maven.plugins + maven-surefire-plugin + + + + + + + diff --git a/src/main/java/org/janelia/saalfeldlab/n5/s3/AmazonS3KeyValueAccess.java b/src/main/java/org/janelia/saalfeldlab/n5/s3/AmazonS3KeyValueAccess.java index b3aa067..863e6a2 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/s3/AmazonS3KeyValueAccess.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/s3/AmazonS3KeyValueAccess.java @@ -37,15 +37,14 @@ import java.io.Writer; import java.net.URI; import java.net.URISyntaxException; -import java.net.URL; import java.nio.channels.NonReadableChannelException; import java.nio.charset.StandardCharsets; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.Iterator; import java.util.List; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import java.util.stream.Collectors; import org.janelia.saalfeldlab.n5.KeyValueAccess; @@ -54,7 +53,6 @@ import org.janelia.saalfeldlab.n5.N5URI; import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3URI; import com.amazonaws.services.s3.model.CreateBucketRequest; import com.amazonaws.services.s3.model.DeleteObjectsRequest; import com.amazonaws.services.s3.model.GetObjectMetadataRequest; @@ -69,27 +67,56 @@ public class AmazonS3KeyValueAccess implements KeyValueAccess { - private static final Pattern AWS_ENDPOINT_PATTERN = Pattern.compile("^(.+\\.)?(s3\\..*amazonaws\\.com)$"); - private final AmazonS3 s3; + private final URI containerURI; private final String bucketName; + private static URI uncheckedContainterLocationStringToURI(String uri) { + + try { + return N5URI.encodeAsUri(uri); + } catch (URISyntaxException e) { + throw new N5Exception("Container location " + uri + " is an invalid URI", e); + } + } + /** * Opens an {@link AmazonS3KeyValueAccess} using an {@link AmazonS3} client and a given bucket name. + *

+ * If the bucket does not exist and {@code createBucket==true}, the bucket will be created. + * If the bucket does not exist and {@code createBucket==false}, the bucket will not be + * created and all subsequent attempts to read attributes, groups, or datasets will fail. * + * @param s3 the s3 instance + * @param containerURI the URI that points to the n5 container root. + * @param createBucket whether {@code bucketName} should be created if it doesn't exist + * @throws N5Exception.N5IOException if the access could not be created + * @deprecated containerURI must be valid URI, call constructor with URI instead of String {@link AmazonS3KeyValueAccess#AmazonS3KeyValueAccess(AmazonS3, URI, boolean)} + */ + @Deprecated + public AmazonS3KeyValueAccess(final AmazonS3 s3, String containerURI, final boolean createBucket) throws N5Exception.N5IOException { + + this(s3, uncheckedContainterLocationStringToURI(containerURI), createBucket); + } + + /** + * Opens an {@link AmazonS3KeyValueAccess} using an {@link AmazonS3} client and a given bucket name. + *

* If the bucket does not exist and {@code createBucket==true}, the bucket will be created. * If the bucket does not exist and {@code createBucket==false}, the bucket will not be * created and all subsequent attempts to read attributes, groups, or datasets will fail. * - * @param s3 the s3 instance - * @param bucketName the bucket name + * @param s3 the s3 instance + * @param containerURI the URI that points to the n5 container root. * @param createBucket whether {@code bucketName} should be created if it doesn't exist * @throws N5Exception.N5IOException if the access could not be created */ - public AmazonS3KeyValueAccess(final AmazonS3 s3, final String bucketName, final boolean createBucket) throws N5Exception.N5IOException { + public AmazonS3KeyValueAccess(final AmazonS3 s3, final URI containerURI, final boolean createBucket) throws N5Exception.N5IOException { this.s3 = s3; - this.bucketName = bucketName; + this.containerURI = containerURI; + + this.bucketName = AmazonS3Utils.getS3Bucket(containerURI); if (!s3.doesBucketExistV2(bucketName)) { if (createBucket) { @@ -110,7 +137,10 @@ public AmazonS3KeyValueAccess(final AmazonS3 s3, final String bucketName, final @Override public String[] components(final String path) { - return Arrays.stream(path.split("/")) + final String[] baseComponents = path.split("/"); + if (baseComponents.length <= 1) + return baseComponents; + return Arrays.stream(baseComponents) .filter(x -> !x.isEmpty()) .toArray(String[]::new); } @@ -123,8 +153,8 @@ public String compose(final String... components) { return normalize( Arrays.stream(components) - .filter(x -> !x.isEmpty()) - .collect(Collectors.joining("/")) + .filter(x -> !x.isEmpty()) + .collect(Collectors.joining("/")) ); } @@ -132,8 +162,8 @@ public String compose(final String... components) { /** * Compose a path from a base uri and subsequent components. * - * @param uri the base path uri - * @param components the path components + * @param uri the base path uri to resolve the components against + * @param components the components of the group path, relative to the n5 container * @return the path */ @Override @@ -141,13 +171,7 @@ public String compose(final URI uri, final String... components) { final String[] uriComponents = new String[components.length + 1]; System.arraycopy(components, 0, uriComponents, 1, components.length); - if ("s3".equalsIgnoreCase(uri.getScheme())) { - // when using the s3 scheme, the bucket name is the "host", and the group is the "path" - uriComponents[0] = uri.getPath(); - } else { - // when using the http(s) scheme, need to do more checks, getS3Key does them - uriComponents[0] = getS3Key(uri.toString()); - } + uriComponents[0] = AmazonS3Utils.getS3Key(uri); return compose(uriComponents); } @@ -155,7 +179,7 @@ public String compose(final URI uri, final String... components) { public String parent(final String path) { final String[] components = components(path); - final String[] parentComponents =Arrays.copyOf(components, components.length - 1); + final String[] parentComponents = Arrays.copyOf(components, components.length - 1); return compose(parentComponents); } @@ -168,9 +192,9 @@ public String relativize(final String path, final String base) { * It's not true that the inputs are always referencing absolute paths, but it doesn't matter in this * case, since we only care about the relative portion of `path` to `base`, so the result always * ignores the absolute prefix anyway. */ - return getS3Key(normalize(uri("/" + base).relativize(uri("/" + path)).toString())); + return AmazonS3Utils.getS3Key(normalize(uri("/" + base).relativize(uri("/" + path)).toString())); } catch (final URISyntaxException e) { - throw new N5Exception("Cannot relativize path (" + path +") with base (" + base + ")", e); + throw new N5Exception("Cannot relativize path (" + path + ") with base (" + base + ")", e); } } @@ -180,47 +204,40 @@ public String normalize(final String path) { return N5URI.normalizeGroupPath(path); } + /** + * Create a URI that is the result of resolving the `normalPath` against the {@link #containerURI}. + * NOTE: {@link URI#resolve(URI)} always removes the last member of the receiver URIs path. + * That is undesirable behavior here, as we want to potentially keep the containerURI's + * full path, and just append `normalPath`. However, it's more complicated, as `normalPath` + * can also contain leading overlap with the trailing members of `containerURI.getPath()`. + * To properly resolve the two paths, we generate {@link Path}s from the results of {@link URI#getPath()} + * and use {@link Path#resolve(Path)}, which results in a guaranteed absolute path, with the + * desired path resolution behavior. That then is used to construct a new {@link URI}. + * Any query or fragment portions are ignored. Scheme and Authority are always + * inherited from {@link #containerURI}. + * + * @param normalPath EITHER a normalized path, or a valid URI + * @return the URI generated from resolving normalPath against containerURI + * @throws URISyntaxException if the given normal path is not a valid URI + */ @Override public URI uri(final String normalPath) throws URISyntaxException { - final URL url = s3.getUrl(bucketName, normalize(normalPath)); - final Matcher matcher = AWS_ENDPOINT_PATTERN.matcher(url.getHost()); - if( matcher.find() ) - return N5URI.from( - "s3://" + bucketName + (normalPath.startsWith("/") ? normalPath : "/" + normalPath), null, null) - .getURI(); - else { - return url.toURI(); - } - } + if (normalize(normalPath).equals(normalize("/"))) + return containerURI; - private String getS3Bucket(final String uri) { + final Path containerPath = Paths.get(containerURI.getPath()); + final Path givenPath = Paths.get(URI.create(normalPath).getPath()); - try { - return new AmazonS3URI(uri).getBucket(); - } catch (final IllegalArgumentException e) {} - try { - // parse bucket manually when AmazonS3URI can't - final String path = new URI(uri).getPath().replaceFirst("^/", ""); - return path.substring(0, path.indexOf('/')); - } catch (final URISyntaxException e) { + final Path resolvedPath = containerPath.resolve(givenPath); + final String[] pathParts = new String[resolvedPath.getNameCount() + 1]; + pathParts[0] = "/"; + for (int i = 0; i < resolvedPath.getNameCount(); i++) { + pathParts[i + 1] = resolvedPath.getName(i).toString(); } - return null; - } + final String normalResolvedPath = compose(pathParts); - private String getS3Key(final String uri) { - - try { - // if key is null, return the empty string - final String key = new AmazonS3URI(uri).getKey(); - return key == null ? "" : key; - } catch (final IllegalArgumentException e) {} - try { - // parse key manually when AmazonS3URI can't - final String path = new URI(uri).getPath().replaceFirst("^/", ""); - return path.substring(path.indexOf('/') + 1); - } catch (final URISyntaxException e) {} - return null; + return new URI(containerURI.getScheme(), containerURI.getAuthority(), normalResolvedPath, null, null); } /** @@ -230,7 +247,7 @@ private String getS3Key(final String uri) { * either {@code path} or {@code path + "/"} is a key. * * @param normalPath is expected to be in normalized form, no further - * efforts are made to normalize it. + * efforts are made to normalize it. * @return {@code true} if {@code path} exists, {@code false} otherwise */ @Override @@ -239,24 +256,6 @@ public boolean exists(final String normalPath) { return isDirectory(normalPath) || isFile(normalPath); } - /** - * Find the smallest key with the given {@code prefix}. - * - * @return shortest key with the given {@code prefix}, or {@code null} if there is no key with that prefix. - */ - // TODO: REMOVE? - private String shortestKeyWithPrefix(final String prefix) { - - final ListObjectsV2Request listObjectsRequest = new ListObjectsV2Request() - .withBucketName(bucketName) - .withPrefix(prefix) - .withMaxKeys(1); - final ListObjectsV2Result objectsListing = s3.listObjectsV2(listObjectsRequest); - return objectsListing.getKeyCount() > 0 - ? objectsListing.getObjectSummaries().get(0).getKey() - : null; - } - private ListObjectsV2Result queryPrefix(final String prefix) { final ListObjectsV2Request listObjectsRequest = new ListObjectsV2Request() @@ -278,16 +277,12 @@ private boolean keyExists(final String key) { * returns the correct key, but I'm not confident we can count on that in general. * HeadObjectFunction (found by Caleb) is probably preferable for that reason. -John */ - // final ListObjectsV2Result objectsListing = queryPrefix(key); - // return objectsListing.getKeyCount() > 0 && - // objectsListing.getObjectSummaries().get(0).getKey().equals(key); - try { final ObjectMetadata objMeta = new HeadObjectFunction(s3).apply(new GetObjectMetadataRequest(bucketName, key)); return objMeta != null; - } catch (Exception e) {} - - return false; + } catch (Exception e) { + return false; + } } /** @@ -333,14 +328,17 @@ private static String removeLeadingSlash(final String path) { * leading "/", and then checks whether resulting {@code path} is a key. * * @param normalPath is expected to be in normalized form, no further - * efforts are made to normalize it. + * efforts are made to normalize it. * @return {@code true} if {@code path} (with trailing "/") exists as a key, {@code false} otherwise */ @Override public boolean isDirectory(final String normalPath) { final String key = removeLeadingSlash(addTrailingSlash(normalPath)); - return key.isEmpty() || prefixExists(key); + if (key.equals(normalize("/"))) { + return s3.doesBucketExistV2(bucketName); + } + return prefixExists(key); } /** @@ -350,7 +348,7 @@ public boolean isDirectory(final String normalPath) { * leading "/" and checks whether the resulting {@code path} is a key. * * @param normalPath is expected to be in normalized form, no further - * efforts are made to normalize it. + * efforts are made to normalize it. * @return {@code true} if {@code path} exists as a key and has no trailing slash, {@code false} otherwise */ @Override @@ -360,13 +358,13 @@ public boolean isFile(final String normalPath) { } @Override - public LockedChannel lockForReading(final String normalPath) throws IOException { + public LockedChannel lockForReading(final String normalPath) { return new S3ObjectChannel(removeLeadingSlash(normalPath), true); } @Override - public LockedChannel lockForWriting(final String normalPath) throws IOException { + public LockedChannel lockForWriting(final String normalPath) { return new S3ObjectChannel(removeLeadingSlash(normalPath), false); } @@ -395,7 +393,6 @@ private String[] list(final String normalPath, final boolean onlyDirectories) { for (final String commonPrefix : objectsListing.getCommonPrefixes()) { if (!onlyDirectories || commonPrefix.endsWith("/")) { final String relativePath = relativize(commonPrefix, prefix); - // TODO: N5AmazonS3Reader#list used replaceBackSlashes(relativePath) here. Is this necessary? if (!relativePath.isEmpty()) subGroups.add(relativePath); } @@ -412,7 +409,7 @@ public String[] list(final String normalPath) throws IOException { } @Override - public void createDirectories(final String normalPath) throws IOException { + public void createDirectories(final String normalPath) { String path = ""; for (final String component : components(removeLeadingSlash(normalPath))) { @@ -431,7 +428,7 @@ public void createDirectories(final String normalPath) throws IOException { } @Override - public void delete(final String normalPath) throws IOException { + public void delete(final String normalPath) { if (!s3.doesBucketExistV2(bucketName)) return; @@ -442,22 +439,22 @@ public void delete(final String normalPath) throws IOException { // need to delete all objects before deleting the bucket // see: https://docs.aws.amazon.com/AmazonS3/latest/userguide/delete-bucket.html ObjectListing objectListing = s3.listObjects(bucketName); - while (true) { - final Iterator objIter = objectListing.getObjectSummaries().iterator(); - while (objIter.hasNext()) { - s3.deleteObject(bucketName, objIter.next().getKey()); - } - - // If the bucket contains many objects, the listObjects() call - // might not return all of the objects in the first listing. Check to - // see whether the listing was truncated. If so, retrieve the next page of objects - // and delete them. - if (objectListing.isTruncated()) { - objectListing = s3.listNextBatchOfObjects(objectListing); - } else { - break; - } - } + while (true) { + final Iterator objIter = objectListing.getObjectSummaries().iterator(); + while (objIter.hasNext()) { + s3.deleteObject(bucketName, objIter.next().getKey()); + } + + // If the bucket contains many objects, the listObjects() call + // might not return all of the objects in the first listing. Check to + // see whether the listing was truncated. If so, retrieve the next page of objects + // and delete them. + if (objectListing.isTruncated()) { + objectListing = s3.listNextBatchOfObjects(objectListing); + } else { + break; + } + } s3.deleteBucket(bucketName); return; @@ -467,7 +464,7 @@ public void delete(final String normalPath) throws IOException { if (!path.endsWith("/")) { s3.deleteObjects(new DeleteObjectsRequest(bucketName) - .withKeys(new String[]{path})); + .withKeys(path)); } final String prefix = addTrailingSlash(path); @@ -491,12 +488,12 @@ public void delete(final String normalPath) throws IOException { /** * Helper class that drains the rest of the {@link S3ObjectInputStream} on {@link #close()}. - * + *

* Without draining the stream AWS S3 SDK sometimes outputs the following warning message: * "... Not all bytes were read from the S3ObjectInputStream, aborting HTTP connection ...". - * + *

* Draining the stream helps to avoid this warning and possibly reuse HTTP connections. - * + *

* Calling {@link S3ObjectInputStream#abort()} does not prevent this warning as discussed here: * https://github.com/aws/aws-sdk-java/issues/1211 */ @@ -571,7 +568,7 @@ private class S3ObjectChannel implements LockedChannel { final boolean readOnly; private final ArrayList resources = new ArrayList<>(); - protected S3ObjectChannel(final String path, final boolean readOnly) throws IOException { + protected S3ObjectChannel(final String path, final boolean readOnly) { this.path = path; this.readOnly = readOnly; @@ -585,7 +582,7 @@ private void checkWritable() { } @Override - public InputStream newInputStream() throws IOException { + public InputStream newInputStream() { final S3ObjectInputStream in = s3.getObject(bucketName, path).getObjectContent(); final S3ObjectInputStreamDrain s3in = new S3ObjectInputStreamDrain(in); @@ -596,7 +593,7 @@ public InputStream newInputStream() throws IOException { } @Override - public Reader newReader() throws IOException { + public Reader newReader() { final InputStreamReader reader = new InputStreamReader(newInputStream(), StandardCharsets.UTF_8); synchronized (resources) { @@ -606,7 +603,7 @@ public Reader newReader() throws IOException { } @Override - public OutputStream newOutputStream() throws IOException { + public OutputStream newOutputStream() { checkWritable(); final S3OutputStream s3Out = new S3OutputStream(); @@ -643,13 +640,13 @@ final class S3OutputStream extends OutputStream { private boolean closed = false; @Override - public void write(final byte[] b, final int off, final int len) throws IOException { + public void write(final byte[] b, final int off, final int len) { buf.write(b, off, len); } @Override - public void write(final int b) throws IOException { + public void write(final int b) { buf.write(b); } diff --git a/src/main/java/org/janelia/saalfeldlab/n5/s3/AmazonS3Utils.java b/src/main/java/org/janelia/saalfeldlab/n5/s3/AmazonS3Utils.java new file mode 100644 index 0000000..a076443 --- /dev/null +++ b/src/main/java/org/janelia/saalfeldlab/n5/s3/AmazonS3Utils.java @@ -0,0 +1,212 @@ +package org.janelia.saalfeldlab.n5.s3; + +import com.amazonaws.auth.AWSCredentials; +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.AnonymousAWSCredentials; +import com.amazonaws.auth.DefaultAWSCredentialsProviderChain; +import com.amazonaws.client.builder.AwsClientBuilder; +import com.amazonaws.regions.Regions; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import com.amazonaws.services.s3.AmazonS3URI; +import com.amazonaws.services.s3.model.AmazonS3Exception; +import com.amazonaws.services.s3.model.ListObjectsV2Request; +import org.janelia.saalfeldlab.n5.N5Exception; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import javax.annotation.Nullable; + +public class AmazonS3Utils { + public static final Pattern AWS_ENDPOINT_PATTERN = Pattern.compile("^(.+\\.)?(s3\\..*amazonaws\\.com)", Pattern.CASE_INSENSITIVE); + public final static Pattern S3_SCHEME = Pattern.compile("s3", Pattern.CASE_INSENSITIVE); + + private AmazonS3Utils() { + + } + + public static String getS3Bucket(final String uri) { + try { + return getS3Bucket(new URI(uri)); + } catch (final URISyntaxException e) { + } + return null; + } + public static String getS3Bucket(final URI uri) { + + try { + return new AmazonS3URI(uri).getBucket(); + } catch (final IllegalArgumentException e) { + } + // parse bucket manually when AmazonS3URI can't + final String path = uri.getPath().replaceFirst("^/", ""); + return path.split("/")[0]; + } + + public static String getS3Key(final String uri) { + try { + return getS3Key(new URI(uri)); + } catch (final URISyntaxException e) { + } + return ""; + } + public static String getS3Key(final URI uri) { + + try { + // if key is null, return the empty string + final String key = new AmazonS3URI(uri).getKey(); + return key == null ? "" : key; + } catch (final IllegalArgumentException e) { + } + // parse key manually when AmazonS3URI can't + final String path = uri.getPath().replaceFirst("^/", ""); + return path.substring(path.indexOf('/') + 1); + } + + public static boolean areAnonymous(final AWSCredentialsProvider credsProvider) { + + final AWSCredentials creds = credsProvider.getCredentials(); + // AnonymousAWSCredentials do not have an equals method + if (creds.getClass().equals(AnonymousAWSCredentials.class)) + return true; + + return creds.getAWSAccessKeyId() == null && creds.getAWSSecretKey() == null; + } + + public static Regions getS3Region(final AmazonS3URI uri, @Nullable final String region) { + + final Regions regionFromUri = parseRegion(uri.getRegion()); + return regionFromUri != null ? regionFromUri : parseRegion(region); + } + + private static Regions parseRegion(String stringRegionFromUri) { + + return stringRegionFromUri != null ? Regions.fromName(stringRegionFromUri) : null; + } + + public static AWSStaticCredentialsProvider getS3Credentials(final AWSCredentials s3Credentials, final boolean s3Anonymous) { + + AWSCredentials credentials = null; + final AWSStaticCredentialsProvider credentialsProvider; + if (s3Credentials != null) { + credentials = s3Credentials; + credentialsProvider = new AWSStaticCredentialsProvider(credentials); + } else { + // if not anonymous, try finding credentials + if (!s3Anonymous) { + try { + credentials = new DefaultAWSCredentialsProviderChain().getCredentials(); + } catch (final Exception e) { + System.out.println("Could not load AWS credentials, falling back to anonymous."); + } + credentialsProvider = new AWSStaticCredentialsProvider( + credentials == null ? new AnonymousAWSCredentials() : credentials); + } else + credentialsProvider = new AWSStaticCredentialsProvider(new AnonymousAWSCredentials()); + } + + return credentialsProvider; + } + + public static AmazonS3 createS3(final String uri) { + + return createS3(uri, (String)null, null, null); + } + + public static AmazonS3 createS3(final String uri, @Nullable final String s3Endpoint, @Nullable final AWSCredentialsProvider s3Credentials, @Nullable String region) { + + try { + final AmazonS3URI s3Uri = new AmazonS3URI(uri); + return createS3(s3Uri, s3Endpoint, s3Credentials, region); + } catch (final IllegalArgumentException e) { + // if AmazonS3URI does not like the form of the uri + try { + final URI asURI = new URI(uri); + final URI endpointUri = new URI(asURI.getScheme(), asURI.getAuthority(), null, null); + return createS3(AmazonS3Utils.getS3Bucket(uri), s3Credentials, new AwsClientBuilder.EndpointConfiguration(endpointUri.toString(), null), null); + } catch (final URISyntaxException e1) { + throw new N5Exception("Could not create s3 client from uri: " + uri, e1); + } + } + } + + public static AmazonS3 createS3(final AmazonS3URI s3Uri, @Nullable final String s3Endpoint, @Nullable final AWSCredentialsProvider s3Credentials, @Nullable final String region) { + + AwsClientBuilder.EndpointConfiguration endpointConfiguration = null; + if (!S3_SCHEME.matcher(s3Uri.getURI().getScheme()).matches()) { + endpointConfiguration = createEndpointConfiguration(s3Uri, s3Endpoint); + } + return createS3(s3Uri.getBucket(), s3Credentials, endpointConfiguration, getS3Region(s3Uri, region)); + } + + public static AwsClientBuilder.EndpointConfiguration createEndpointConfiguration(final AmazonS3URI s3Uri, @Nullable final String s3Endpoint) { + + AwsClientBuilder.EndpointConfiguration endpointConfiguration; + if (s3Endpoint != null) + endpointConfiguration = new AwsClientBuilder.EndpointConfiguration(s3Endpoint, null); + else { + final Matcher matcher = AmazonS3Utils.AWS_ENDPOINT_PATTERN.matcher(s3Uri.getURI().getHost()); + if (matcher.find()) + endpointConfiguration = new AwsClientBuilder.EndpointConfiguration(matcher.group(2), s3Uri.getRegion()); + else + endpointConfiguration = new AwsClientBuilder.EndpointConfiguration(s3Uri.getURI().getHost(), s3Uri.getRegion()); + } + return endpointConfiguration; + } + + public static AmazonS3 createS3( + final String bucketName, + @Nullable final AWSCredentialsProvider credentialsProvider, + @Nullable final AwsClientBuilder.EndpointConfiguration endpointConfiguration, + @Nullable final Regions region) { + + final boolean isAmazon = endpointConfiguration == null || AmazonS3Utils.AWS_ENDPOINT_PATTERN.matcher(endpointConfiguration.getServiceEndpoint()).find(); + final AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard(); + + if (!isAmazon) + builder.withPathStyleAccessEnabled(true); + + if (credentialsProvider != null) + builder.withCredentials(credentialsProvider); + + if (endpointConfiguration != null) + builder.withEndpointConfiguration(endpointConfiguration); + else if (region != null) + builder.withRegion(region); + else + builder.withRegion("us-east-1"); + + AmazonS3 s3 = builder.build(); + // try to listBucket if we are anonymous, if we cannot, don't use anonymous. + if (credentialsProvider != null && AmazonS3Utils.areAnonymous(credentialsProvider)) { + + // I initially tried checking whether the bucket exists, but + // that, apparently, returns even when the client does not have access + if (!canListBucket(s3, bucketName)) { + // bucket not detected with anonymous credentials, try detecting credentials + // and return it even if it can't detect the bucket, since there's nothing else to do + s3 = createS3(null, new DefaultAWSCredentialsProviderChain(), endpointConfiguration, region); + } + } + return s3; + } + + private static boolean canListBucket(final AmazonS3 s3, final String bucket) { + + final ListObjectsV2Request request = new ListObjectsV2Request(); + request.setBucketName(bucket); + request.setMaxKeys(1); + + try { + // list objects will throw an AmazonS3Exception (Access Denied) if this client does not have access + s3.listObjectsV2(request); + return true; + } catch (final AmazonS3Exception e) { + return false; + } + } +} diff --git a/src/main/java/org/janelia/saalfeldlab/n5/s3/N5AmazonS3Reader.java b/src/main/java/org/janelia/saalfeldlab/n5/s3/N5AmazonS3Reader.java index 34108dd..c6cde2e 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/s3/N5AmazonS3Reader.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/s3/N5AmazonS3Reader.java @@ -35,16 +35,20 @@ import org.janelia.saalfeldlab.n5.N5KeyValueReader; /** - * TODO: javadoc + * + * @deprecated This class is deprecated and may be removed in a future release. + * Replace with either `N5Factory.openReader()` or `N5KeyValueAccessReader` with + * an `AmazonS3KeyValueAccess` backend. */ +@Deprecated public class N5AmazonS3Reader extends N5KeyValueReader { - /** - * TODO: reduce number of constructors ? - */ - /** * Opens an {@link N5Reader} with an {@link AmazonS3} storage backend. + * + * @deprecated This class is deprecated and may be removed in a future release. + * Replace with either `N5Factory.openReader()` or `N5KeyValueAccessReader` with + * an `AmazonS3KeyValueAccess` backend. * * @param s3 the amazon s3 instance * @param bucketName the bucket name @@ -59,10 +63,11 @@ public class N5AmazonS3Reader extends N5KeyValueReader { * independent writer will not be tracked. * @throws N5Exception if the reader could not be created */ + @Deprecated public N5AmazonS3Reader(final AmazonS3 s3, final String bucketName, final String basePath, final GsonBuilder gsonBuilder, final boolean cacheMeta) throws N5Exception { super( - new AmazonS3KeyValueAccess(s3, bucketName, false), + new AmazonS3KeyValueAccess(s3, "s3://" + bucketName + "/" + basePath, false), basePath, gsonBuilder, cacheMeta); diff --git a/src/main/java/org/janelia/saalfeldlab/n5/s3/N5AmazonS3Writer.java b/src/main/java/org/janelia/saalfeldlab/n5/s3/N5AmazonS3Writer.java index 96752b3..af6a433 100644 --- a/src/main/java/org/janelia/saalfeldlab/n5/s3/N5AmazonS3Writer.java +++ b/src/main/java/org/janelia/saalfeldlab/n5/s3/N5AmazonS3Writer.java @@ -28,8 +28,6 @@ */ package org.janelia.saalfeldlab.n5.s3; -import java.io.IOException; - import org.janelia.saalfeldlab.n5.N5Writer; import org.janelia.saalfeldlab.n5.N5Exception; import org.janelia.saalfeldlab.n5.N5KeyValueWriter; @@ -38,14 +36,15 @@ import com.google.gson.GsonBuilder; /** - * TODO: javadoc + * This class is used to create an N5Writer with an Amazon S3 storage backend. + * + * @deprecated This class is deprecated and may be removed in a future release. + * Replace with either `N5Factory.openWriter()` or `N5KeyValueAccessWriter` with + * an `AmazonS3KeyValueAccess` backend. */ +@Deprecated public class N5AmazonS3Writer extends N5KeyValueWriter { - /** - * TODO: reduce number of constructors ? - */ - /** * Opens an {@link N5Writer} with an {@link AmazonS3} storage backend. * @@ -65,7 +64,7 @@ public class N5AmazonS3Writer extends N5KeyValueWriter { public N5AmazonS3Writer(final AmazonS3 s3, final String bucketName, final String basePath, final GsonBuilder gsonBuilder, final boolean cacheAttributes) throws N5Exception { super( - new AmazonS3KeyValueAccess(s3, bucketName, true), + new AmazonS3KeyValueAccess(s3, "s3://" + bucketName + "/" + basePath, true), basePath, gsonBuilder, cacheAttributes); diff --git a/src/test/java/org/janelia/saalfeldlab/n5/s3/AbstractN5AmazonS3BucketRootTest.java b/src/test/java/org/janelia/saalfeldlab/n5/s3/AbstractN5AmazonS3BucketRootTest.java deleted file mode 100644 index f518cdc..0000000 --- a/src/test/java/org/janelia/saalfeldlab/n5/s3/AbstractN5AmazonS3BucketRootTest.java +++ /dev/null @@ -1,48 +0,0 @@ -/*- - * #%L - * N5 AWS S3 - * %% - * Copyright (C) 2017 - 2022, Saalfeld Lab - * %% - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * #L% - */ -package org.janelia.saalfeldlab.n5.s3; - -import com.amazonaws.services.s3.AmazonS3; -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import org.junit.AfterClass; - -public abstract class AbstractN5AmazonS3BucketRootTest extends AbstractN5AmazonS3Test { - - public AbstractN5AmazonS3BucketRootTest(final AmazonS3 s3) { - - super(s3); - } - - @Override - protected String tempN5Location() throws URISyntaxException { - return new URI("s3", tempBucketName(), "/", null).toString(); - } -} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/s3/AbstractN5AmazonS3ContainerPathTest.java b/src/test/java/org/janelia/saalfeldlab/n5/s3/AbstractN5AmazonS3ContainerPathTest.java deleted file mode 100644 index 5a4d2a4..0000000 --- a/src/test/java/org/janelia/saalfeldlab/n5/s3/AbstractN5AmazonS3ContainerPathTest.java +++ /dev/null @@ -1,63 +0,0 @@ -/*- - * #%L - * N5 AWS S3 - * %% - * Copyright (C) 2017 - 2022, Saalfeld Lab - * %% - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * #L% - */ -package org.janelia.saalfeldlab.n5.s3; - -import com.amazonaws.services.s3.AmazonS3; -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import org.junit.AfterClass; -import org.junit.BeforeClass; - -public abstract class AbstractN5AmazonS3ContainerPathTest extends AbstractN5AmazonS3Test { - - protected static String bucketName; - - public AbstractN5AmazonS3ContainerPathTest(final AmazonS3 s3) { - - super(s3); - } - - @BeforeClass - public static void setup() throws IOException, URISyntaxException { - bucketName = tempBucketName(); - } - - @Override - protected String tempN5Location() throws URISyntaxException { - return new URI("s3", bucketName, tempContainerPath(), null).toString(); - } - - @AfterClass - public static void cleanup() throws IOException { - - s3.deleteBucket(bucketName); - } - -} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/s3/AbstractN5AmazonS3Test.java b/src/test/java/org/janelia/saalfeldlab/n5/s3/AbstractN5AmazonS3Test.java deleted file mode 100644 index 8835ba2..0000000 --- a/src/test/java/org/janelia/saalfeldlab/n5/s3/AbstractN5AmazonS3Test.java +++ /dev/null @@ -1,191 +0,0 @@ -/*- - * #%L - * N5 AWS S3 - * %% - * Copyright (C) 2017 - 2022, Saalfeld Lab - * %% - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * #L% - */ -package org.janelia.saalfeldlab.n5.s3; - -import java.io.IOException; -import java.net.URI; -import java.net.URISyntaxException; -import java.security.SecureRandom; - -import org.janelia.saalfeldlab.n5.AbstractN5Test; -import org.janelia.saalfeldlab.n5.N5Exception; -import org.janelia.saalfeldlab.n5.N5Reader; -import org.janelia.saalfeldlab.n5.N5Writer; -import org.junit.Assert; -import org.junit.Test; - -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3URI; -import com.google.gson.GsonBuilder; - -/** - * Base class for testing Amazon Web Services N5 implementation. - * Tests that are specific to AWS S3 can be added here. - * - * @author Igor Pisarev <pisarevi@janelia.hhmi.org> - */ -public abstract class AbstractN5AmazonS3Test extends AbstractN5Test { - - protected static AmazonS3 s3; - - public AbstractN5AmazonS3Test(final AmazonS3 s3) { - - AbstractN5AmazonS3Test.s3 = s3; - } - - private static final SecureRandom random = new SecureRandom(); - - private static String generateName(final String prefix, final String suffix) { - - return prefix + Long.toUnsignedString(random.nextLong()) + suffix; - } - - protected static String tempBucketName() { - - return generateName("n5-test-", "-bucket"); - } - - protected static String tempContainerPath() { - - return generateName("/n5-test-", ".n5"); - } - - @Override protected N5Writer createN5Writer() throws IOException, URISyntaxException { - - final String location = tempN5Location(); - final String bucketName = getS3Bucket( location ); - final String basePath = getS3Key(location); - return new N5AmazonS3Writer(s3, bucketName, basePath, new GsonBuilder()) { - - @Override public void close() { - - remove(); - super.close(); - } - }; - } - - @Override - protected N5Writer createN5Writer(final String location, final GsonBuilder gson) throws IOException, URISyntaxException { - - final String bucketName = getS3Bucket(location); - final String basePath = getS3Key(location); - return new N5AmazonS3Writer(s3, bucketName, basePath, gson); - } - - @Override - protected N5Reader createN5Reader(final String location, final GsonBuilder gson) throws IOException, URISyntaxException { - - final String bucketName = getS3Bucket(location); - final String basePath = getS3Key(location); - return new N5AmazonS3Reader(s3, bucketName, basePath, gson); - } - - protected String getS3Bucket(final String uri) { - - try { - return new AmazonS3URI(uri).getBucket(); - } catch (final IllegalArgumentException e) {} - try { - // parse bucket manually when AmazonS3URI can't - final String path = new URI(uri).getPath().replaceFirst("^/", ""); - return path.substring(0, path.indexOf('/')); - } catch (final URISyntaxException e) { - } - return null; - } - - protected String getS3Key(final String uri) { - - try { - // if key is null, return the empty string - final String key = new AmazonS3URI(uri).getKey(); - return key == null ? "" : key; - } catch (final IllegalArgumentException e) {} - try { - // parse key manually when AmazonS3URI can't - final String path = new URI(uri).getPath().replaceFirst("^/", ""); - return path.substring(path.indexOf('/') + 1); - } catch (final URISyntaxException e) {} - return ""; - } - - /** - * Currently, {@code N5AmazonS3Reader#exists(String)} is implemented by listing objects under that group. - * This test case specifically tests its correctness. - * - * @throws IOException - */ - @Test - public void testExistsUsingListingObjects() throws IOException, URISyntaxException { - - try (N5Writer n5 = createN5Writer()) { - n5.createGroup("/one/two/three"); - - Assert.assertTrue(n5.exists("")); - Assert.assertTrue(n5.exists("/")); - - Assert.assertTrue(n5.exists("one")); - Assert.assertTrue(n5.exists("one/")); - Assert.assertTrue(n5.exists("/one")); - Assert.assertTrue(n5.exists("/one/")); - - Assert.assertTrue(n5.exists("one/two")); - Assert.assertTrue(n5.exists("one/two/")); - Assert.assertTrue(n5.exists("/one/two")); - Assert.assertTrue(n5.exists("/one/two/")); - - Assert.assertTrue(n5.exists("one/two/three")); - Assert.assertTrue(n5.exists("one/two/three/")); - Assert.assertTrue(n5.exists("/one/two/three")); - Assert.assertTrue(n5.exists("/one/two/three/")); - - Assert.assertFalse(n5.exists("one/tw")); - Assert.assertFalse(n5.exists("one/tw/")); - Assert.assertFalse(n5.exists("/one/tw")); - Assert.assertFalse(n5.exists("/one/tw/")); - - Assert.assertArrayEquals(new String[]{"one"}, n5.list("/")); - Assert.assertArrayEquals(new String[]{"two"}, n5.list("/one")); - Assert.assertArrayEquals(new String[]{"three"}, n5.list("/one/two")); - - Assert.assertArrayEquals(new String[]{}, n5.list("/one/two/three")); - Assert.assertThrows(N5Exception.N5IOException.class, () -> n5.list("/one/tw")); - - Assert.assertTrue(n5.remove("/one/two/three")); - Assert.assertFalse(n5.exists("/one/two/three")); - Assert.assertTrue(n5.exists("/one/two")); - Assert.assertTrue(n5.exists("/one")); - - Assert.assertTrue(n5.remove("/one")); - Assert.assertFalse(n5.exists("/one/two")); - Assert.assertFalse(n5.exists("/one")); - } - } -} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/s3/AmazonS3UtilsTest.java b/src/test/java/org/janelia/saalfeldlab/n5/s3/AmazonS3UtilsTest.java new file mode 100644 index 0000000..e9e727f --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/s3/AmazonS3UtilsTest.java @@ -0,0 +1,41 @@ +package org.janelia.saalfeldlab.n5.s3; + +import static org.junit.Assert.assertEquals; + +import java.net.URI; +import java.net.URISyntaxException; + +import org.junit.Test; + +public class AmazonS3UtilsTest { + + @Test + public void testUriParsing() throws URISyntaxException { + + // dummy client + String[] prefixes = new String[]{ + "s3://", + "https://s3-eu-west-1.amazonaws.com/", + "http://localhost:8001/", + }; + + String[] buckets = new String[]{ + "zarr-n5-demo", + "static.wk.org"}; + + String[] paths = new String[]{ + "", + "foo.zarr", + "data/sample"}; + + for (String prefix : prefixes) + for (String bucket : buckets) + for (String path : paths) { + URI uri = new URI(prefix + bucket + "/" + path); + assertEquals("bucket from uri", bucket, AmazonS3Utils.getS3Bucket(uri)); + assertEquals("key from uri", path, AmazonS3Utils.getS3Key(uri)); + } + + } + +} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/s3/IgnoreTestCasesByParameter.java b/src/test/java/org/janelia/saalfeldlab/n5/s3/IgnoreTestCasesByParameter.java new file mode 100644 index 0000000..6196edb --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/s3/IgnoreTestCasesByParameter.java @@ -0,0 +1,64 @@ +package org.janelia.saalfeldlab.n5.s3; + +import org.junit.runner.Description; +import org.junit.runner.Runner; +import org.junit.runner.manipulation.Filter; +import org.junit.runner.manipulation.Filterable; +import org.junit.runner.manipulation.NoTestsRemainException; +import org.junit.runners.model.InitializationError; +import org.junit.runners.parameterized.BlockJUnit4ClassRunnerWithParametersFactory; +import org.junit.runners.parameterized.TestWithParameters; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; +import java.util.Objects; + +public class IgnoreTestCasesByParameter extends BlockJUnit4ClassRunnerWithParametersFactory { + + private static Filter skipAll = new Filter() { + + @Override public boolean shouldRun(Description description) { + + return false; + } + + @Override public String describe() { + + return "Backend Tests Not Enabled"; + } + }; + @Override public Runner createRunnerForTestWithParameters(TestWithParameters test) throws InitializationError { + + final IgnoreParameter annotation = test.getTestClass().getAnnotation(IgnoreParameter.class); + final int ignoreIndex = annotation != null ? annotation.index() : -1; + final int idx = ignoreIndex == -1 ? test.getParameters().size() - 1 : ignoreIndex; + + final Object ignoreTests = test.getParameters().get(idx); + final Runner runnerForTestWithParameters = super.createRunnerForTestWithParameters(test); + if (Objects.equals(ignoreTests, true)) { + if (runnerForTestWithParameters instanceof Filterable) { + try { + ((Filterable)runnerForTestWithParameters).filter(skipAll); + } catch (NoTestsRemainException ignored) { + } + } + }; + return runnerForTestWithParameters; + } + + @Retention(RetentionPolicy.RUNTIME) + @Target(ElementType.TYPE) + public @interface IgnoreParameter { + /** + * Optional index to specify where the boolean that is used to determine + * whether a ParameterizedTest run is skipped or not. + * + * Default value is -1, which uses the final index of the paremeter list. + * + * @return index to query for whether to ignore the test cases or not. + */ + int index() default -1; + } +} \ No newline at end of file diff --git a/src/test/java/org/janelia/saalfeldlab/n5/s3/N5AmazonS3MockTests.java b/src/test/java/org/janelia/saalfeldlab/n5/s3/N5AmazonS3MockTests.java new file mode 100644 index 0000000..9a5238f --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/s3/N5AmazonS3MockTests.java @@ -0,0 +1,27 @@ +package org.janelia.saalfeldlab.n5.s3; + +import com.amazonaws.services.s3.AmazonS3; +import org.janelia.saalfeldlab.n5.s3.mock.MockS3Factory; +import org.junit.Ignore; +import org.junit.Test; + +public class N5AmazonS3MockTests extends N5AmazonS3Tests { + + + static { + /* Should be no Erroneous Backend Failures with Mock Backend */ + skipErroneousBackendFailures = false; + } + + @Override + protected AmazonS3 getS3() { + + return MockS3Factory.getOrCreateS3(); + } + + @Test + @Ignore("Erroneous NoSuchBucket Skipped for Mock Tests") + @Override + public void testErroneousNoSuchBucketFailure() { + } +} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/s3/N5AmazonS3Tests.java b/src/test/java/org/janelia/saalfeldlab/n5/s3/N5AmazonS3Tests.java new file mode 100644 index 0000000..741c6d3 --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/s3/N5AmazonS3Tests.java @@ -0,0 +1,317 @@ +/*- + * #%L + * N5 AWS S3 + * %% + * Copyright (C) 2017 - 2022, Saalfeld Lab + * %% + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * + * 1. Redistributions of source code must retain the above copyright notice, + * this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + * #L% + */ +package org.janelia.saalfeldlab.n5.s3; + +import com.amazonaws.AmazonServiceException; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.model.AmazonS3Exception; +import com.google.gson.GsonBuilder; +import org.janelia.saalfeldlab.n5.AbstractN5Test; +import org.janelia.saalfeldlab.n5.KeyValueAccess; +import org.janelia.saalfeldlab.n5.N5Exception; +import org.janelia.saalfeldlab.n5.N5KeyValueReader; +import org.janelia.saalfeldlab.n5.N5KeyValueWriter; +import org.janelia.saalfeldlab.n5.N5Reader; +import org.janelia.saalfeldlab.n5.N5URI; +import org.janelia.saalfeldlab.n5.N5Writer; +import org.janelia.saalfeldlab.n5.s3.backend.BackendS3Factory; +import org.junit.Ignore; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestWatcher; +import org.junit.runner.Description; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.junit.runners.model.Statement; + +import java.net.URI; +import java.net.URISyntaxException; +import java.security.SecureRandom; +import java.util.Arrays; +import java.util.Collection; +import java.util.function.Supplier; + +import static org.janelia.saalfeldlab.n5.s3.AmazonS3Utils.getS3Bucket; +import static org.junit.Assume.assumeTrue; + +/** + * Base class for testing Amazon Web Services N5 implementation. + * Tests that are specific to AWS S3 can be added here. + * + * @author Igor Pisarev <pisarevi@janelia.hhmi.org> + */ +@RunWith(Parameterized.class) +public class N5AmazonS3Tests extends AbstractN5Test { + + public static class SkipErroneousNoSuchBucketFailure extends TestWatcher { + + private void assumeFailIfNoSuchBucket(Throwable exception) { + + if (exception.getCause() instanceof AmazonServiceException) + assumeFailIfNoSuchBucket(((AmazonServiceException)exception.getCause())); + } + + private void assumeFailIfNoSuchBucket(AmazonServiceException exception) { + + final int statusCode = exception.getStatusCode(); + final String errorCode = exception.getErrorCode(); + if (errorCode == null) { + throw exception; + } + assumeTrue("Erroneous NoSuchBucket Exception. Rerun to verify if this is a true test failure", statusCode != 404 && errorCode.equals("NoSuchBucket")); + } + + @Override + public Statement apply(final Statement base, final Description description) { + + try { + base.evaluate(); + } catch (N5Exception | AmazonServiceException exception) { + assumeFailIfNoSuchBucket(exception); + throw exception; + } catch (Throwable ignore) { + } + return base; + } + } + + public enum LocationInBucket { + ROOT(() -> "/", N5AmazonS3Tests::tempBucketName), + KEY(N5AmazonS3Tests::tempContainerPath, tempBucketName()::toString); + + public final Supplier getContainerPath; + private final Supplier getBucketName; + + LocationInBucket(Supplier tempContainerPath, Supplier tempBucketaName) { + + this.getContainerPath = tempContainerPath; + this.getBucketName = tempBucketaName; + } + + String getPath() { + + return getContainerPath.get(); + } + + String getBucketName() { + + return getBucketName.get(); + } + } + + public enum UseCache { + CACHE(true), + NO_CACHE(false); + + final boolean cache; + + UseCache(boolean cache) { + + this.cache = cache; + } + } + + @Parameterized.Parameters(name = "Container at {0}, {1}") + public static Collection data() { + + return Arrays.asList(new Object[][]{ + {LocationInBucket.ROOT, UseCache.NO_CACHE}, + {LocationInBucket.ROOT, UseCache.CACHE}, + {LocationInBucket.KEY, UseCache.NO_CACHE}, + {LocationInBucket.KEY, UseCache.CACHE} + }); + } + + private static final SecureRandom random = new SecureRandom(); + + protected static boolean skipErroneousBackendFailures = true; + + @Rule + public TestWatcher skipErroneousWatcher = null; + + @Parameterized.Parameter() + public LocationInBucket containerLocation; + + @Parameterized.Parameter(1) + public UseCache useCache; + + protected static AmazonS3 lateinitS3 = null; + + @Parameterized.AfterParam() + public static void removeTestBuckets() { + + for (LocationInBucket location : LocationInBucket.values()) { + final String bucketName = location.getBucketName(); + try { + final AmazonS3KeyValueAccess kva = new AmazonS3KeyValueAccess(lateinitS3, N5URI.encodeAsUri("s3://" + bucketName), false); + kva.delete(kva.normalize("/")); + } catch (Exception e) { + if (!lateinitS3.doesBucketExistV2(bucketName)) + continue; + System.err.println("Exception After Tests, Could Not Delete Test Bucket:" + bucketName); + e.printStackTrace(); + } + } + } + + private static String generateName(final String prefix, final String suffix) { + + return prefix + Long.toUnsignedString(random.nextLong()) + suffix; + } + + public static String tempBucketName() { + + return generateName("n5-test-", "-bucket"); + } + + public static String tempContainerPath() { + + return generateName("/n5-test-", ".n5"); + } + + { + if (skipErroneousBackendFailures) + skipErroneousWatcher = new SkipErroneousNoSuchBucketFailure(); + lateinitS3 = getS3(); + } + + protected AmazonS3 getS3() { + + return BackendS3Factory.getOrCreateS3(); + } + + @Override + protected String tempN5Location() throws URISyntaxException { + + final String containerPath = containerLocation.getPath(); + final String testBucket = containerLocation.getBucketName(); + return new URI("s3", testBucket, containerPath, null).toString(); + } + + @Override protected N5Writer createN5Writer() throws URISyntaxException { + + final String s3ContainerUri = tempN5Location(); + + return delayedBucketCreationWriter(s3ContainerUri, new GsonBuilder()); + } + + private N5KeyValueWriter delayedBucketCreationWriter(String s3ContainerUri, GsonBuilder gson) { + + final String bucketName = getS3Bucket(s3ContainerUri); + + final KeyValueAccess s3kva = new AmazonS3KeyValueAccess(getS3(), URI.create(s3ContainerUri), true); + + return new N5KeyValueWriter(s3kva, s3ContainerUri, gson, useCache.cache) { + + { + final URI containerUri; + try { + containerUri = s3kva.uri(""); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + final boolean localS3 = containerUri.getAuthority().contains("localhost"); + if (!localS3) { + /* Creating a bucket on S3 only provides a guarantee of eventual consistency. To + * ensure the bucket is created before testing, we wait to ensure it's visible before continuing. + * https://docs.aws.amazon.com/AmazonS3/latest/dev/Introduction.html#ConsistencyModel */ + int timeoutMs = 5 * 1000; + while (timeoutMs > 0) { + if (getS3().doesBucketExistV2(bucketName)) + break; + else + try { + Thread.sleep(100); + timeoutMs -= 100; + } catch (final InterruptedException e) { + e.printStackTrace(); + } + } + if (timeoutMs < 0) + throw new RuntimeException("Attempt to create bucket and wait for consistency failed."); + } + } + }; + } + + @Override + protected N5Writer createN5Writer(final String location, final GsonBuilder gson) { + + return delayedBucketCreationWriter(location, gson); + } + + @Override + protected N5Reader createN5Reader(final String location, final GsonBuilder gson) { + + final KeyValueAccess s3kva; + try { + s3kva = new AmazonS3KeyValueAccess(getS3(), new URI(location), false); + } catch (URISyntaxException e) { + throw new RuntimeException(e); + } + + return new N5KeyValueReader(s3kva, location, gson, useCache.cache); + } + + @Test + @Override + public void testWriterSeparation() { + + /* The base test will fail when `container is bucket root` parameter is true; Skip the test in that case. */ + assumeTrue("Writer Separation fails when container is at the bucket root, since the writers are at the same location", containerLocation != LocationInBucket.ROOT); + } + + @Test + @Ignore("This seems not to work as expected when run in maven specifically") + public void testErroneousNoSuchBucketFailure() { + + throw new AmazonS3Exception( + "This Exception should trigger a skipped test, not a failure", + new AmazonS3Exception("Erroneous NoSuchBucket Failure") { + + { + setErrorCode("NoSuchBucket"); + setStatusCode(404); + } + }); + + } +// public static void deleteAllTestBuckets() { +// +// final AmazonS3 s3 = AmazonS3ClientBuilder.defaultClient(); +// for (Bucket bucket : s3.listBuckets()) { +// final String bucketName = bucket.getName(); +// if (bucketName.startsWith("n5-test-")) { +// final String containerURI = "s3://" + bucketName; +// final N5KeyValueWriter writer = new N5KeyValueWriter(new AmazonS3KeyValueAccess(s3, containerURI, false), containerURI, new GsonBuilder(), false); +// writer.remove(); +// } +// } +// } +} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/s3/backend/BackendUriPathTests.java b/src/test/java/org/janelia/saalfeldlab/n5/s3/backend/BackendUriPathTests.java new file mode 100644 index 0000000..f70e5d2 --- /dev/null +++ b/src/test/java/org/janelia/saalfeldlab/n5/s3/backend/BackendUriPathTests.java @@ -0,0 +1,124 @@ +package org.janelia.saalfeldlab.n5.s3.backend; + +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.AnonymousAWSCredentials; +import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; +import org.janelia.saalfeldlab.n5.N5URI; +import org.janelia.saalfeldlab.n5.s3.AmazonS3KeyValueAccess; +import org.janelia.saalfeldlab.n5.s3.N5AmazonS3Tests; +import org.junit.After; +import org.junit.Assume; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Arrays; +import java.util.Collection; + +import static org.junit.Assert.assertEquals; + +@RunWith(Parameterized.class) +public class BackendUriPathTests { + + @Parameterized.Parameters(name = "path: \"{0}\"") + public static Collection data() throws URISyntaxException { + + return Arrays.asList(new Object[][]{ + {""}, + {"foo.zarr"}, + {"amazonaws.com.zarr"}, + }); + } + + @Parameterized.Parameter() + public String path; + + private String bucket; + private AmazonS3 s3; + + @After + public void removeTestBucket() { + + if (s3 != null && bucket != null && s3.doesBucketExistV2(bucket)) { + s3.deleteBucket(bucket); + } + s3 = null; + bucket = null; + } + + /** + * @param s3 to store in this instance for @After {@link #removeTestBucket()} + * @return the bucket name, which is stored internally for removal during @After {@link #removeTestBucket()} + */ + protected String getTempBucket(final AmazonS3 s3) { + + this.s3 = s3; + this.bucket = N5AmazonS3Tests.tempBucketName(); + return this.bucket; + } + + @Test + public void testS3URIs() throws URISyntaxException { + + s3 = AmazonS3ClientBuilder.standard().build(); + + final URI s3URI = N5URI.encodeAsUri("s3://" + getTempBucket(s3)); + final AmazonS3KeyValueAccess kvep = new AmazonS3KeyValueAccess(s3, s3URI, true); + check(kvep, s3URI, path); + } + + @Test + public void testS3URIsWithPathStyleAccess() throws URISyntaxException { + + // ensure we return s3:// uris even when the client uses an amazon client with path style access + s3 = AmazonS3ClientBuilder.standard() + .withPathStyleAccessEnabled(true) + .withEndpointConfiguration(new EndpointConfiguration("s3.amazonaws.com", "us-east-1")) + .build(); + + final URI s3URI = N5URI.encodeAsUri("s3://" + getTempBucket(s3)); + final AmazonS3KeyValueAccess kvep = new AmazonS3KeyValueAccess(s3, s3URI, true); + check(kvep, s3URI, path); + } + + @Test + public void testEMBLUriPaths() throws URISyntaxException { + + testPathAtPublicURI(N5URI.encodeAsUri("https://s3.embl.de/i2k-2020"), path); + } + + @Test + public void testEMIUriPaths() throws URISyntaxException { + + testPathAtPublicURI(N5URI.encodeAsUri("https://uk1s3.embassy.ebi.ac.uk/idr"), path); + } + + private static void testPathAtPublicURI(URI uri, String path) throws URISyntaxException { + + final AmazonS3 s3; + try { + s3 = AmazonS3ClientBuilder.standard() + .withEndpointConfiguration(new EndpointConfiguration(uri.getAuthority(), "")) + .withPathStyleAccessEnabled(true) + .withCredentials(new AWSStaticCredentialsProvider(new AnonymousAWSCredentials())) + .build(); + } catch (Exception e) { + Assume.assumeNoException(e); + throw e; + } + + final AmazonS3KeyValueAccess kva = new AmazonS3KeyValueAccess(s3, uri, false); + check(kva, uri, path); + } + + private static void check(final AmazonS3KeyValueAccess kv, final URI uri, final String path) throws URISyntaxException { + + final String expected = String.join("/", uri.toString(), path).replaceFirst("/$", ""); + final URI actual = kv.uri(path); + assertEquals(expected, actual.toString()); + } +} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/s3/backend/BackendUriTest.java b/src/test/java/org/janelia/saalfeldlab/n5/s3/backend/BackendUriTest.java deleted file mode 100644 index 0de6ea5..0000000 --- a/src/test/java/org/janelia/saalfeldlab/n5/s3/backend/BackendUriTest.java +++ /dev/null @@ -1,104 +0,0 @@ -package org.janelia.saalfeldlab.n5.s3.backend; - -import static org.junit.Assert.assertTrue; - -import java.net.URISyntaxException; - -import org.janelia.saalfeldlab.n5.s3.AmazonS3KeyValueAccess; -import org.junit.Test; - -import com.amazonaws.auth.AWSStaticCredentialsProvider; -import com.amazonaws.auth.AnonymousAWSCredentials; -import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration; -import com.amazonaws.services.s3.AmazonS3; -import com.amazonaws.services.s3.AmazonS3ClientBuilder; - -public class BackendUriTest { - - @Test - public void testS3Uris() throws URISyntaxException { - - final AmazonS3 s3 = AmazonS3ClientBuilder.standard() - .withCredentials(new AWSStaticCredentialsProvider(new AnonymousAWSCredentials())) - .build(); - - final String bucket = "demo-n5-zarr"; - String path = ""; - - // s3 style - final AmazonS3KeyValueAccess kv = new AmazonS3KeyValueAccess(s3, bucket, false); - assertTrue(kv.uri(path).toString(), check(path, kv, "s3:/", bucket, path)); - - path = "foo.zarr"; - assertTrue(kv.uri(path).toString(), check(path, kv, "s3:/", bucket, path)); - - path = "amazonaws.com.zarr"; - assertTrue(kv.uri(path).toString(), check(path, kv, "s3:/", bucket, path)); - - - // ensure we return s3:// uris even when the client uses an amazon client with path style access - final AmazonS3 s3ep = AmazonS3ClientBuilder.standard() - .withPathStyleAccessEnabled(true) - .withEndpointConfiguration(new EndpointConfiguration("s3.amazonaws.com", "us-east-1")) - .withCredentials(new AWSStaticCredentialsProvider(new AnonymousAWSCredentials())) - .build(); - - final AmazonS3KeyValueAccess kvep = new AmazonS3KeyValueAccess(s3ep, bucket, false); - assertTrue(kvep.uri(path).toString(), check(path, kvep, "s3:/", bucket, path)); - - path = "foo.zarr"; - assertTrue(kvep.uri(path).toString(), check(path, kvep, "s3:/", bucket, path)); - - path = "amazonaws.com.zarr"; - assertTrue(kvep.uri(path).toString(), check(path, kvep, "s3:/", bucket, path)); - } - - @Test - public void testEndpointUris() throws URISyntaxException { - - // embl endpoint - final String emblEndpoint = "s3.embl.de"; - final String emblBucket = "i2k-2020"; - String path = ""; - - final AmazonS3 emblS3 = AmazonS3ClientBuilder.standard() - .withEndpointConfiguration(new EndpointConfiguration(emblEndpoint, "")) - .withPathStyleAccessEnabled(true) - .withCredentials(new AWSStaticCredentialsProvider(new AnonymousAWSCredentials())) - .build(); - - final AmazonS3KeyValueAccess ekv = new AmazonS3KeyValueAccess(emblS3, emblBucket, false); - assertTrue(ekv.uri(path).toString(), check(path, ekv, "https:/", emblEndpoint, emblBucket, path)); - - path = "foo.zarr"; - assertTrue(ekv.uri(path).toString(), check(path, ekv, "https:/", emblEndpoint, emblBucket, path)); - - path = "amazonaws.com.zarr"; - assertTrue(ekv.uri(path).toString(), check(path, ekv, "https:/", emblEndpoint, emblBucket, path)); - - - // idr endpoint - final String idrEndpoint = "uk1s3.embassy.ebi.ac.uk"; - final String idrBucket = "idr"; - - final AmazonS3 idrs3 = AmazonS3ClientBuilder.standard() - .withEndpointConfiguration(new EndpointConfiguration(idrEndpoint, "")) - .withPathStyleAccessEnabled(true) - .withCredentials(new AWSStaticCredentialsProvider(new AnonymousAWSCredentials())) - .build(); - - final AmazonS3KeyValueAccess ikv = new AmazonS3KeyValueAccess(idrs3, idrBucket, false); - assertTrue(ikv.uri(path).toString(), check(path, ikv, "https:/", idrEndpoint, idrBucket, path)); - - path = "foo.zarr"; - assertTrue(ikv.uri(path).toString(), check(path, ikv, "https:/", idrEndpoint, idrBucket, path)); - - path = "amazonaws.com.zarr"; - assertTrue(ikv.uri(path).toString(), check(path, ikv, "https:/", idrEndpoint, idrBucket, path)); - } - - private static final boolean check(final String path, final AmazonS3KeyValueAccess kv, final String... components) throws URISyntaxException { - - return kv.uri(path).toString().startsWith(String.join("/", components)); - } -} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/s3/backend/CachedN5AmazonS3BucketRootBackendTest.java b/src/test/java/org/janelia/saalfeldlab/n5/s3/backend/CachedN5AmazonS3BucketRootBackendTest.java deleted file mode 100644 index ef1e8b5..0000000 --- a/src/test/java/org/janelia/saalfeldlab/n5/s3/backend/CachedN5AmazonS3BucketRootBackendTest.java +++ /dev/null @@ -1,68 +0,0 @@ -/*- - * #%L - * N5 AWS S3 - * %% - * Copyright (C) 2017 - 2022, Saalfeld Lab - * %% - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * #L% - */ -package org.janelia.saalfeldlab.n5.s3.backend; - -import java.io.IOException; -import java.net.URISyntaxException; - -import org.janelia.saalfeldlab.n5.N5Writer; -import org.janelia.saalfeldlab.n5.s3.AbstractN5AmazonS3BucketRootTest; - -import com.google.gson.GsonBuilder; - -/** - * Initiates testing of the Amazon Web Services S3-based N5 implementation using actual S3 backend. - * The test N5 container is created at the root of the new temporary bucket. - * - * @author Igor Pisarev <pisarevi@janelia.hhmi.org> - */ -public class CachedN5AmazonS3BucketRootBackendTest extends AbstractN5AmazonS3BucketRootTest { - - public CachedN5AmazonS3BucketRootBackendTest() { - - super(BackendS3Factory.getOrCreateS3()); - } - - @Override - protected N5Writer createN5Writer(final String location, final GsonBuilder gson) throws IOException, URISyntaxException { - - final String bucketName = getS3Bucket(location); - N5AmazonS3DelayedWriter.sleep(); - return new N5AmazonS3DelayedWriter(s3, bucketName, gson, true) { - - @Override public void close() { - - remove(); - super.close(); - } - - }; - } - -} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/s3/backend/CachedN5AmazonS3ContainerPathBackendTest.java b/src/test/java/org/janelia/saalfeldlab/n5/s3/backend/CachedN5AmazonS3ContainerPathBackendTest.java deleted file mode 100644 index 00aee7a..0000000 --- a/src/test/java/org/janelia/saalfeldlab/n5/s3/backend/CachedN5AmazonS3ContainerPathBackendTest.java +++ /dev/null @@ -1,68 +0,0 @@ -/*- - * #%L - * N5 AWS S3 - * %% - * Copyright (C) 2017 - 2022, Saalfeld Lab - * %% - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * #L% - */ -package org.janelia.saalfeldlab.n5.s3.backend; - -import java.io.IOException; -import java.net.URISyntaxException; - -import org.janelia.saalfeldlab.n5.N5Writer; -import org.janelia.saalfeldlab.n5.s3.AbstractN5AmazonS3ContainerPathTest; - -import com.google.gson.GsonBuilder; - -/** - * Initiates testing of the Amazon Web Services S3-based N5 implementation using actual S3 backend. - * A non-trivial container path is used to create the test N5 container in the temporary bucket. - * - * @author Igor Pisarev <pisarevi@janelia.hhmi.org> - */ -public class CachedN5AmazonS3ContainerPathBackendTest extends AbstractN5AmazonS3ContainerPathTest { - - public CachedN5AmazonS3ContainerPathBackendTest() { - - super(BackendS3Factory.getOrCreateS3()); - } - - @Override - protected N5Writer createN5Writer(final String location, final GsonBuilder gson) throws IOException, URISyntaxException { - - final String bucketName = getS3Bucket(location); - final String basePath = getS3Key(location); - return new N5AmazonS3DelayedWriter(s3, bucketName, basePath, gson, true) { - - @Override public void close() { - - remove(); - super.close(); - } - - }; - } - -} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/s3/backend/N5AmazonS3BucketRootBackendTest.java b/src/test/java/org/janelia/saalfeldlab/n5/s3/backend/N5AmazonS3BucketRootBackendTest.java deleted file mode 100644 index d282be1..0000000 --- a/src/test/java/org/janelia/saalfeldlab/n5/s3/backend/N5AmazonS3BucketRootBackendTest.java +++ /dev/null @@ -1,60 +0,0 @@ -/*- - * #%L - * N5 AWS S3 - * %% - * Copyright (C) 2017 - 2022, Saalfeld Lab - * %% - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * #L% - */ -package org.janelia.saalfeldlab.n5.s3.backend; - -import java.io.IOException; -import java.net.URISyntaxException; - -import org.janelia.saalfeldlab.n5.N5Writer; -import org.janelia.saalfeldlab.n5.s3.AbstractN5AmazonS3BucketRootTest; - -import com.google.gson.GsonBuilder; - -/** - * Initiates testing of the Amazon Web Services S3-based N5 implementation using actual S3 backend. - * The test N5 container is created at the root of the new temporary bucket. - * - * @author Igor Pisarev <pisarevi@janelia.hhmi.org> - */ -public class N5AmazonS3BucketRootBackendTest extends AbstractN5AmazonS3BucketRootTest { - - public N5AmazonS3BucketRootBackendTest() { - - super(BackendS3Factory.getOrCreateS3()); - } - - @Override - protected N5Writer createN5Writer(final String location, final GsonBuilder gson) throws IOException, URISyntaxException { - - final String bucketName = getS3Bucket(location); - N5AmazonS3DelayedWriter.sleep(); - return new N5AmazonS3DelayedWriter(s3, bucketName, gson, false); - } - -} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/s3/backend/N5AmazonS3ContainerPathBackendTest.java b/src/test/java/org/janelia/saalfeldlab/n5/s3/backend/N5AmazonS3ContainerPathBackendTest.java deleted file mode 100644 index f192f66..0000000 --- a/src/test/java/org/janelia/saalfeldlab/n5/s3/backend/N5AmazonS3ContainerPathBackendTest.java +++ /dev/null @@ -1,60 +0,0 @@ -/*- - * #%L - * N5 AWS S3 - * %% - * Copyright (C) 2017 - 2022, Saalfeld Lab - * %% - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * #L% - */ -package org.janelia.saalfeldlab.n5.s3.backend; - -import java.io.IOException; -import java.net.URISyntaxException; - -import org.janelia.saalfeldlab.n5.N5Writer; -import org.janelia.saalfeldlab.n5.s3.AbstractN5AmazonS3ContainerPathTest; - -import com.google.gson.GsonBuilder; - -/** - * Initiates testing of the Amazon Web Services S3-based N5 implementation using actual S3 backend. - * A non-trivial container path is used to create the test N5 container in the temporary bucket. - * - * @author Igor Pisarev <pisarevi@janelia.hhmi.org> - */ -public class N5AmazonS3ContainerPathBackendTest extends AbstractN5AmazonS3ContainerPathTest { - - public N5AmazonS3ContainerPathBackendTest() { - - super(BackendS3Factory.getOrCreateS3()); - } - - @Override - protected N5Writer createN5Writer(final String location, final GsonBuilder gson) throws IOException, URISyntaxException { - - final String bucketName = getS3Bucket(location); - final String basePath = getS3Key(location); - return new N5AmazonS3DelayedWriter(s3, bucketName, basePath, gson, false); - } - -} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/s3/backend/N5AmazonS3DelayedWriter.java b/src/test/java/org/janelia/saalfeldlab/n5/s3/backend/N5AmazonS3DelayedWriter.java deleted file mode 100644 index cc1e6fc..0000000 --- a/src/test/java/org/janelia/saalfeldlab/n5/s3/backend/N5AmazonS3DelayedWriter.java +++ /dev/null @@ -1,144 +0,0 @@ -/*- - * #%L - * N5 AWS S3 - * %% - * Copyright (C) 2017 - 2022, Saalfeld Lab - * %% - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * #L% - */ -package org.janelia.saalfeldlab.n5.s3.backend; - -import java.io.IOException; -import java.util.Map; - -import org.janelia.saalfeldlab.n5.AbstractN5Test; -import org.janelia.saalfeldlab.n5.DataBlock; -import org.janelia.saalfeldlab.n5.DatasetAttributes; -import org.janelia.saalfeldlab.n5.s3.N5AmazonS3Writer; - -import com.amazonaws.services.s3.AmazonS3; -import com.google.gson.GsonBuilder; - -/** - * Helper class for dealing with eventual consistency of S3 store. - * - * S3 store has a concept of eventual consistency: for example, when an object - * is requested after it has been overwritten, - * it may still return the old version of the object, but eventually it will - * return the updated version. - * - * The tests in {@link AbstractN5Test} write and then immediately read - * attributes and data blocks to verify them. - * Eventual consistency makes some of these tests fail. To solve or at least - * minimize this effect, - * this class adds a 1s delay after each modification request. - * - * @see https://docs.aws.amazon.com/AmazonS3/latest/dev/Introduction.html#ConsistencyModel - */ -class N5AmazonS3DelayedWriter extends N5AmazonS3Writer { - - private static final long delayMsec = 1000; - - public N5AmazonS3DelayedWriter( - final AmazonS3 s3, - final String bucketName, - final GsonBuilder gson, - final boolean cacheAttributes) - throws IOException { - - super(s3, bucketName, gson, cacheAttributes); - sleep(); - } - - public N5AmazonS3DelayedWriter( - final AmazonS3 s3, - final String bucketName, - final String basePath, - final GsonBuilder gson, - final boolean cacheAttributes) - throws IOException { - - super(s3, bucketName, basePath, gson, cacheAttributes); - sleep(); - } - - @Override - public void createGroup(final String pathName) { - - super.createGroup(pathName); - sleep(); - } - - @Override - public void setAttributes( - final String pathName, - final Map attributes) { - - super.setAttributes(pathName, attributes); - sleep(); - } - - @Override - public void writeBlock( - final String pathName, - final DatasetAttributes datasetAttributes, - final DataBlock dataBlock) { - - super.writeBlock(pathName, datasetAttributes, dataBlock); - sleep(); - } - - @Override - public boolean deleteBlock(final String pathName, final long... gridPosition) { - - final boolean ret = super.deleteBlock(pathName, gridPosition); - sleep(); - return ret; - } - - @Override - public boolean remove() { - - final boolean ret = super.remove(); - sleep(); - return ret; - } - - @Override - public boolean remove(final String pathName) { - - final boolean ret = super.remove(pathName); - sleep(); - return ret; - } - - static void sleep() { - - try { - Thread.sleep(delayMsec); - } catch (final InterruptedException e) { - e.printStackTrace(); - } - } -} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/s3/mock/CachedN5AmazonS3BucketRootMockTest.java b/src/test/java/org/janelia/saalfeldlab/n5/s3/mock/CachedN5AmazonS3BucketRootMockTest.java deleted file mode 100644 index 7bd2271..0000000 --- a/src/test/java/org/janelia/saalfeldlab/n5/s3/mock/CachedN5AmazonS3BucketRootMockTest.java +++ /dev/null @@ -1,61 +0,0 @@ -/*- - * #%L - * N5 AWS S3 - * %% - * Copyright (C) 2017 - 2022, Saalfeld Lab - * %% - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * #L% - */ -package org.janelia.saalfeldlab.n5.s3.mock; - -import java.io.IOException; -import java.net.URISyntaxException; - -import org.janelia.saalfeldlab.n5.N5Writer; -import org.janelia.saalfeldlab.n5.s3.AbstractN5AmazonS3BucketRootTest; -import org.janelia.saalfeldlab.n5.s3.N5AmazonS3Writer; - -import com.google.gson.GsonBuilder; - -/** - * Initiates testing of the Amazon Web Services S3-based N5 implementation using - * S3 mock library. - * The test N5 container is created at the root of the new temporary bucket. - * - * @author Igor Pisarev <pisarevi@janelia.hhmi.org> - */ -public class CachedN5AmazonS3BucketRootMockTest extends AbstractN5AmazonS3BucketRootTest { - - public CachedN5AmazonS3BucketRootMockTest() { - - super(MockS3Factory.getOrCreateS3()); - } - - @Override - protected N5Writer createN5Writer(final String location, final GsonBuilder gson) throws IOException, URISyntaxException { - - final String bucketName = getS3Bucket(location); - final String basePath = getS3Key(location); - return new N5AmazonS3Writer(s3, bucketName, basePath, gson, true); - } -} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/s3/mock/CachedN5AmazonS3ContainerPathMockTest.java b/src/test/java/org/janelia/saalfeldlab/n5/s3/mock/CachedN5AmazonS3ContainerPathMockTest.java deleted file mode 100644 index cd9090a..0000000 --- a/src/test/java/org/janelia/saalfeldlab/n5/s3/mock/CachedN5AmazonS3ContainerPathMockTest.java +++ /dev/null @@ -1,60 +0,0 @@ -/*- - * #%L - * N5 AWS S3 - * %% - * Copyright (C) 2017 - 2022, Saalfeld Lab - * %% - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * #L% - */ -package org.janelia.saalfeldlab.n5.s3.mock; - -import java.io.IOException; -import java.net.URISyntaxException; - -import org.janelia.saalfeldlab.n5.N5Writer; -import org.janelia.saalfeldlab.n5.s3.AbstractN5AmazonS3ContainerPathTest; -import org.janelia.saalfeldlab.n5.s3.N5AmazonS3Writer; - -import com.google.gson.GsonBuilder; - -/** - * Initiates testing of the Amazon Web Services S3-based N5 implementation using S3 mock library. - * A non-trivial container path is used to create the test N5 container in the temporary bucket. - * - * @author Igor Pisarev <pisarevi@janelia.hhmi.org> - */ -public class CachedN5AmazonS3ContainerPathMockTest extends AbstractN5AmazonS3ContainerPathTest { - - public CachedN5AmazonS3ContainerPathMockTest() { - - super(MockS3Factory.getOrCreateS3()); - } - - @Override - protected N5Writer createN5Writer(final String location, final GsonBuilder gson) throws IOException, URISyntaxException { - - final String bucketName = getS3Bucket(location); - final String basePath = getS3Key(location); - return new N5AmazonS3Writer(s3, bucketName, basePath, gson, true); - } -} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/s3/mock/MockUriTest.java b/src/test/java/org/janelia/saalfeldlab/n5/s3/mock/MockUriTest.java deleted file mode 100644 index 9b22955..0000000 --- a/src/test/java/org/janelia/saalfeldlab/n5/s3/mock/MockUriTest.java +++ /dev/null @@ -1,30 +0,0 @@ -package org.janelia.saalfeldlab.n5.s3.mock; - -import static org.junit.Assert.assertTrue; - -import java.net.URISyntaxException; - -import org.janelia.saalfeldlab.n5.s3.AmazonS3KeyValueAccess; -import org.junit.Test; - -import com.amazonaws.services.s3.AmazonS3; - -public class MockUriTest { - - @Test - public void testS3Uris() throws URISyntaxException { - - // dummy client - final AmazonS3 s3 = MockS3Factory.getOrCreateS3(); - final String bucket = "zarr-n5-demo"; - String path = ""; - - s3.createBucket(bucket); - final AmazonS3KeyValueAccess kv = new AmazonS3KeyValueAccess(s3, bucket, false); - - assertTrue(kv.uri(path).toString().startsWith("http://localhost:8001/" + bucket + "/" + path)); - - path = "foo.zarr"; - assertTrue(kv.uri(path).toString().startsWith("http://localhost:8001/" + bucket + "/" + path)); - } -} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/s3/mock/N5AmazonS3BucketRootMockTest.java b/src/test/java/org/janelia/saalfeldlab/n5/s3/mock/N5AmazonS3BucketRootMockTest.java deleted file mode 100644 index a0e9b8a..0000000 --- a/src/test/java/org/janelia/saalfeldlab/n5/s3/mock/N5AmazonS3BucketRootMockTest.java +++ /dev/null @@ -1,46 +0,0 @@ -/*- - * #%L - * N5 AWS S3 - * %% - * Copyright (C) 2017 - 2022, Saalfeld Lab - * %% - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * #L% - */ -package org.janelia.saalfeldlab.n5.s3.mock; - -import org.janelia.saalfeldlab.n5.s3.AbstractN5AmazonS3BucketRootTest; - -/** - * Initiates testing of the Amazon Web Services S3-based N5 implementation using - * S3 mock library. - * The test N5 container is created at the root of the new temporary bucket. - * - * @author Igor Pisarev <pisarevi@janelia.hhmi.org> - */ -public class N5AmazonS3BucketRootMockTest extends AbstractN5AmazonS3BucketRootTest { - - public N5AmazonS3BucketRootMockTest() { - - super(MockS3Factory.getOrCreateS3()); - } -} diff --git a/src/test/java/org/janelia/saalfeldlab/n5/s3/mock/N5AmazonS3ContainerPathMockTest.java b/src/test/java/org/janelia/saalfeldlab/n5/s3/mock/N5AmazonS3ContainerPathMockTest.java deleted file mode 100644 index dea0e56..0000000 --- a/src/test/java/org/janelia/saalfeldlab/n5/s3/mock/N5AmazonS3ContainerPathMockTest.java +++ /dev/null @@ -1,45 +0,0 @@ -/*- - * #%L - * N5 AWS S3 - * %% - * Copyright (C) 2017 - 2022, Saalfeld Lab - * %% - * Redistribution and use in source and binary forms, with or without - * modification, are permitted provided that the following conditions are met: - * - * 1. Redistributions of source code must retain the above copyright notice, - * this list of conditions and the following disclaimer. - * 2. Redistributions in binary form must reproduce the above copyright notice, - * this list of conditions and the following disclaimer in the documentation - * and/or other materials provided with the distribution. - * - * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" - * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE - * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR - * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF - * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS - * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN - * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) - * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE - * POSSIBILITY OF SUCH DAMAGE. - * #L% - */ -package org.janelia.saalfeldlab.n5.s3.mock; - -import org.janelia.saalfeldlab.n5.s3.AbstractN5AmazonS3ContainerPathTest; - -/** - * Initiates testing of the Amazon Web Services S3-based N5 implementation using S3 mock library. - * A non-trivial container path is used to create the test N5 container in the temporary bucket. - * - * @author Igor Pisarev <pisarevi@janelia.hhmi.org> - */ -public class N5AmazonS3ContainerPathMockTest extends AbstractN5AmazonS3ContainerPathTest { - - public N5AmazonS3ContainerPathMockTest() { - - super(MockS3Factory.getOrCreateS3()); - } -} diff --git a/src/test/resources/simplelogger.properties b/src/test/resources/simplelogger.properties new file mode 100644 index 0000000..e88f5fe --- /dev/null +++ b/src/test/resources/simplelogger.properties @@ -0,0 +1,27 @@ +# SLF4J's SimpleLogger configuration file +# Simple implementation of Logger that sends all enabled log messages, for all defined loggers, to System.err. +# Default logging detail level for all instances of SimpleLogger. +# Must be one of ("trace", "debug", "info", "warn", or "error"). +# If not specified, defaults to "info". +org.slf4j.simpleLogger.defaultLogLevel=warn +# Logging detail level for a SimpleLogger instance named "xxxxx". +# Must be one of ("trace", "debug", "info", "warn", or "error"). +# If not specified, the default logging detail level is used. +#org.slf4j.simpleLogger.log.xxxxx=trace +# Set to true if you want the current date and time to be included in output messages. +# Default is false, and will output the number of milliseconds elapsed since startup. +#org.slf4j.simpleLogger.showDateTime=false +# The date and time format to be used in the output messages. +# The pattern describing the date and time format is the same that is used in java.text.SimpleDateFormat. +# If the format is not specified or is invalid, the default format is used. +# The default format is yyyy-MM-dd HH:mm:ss:SSS Z. +#org.slf4j.simpleLogger.dateTimeFormat=yyyy-MM-dd HH:mm:ss:SSS Z +# Set to true if you want to output the current thread name. +# Defaults to true. +#org.slf4j.simpleLogger.showThreadName=true +# Set to true if you want the Logger instance name to be included in output messages. +# Defaults to true. +#org.slf4j.simpleLogger.showLogName=true +# Set to true if you want the last component of the name to be included in output messages. +# Defaults to false. +org.slf4j.simpleLogger.showShortLogName=false \ No newline at end of file