AwsS3Test.java

/*
 * Copyright 2013-2015 EMC Corporation. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License").
 * You may not use this file except in compliance with the License.
 * A copy of the License is located at
 *
 * http://www.apache.org/licenses/LICENSE-2.0.txt
 *
 * or in the "license" file accompanying this file. This file is distributed
 * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
 * express or implied. See the License for the specific language governing
 * permissions and limitations under the License.
 */
package com.emc.ecs.sync.storage;

import com.amazonaws.AmazonServiceException;
import com.amazonaws.ClientConfiguration;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.model.*;
import com.emc.ecs.sync.EcsSync;
import com.emc.ecs.sync.config.Protocol;
import com.emc.ecs.sync.config.SyncConfig;
import com.emc.ecs.sync.config.SyncOptions;
import com.emc.ecs.sync.config.storage.AwsS3Config;
import com.emc.ecs.sync.model.ObjectMetadata;
import com.emc.ecs.sync.model.SyncObject;
import com.emc.ecs.sync.storage.file.AbstractFilesystemStorage;
import com.emc.ecs.sync.storage.s3.AwsS3Storage;
import com.emc.ecs.sync.test.TestConfig;
import com.emc.ecs.sync.util.RandomInputStream;
import com.emc.object.util.ChecksumAlgorithm;
import com.emc.object.util.ChecksummedInputStream;
import com.emc.object.util.RunningChecksum;
import com.emc.rest.util.StreamUtil;
import org.junit.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.*;
import java.net.URI;
import java.util.*;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;

public class AwsS3Test {
    private static final Logger log = LoggerFactory.getLogger(AwsS3Test.class);

    private String endpoint;
    private URI endpointUri;
    private String accessKey;
    private String secretKey;
    private AmazonS3 s3;

    @Before
    public void setup() throws Exception {
        Properties syncProperties = TestConfig.getProperties();
        endpoint = syncProperties.getProperty(TestConfig.PROP_S3_ENDPOINT);
        accessKey = syncProperties.getProperty(TestConfig.PROP_S3_ACCESS_KEY_ID);
        secretKey = syncProperties.getProperty(TestConfig.PROP_S3_SECRET_KEY);
        String proxyUri = syncProperties.getProperty(TestConfig.PROP_HTTP_PROXY_URI);
        Assume.assumeNotNull(endpoint, accessKey, secretKey);
        endpointUri = new URI(endpoint);

        ClientConfiguration config = new ClientConfiguration().withSignerOverride("S3SignerType");
        if (proxyUri != null) {
            URI uri = new URI(proxyUri);
            config.setProxyHost(uri.getHost());
            config.setProxyPort(uri.getPort());
        }

        s3 = new AmazonS3Client(new BasicAWSCredentials(accessKey, secretKey), config);
        s3.setEndpoint(endpoint);
    }

    @Test
    public void testVersions() throws Exception {
        String bucket1 = "ecs-sync-s3-test-versions";
        String bucket2 = "ecs-sync-s3-test-versions-2";

        log.info("creating buckets with versioning enabled...");
        createBucket(bucket1, true);
        createBucket(bucket2, true);

        try {
            SyncOptions options = new SyncOptions().withThreadCount(32).withRetryAttempts(0).withForceSync(true).withVerify(true);

            // test data
            com.emc.ecs.sync.config.storage.TestConfig testConfig = new com.emc.ecs.sync.config.storage.TestConfig();
            testConfig.withDiscardData(false).withReadData(true).withObjectCount(50).withMaxSize(10 * 1024);

            AwsS3Config s3Config1 = new AwsS3Config();
            if (endpointUri.getScheme() != null)
                s3Config1.setProtocol(Protocol.valueOf(endpointUri.getScheme().toLowerCase()));
            s3Config1.setHost(endpointUri.getHost());
            s3Config1.setPort(endpointUri.getPort());
            s3Config1.setAccessKey(accessKey);
            s3Config1.setSecretKey(secretKey);
            s3Config1.setLegacySignatures(true);
            s3Config1.setDisableVHosts(true);
            s3Config1.setBucketName(bucket1);
            s3Config1.setPreserveDirectories(true);

            SyncConfig syncConfig = new SyncConfig().withOptions(options);
            syncConfig.setTarget(s3Config1);

            // push it into bucket1
            log.info("writing v1 source data...");
            TestStorage testSource = new TestStorage();
            testSource.withConfig(testConfig).withOptions(options);
            runSync(syncConfig, testSource);

            // 2nd version is delete
            log.info("deleting source objects (for v2)...");
            deleteObjects(bucket1);

            // 3rd version is altered
            log.info("writing v3 source data...");
            TestStorage testSource2 = new TestStorage();
            testSource2.withConfig(testConfig).withOptions(options);
            testSource2.ingest(testSource, null);
            alterContent(testSource2, null, "3");
            runSync(syncConfig, testSource2);
            testSource = testSource2;

            // 4th version is altered again
            log.info("writing v4 source data...");
            testSource2 = new TestStorage();
            testSource2.withConfig(testConfig).withOptions(options);
            testSource2.ingest(testSource, null);
            alterContent(testSource2, null, "4");
            runSync(syncConfig, testSource2);

            // now run migration to bucket2
            s3Config1.setIncludeVersions(true);
            options.withForceSync(false);
            options.setSyncAcl(true);

            AwsS3Config s3Config2 = new AwsS3Config();
            if (endpointUri.getScheme() != null)
                s3Config2.setProtocol(Protocol.valueOf(endpointUri.getScheme().toLowerCase()));
            s3Config2.setHost(endpointUri.getHost());
            s3Config2.setPort(endpointUri.getPort());
            s3Config2.setAccessKey(accessKey);
            s3Config2.setSecretKey(secretKey);
            s3Config2.setLegacySignatures(true);
            s3Config2.setDisableVHosts(true);
            s3Config2.setBucketName(bucket2);
            s3Config2.setCreateBucket(true);
            s3Config2.setIncludeVersions(true);
            s3Config2.setPreserveDirectories(true);

            syncConfig.setSource(s3Config1);
            syncConfig.setTarget(s3Config2);

            log.info("migrating versions to bucket2...");
            runSync(syncConfig, null);

            // verify all versions
            log.info("verifying versions...");
            verifyBuckets(bucket1, bucket2);

            // test verify only sync
            log.info("syncing with verify-only...");
            options.withVerify(false).withVerifyOnly(true);
            runSync(syncConfig, null);

            // add v5 (delete) to bucket1 (testing delete)
            log.info("deleting objects in source (for v5)...");
            deleteObjects(bucket1);

            // test deleted objects
            log.info("migrating v5 to target...");
            options.withVerifyOnly(false).withVerify(true);
            runSync(syncConfig, null);
            log.info("verifying versions...");
            verifyBuckets(bucket1, bucket2);

            // test deleted objects from scratch
            log.info("wiping bucket2...");
            deleteVersionedBucket(bucket2);
            log.info("migrating versions again from scratch...");
            runSync(syncConfig, null);
            log.info("verifying versions...");
            verifyBuckets(bucket1, bucket2);

            // test verify-only with deleted objects
            log.info("syncing with verify-only (all delete markers)...");
            options.withVerify(false).withVerifyOnly(true);
            runSync(syncConfig, null);

        } finally {
            log.info("cleaning up bucket1...");
            deleteVersionedBucket(bucket1);
            log.info("cleaning up bucket2...");
            deleteVersionedBucket(bucket2);
        }
    }

    @Test
    public void testSetAcl() throws Exception {
        String bucket = "ecs-sync-s3-test-acl";
        String key = "test-object";
        createBucket(bucket, true);

        try {
            String content = "hello ACLs";

            s3.putObject(bucket, key, new ByteArrayInputStream(content.getBytes()), null); // 1st version

            AccessControlList acl = new AccessControlList();
            acl.setOwner(new Owner(accessKey, accessKey));
            acl.grantPermission(new CanonicalGrantee(accessKey), Permission.FullControl);
            acl.grantPermission(GroupGrantee.AuthenticatedUsers, Permission.Read);
            acl.grantPermission(GroupGrantee.AuthenticatedUsers, Permission.Write);
            acl.grantPermission(GroupGrantee.AllUsers, Permission.Read);

            PutObjectRequest putRequest = new PutObjectRequest(bucket, key, new ByteArrayInputStream(content.getBytes()), null);
            putRequest.setAccessControlList(acl);

            s3.putObject(putRequest); // 2nd version

            AccessControlList remoteAcl = s3.getObjectAcl(bucket, key);

            verifyAcls(acl, remoteAcl);
        } finally {
            try {
                deleteVersionedBucket(bucket);
            } catch (Throwable t) {
                log.warn("could not delete bucket: " + t.getMessage());
            }
        }
    }

    @Test
    public void testSyncVersionsWithAcls() throws Exception {
        String bucket1 = "ecs-sync-s3-test-sync-acl1";
        String bucket2 = "ecs-sync-s3-test-sync-acl2";
        createBucket(bucket1, true);

        String key1 = "key1", key2 = "key2", key3 = "key3";

        AccessControlList largeAcl = new AccessControlList();
        largeAcl.setOwner(new Owner(accessKey, accessKey));
        largeAcl.grantPermission(new CanonicalGrantee(accessKey), Permission.FullControl);
        largeAcl.grantPermission(GroupGrantee.AuthenticatedUsers, Permission.Read);
        largeAcl.grantPermission(GroupGrantee.AuthenticatedUsers, Permission.Write);
        largeAcl.grantPermission(GroupGrantee.AllUsers, Permission.Read);

        AccessControlList midAcl = new AccessControlList();
        midAcl.setOwner(new Owner(accessKey, accessKey));
        midAcl.grantPermission(new CanonicalGrantee(accessKey), Permission.FullControl);
        midAcl.grantPermission(GroupGrantee.AuthenticatedUsers, Permission.Read);

        AccessControlList defaultAcl = new AccessControlList();
        defaultAcl.setOwner(new Owner(accessKey, accessKey));
        defaultAcl.grantPermission(new CanonicalGrantee(accessKey), Permission.FullControl);

        try {
            // default acls
            s3.putObject(bucket1, key1, new ByteArrayInputStream("data1".getBytes()), null);
            s3.putObject(bucket1, key1, new ByteArrayInputStream("data1".getBytes()), null);
            s3.putObject(bucket1, key1, new ByteArrayInputStream("data1".getBytes()), null);

            // default acl on latest
            PutObjectRequest request = new PutObjectRequest(bucket1, key2, new ByteArrayInputStream("data2".getBytes()), null);
            request.setAccessControlList(largeAcl);
            s3.putObject(request);
            request = new PutObjectRequest(bucket1, key2, new ByteArrayInputStream("data2".getBytes()), null);
            request.setAccessControlList(midAcl);
            s3.putObject(request);
            s3.putObject(bucket1, key2, new ByteArrayInputStream("data2".getBytes()), null);

            // default acl on first version
            s3.putObject(bucket1, key3, new ByteArrayInputStream("data3".getBytes()), null);
            request = new PutObjectRequest(bucket1, key3, new ByteArrayInputStream("data3".getBytes()), null);
            request.setAccessControlList(midAcl);
            s3.putObject(request);
            request = new PutObjectRequest(bucket1, key3, new ByteArrayInputStream("data3".getBytes()), null);
            request.setAccessControlList(largeAcl);
            s3.putObject(request);

            AwsS3Config sourceConfig = new AwsS3Config();
            if (endpointUri.getScheme() != null)
                sourceConfig.setProtocol(Protocol.valueOf(endpointUri.getScheme().toLowerCase()));
            sourceConfig.setHost(endpointUri.getHost());
            sourceConfig.setPort(endpointUri.getPort());
            sourceConfig.setAccessKey(accessKey);
            sourceConfig.setSecretKey(secretKey);
            sourceConfig.setLegacySignatures(true);
            sourceConfig.setDisableVHosts(true);
            sourceConfig.setBucketName(bucket1);
            sourceConfig.setIncludeVersions(true);

            AwsS3Config targetConfig = new AwsS3Config();
            if (endpointUri.getScheme() != null)
                targetConfig.setProtocol(Protocol.valueOf(endpointUri.getScheme().toLowerCase()));
            targetConfig.setHost(endpointUri.getHost());
            targetConfig.setPort(endpointUri.getPort());
            targetConfig.setAccessKey(accessKey);
            targetConfig.setSecretKey(secretKey);
            targetConfig.setLegacySignatures(true);
            targetConfig.setDisableVHosts(true);
            targetConfig.setBucketName(bucket2);
            targetConfig.setCreateBucket(true);
            targetConfig.setIncludeVersions(true);

            SyncOptions options = new SyncOptions().withThreadCount(1).withVerify(true).withSyncAcl(true);

            SyncConfig syncConfig = new SyncConfig().withOptions(options).withSource(sourceConfig).withTarget(targetConfig);

            EcsSync sync = new EcsSync();
            sync.setSyncConfig(syncConfig);
            sync.run();

            Assert.assertEquals(0, sync.getStats().getObjectsFailed());

            List<S3VersionSummary> key1Versions = getVersions(bucket2, key1);
            verifyAcls(defaultAcl, s3.getObjectAcl(bucket2, key1, key1Versions.get(0).getVersionId()));
            verifyAcls(defaultAcl, s3.getObjectAcl(bucket2, key1, key1Versions.get(1).getVersionId()));
            verifyAcls(defaultAcl, s3.getObjectAcl(bucket2, key1, key1Versions.get(2).getVersionId()));

            List<S3VersionSummary> key2Versions = getVersions(bucket2, key2);
            verifyAcls(largeAcl, s3.getObjectAcl(bucket2, key2, key2Versions.get(0).getVersionId()));
            verifyAcls(midAcl, s3.getObjectAcl(bucket2, key2, key2Versions.get(1).getVersionId()));
            verifyAcls(defaultAcl, s3.getObjectAcl(bucket2, key2, key2Versions.get(2).getVersionId()));

            List<S3VersionSummary> key3Versions = getVersions(bucket2, key3);
            verifyAcls(defaultAcl, s3.getObjectAcl(bucket2, key3, key3Versions.get(0).getVersionId()));
            verifyAcls(midAcl, s3.getObjectAcl(bucket2, key3, key3Versions.get(1).getVersionId()));
            verifyAcls(largeAcl, s3.getObjectAcl(bucket2, key3, key3Versions.get(2).getVersionId()));
        } finally {
            deleteVersionedBucket(bucket1);
            deleteVersionedBucket(bucket2);
        }
    }

    @Test
    public void testNormalUpload() throws Exception {
        String bucketName = "ecs-sync-s3-target-test-bucket";
        createBucket(bucketName, false);

        TestStorage source = new TestStorage();
        source.withConfig(new com.emc.ecs.sync.config.storage.TestConfig()).withOptions(new SyncOptions());
        AwsS3Storage s3Target = null;
        try {
            String key = "normal-upload";
            long size = 512 * 1024; // 512KiB
            InputStream stream = new RandomInputStream(size);
            SyncObject object = new SyncObject(source, key, new ObjectMetadata().withContentLength(size), stream, null);

            AwsS3Config s3Config = new AwsS3Config();
            if (endpointUri.getScheme() != null)
                s3Config.setProtocol(Protocol.valueOf(endpointUri.getScheme().toLowerCase()));
            s3Config.setHost(endpointUri.getHost());
            s3Config.setPort(endpointUri.getPort());
            s3Config.setAccessKey(accessKey);
            s3Config.setSecretKey(secretKey);
            s3Config.setLegacySignatures(true);
            s3Config.setDisableVHosts(true);
            s3Config.setBucketName(bucketName);

            s3Target = new AwsS3Storage();
            s3Target.withConfig(s3Config).withOptions(new SyncOptions());
            s3Target.configure(source, null, s3Target);

            String createdKey = s3Target.createObject(object);

            Assert.assertEquals(key, createdKey);

            // verify bytes read from source
            // first wait a tick so the perf counter has at least one interval
            Thread.sleep(1000);
            Assert.assertEquals(size, object.getBytesRead());
            Assert.assertTrue(source.getReadRate() > 0);

            // proper ETag means no MPU was performed
            Assert.assertEquals(object.getMd5Hex(true).toLowerCase(), s3.getObjectMetadata(bucketName, key).getETag().toLowerCase());
        } finally {
            source.close();
            if (s3Target != null) s3Target.close();
            deleteObjects(bucketName);
            s3.deleteBucket(bucketName);
        }
    }

    @Ignore // only perform this test on a co-located S3 store!
    @Test
    public void testVeryLargeUploadStream() throws Exception {
        String bucketName = "ecs-sync-s3-target-test-bucket";
        createBucket(bucketName, false);

        TestStorage source = new TestStorage();
        source.withConfig(new com.emc.ecs.sync.config.storage.TestConfig()).withOptions(new SyncOptions());
        AwsS3Storage s3Target = null;
        try {
            String key = "large-stream-upload";
            long size = 512L * 1024 * 1024 + 10; // 512MB + 10 bytes
            InputStream stream = new RandomInputStream(size);
            SyncObject object = new SyncObject(source, key, new ObjectMetadata().withContentLength(size), stream, null);

            AwsS3Config s3Config = new AwsS3Config();
            if (endpointUri.getScheme() != null)
                s3Config.setProtocol(Protocol.valueOf(endpointUri.getScheme().toLowerCase()));
            s3Config.setHost(endpointUri.getHost());
            s3Config.setPort(endpointUri.getPort());
            s3Config.setAccessKey(accessKey);
            s3Config.setSecretKey(secretKey);
            s3Config.setLegacySignatures(true);
            s3Config.setDisableVHosts(true);
            s3Config.setBucketName(bucketName);

            s3Target = new AwsS3Storage();
            s3Target.withConfig(s3Config).withOptions(new SyncOptions());
            s3Target.configure(source, null, s3Target);

            String createdKey = s3Target.createObject(object);

            Assert.assertEquals(key, createdKey);

            // hyphen denotes an MPU
            Assert.assertTrue(s3.getObjectMetadata(bucketName, key).getETag().contains("-"));

            // verify bytes read from source
            // first wait a tick so the perf counter has at least one interval
            Thread.sleep(1000);
            Assert.assertEquals(size, object.getBytesRead());
            Assert.assertTrue(source.getReadRate() > 0);

            // should upload in series (single thread).. there's no way the stream can be split, so just verifying the MD5
            // should be sufficient. need to read the entire object since we can't use the ETag
            InputStream objectStream = s3.getObject(bucketName, key).getObjectContent();
            ChecksummedInputStream md5Stream = new ChecksummedInputStream(objectStream, new RunningChecksum(ChecksumAlgorithm.MD5));
            byte[] buffer = new byte[128 * 1024];
            int c;
            do {
                c = md5Stream.read(buffer);
            } while (c >= 0);
            md5Stream.close();

            Assert.assertEquals(object.getMd5Hex(true).toLowerCase(), md5Stream.getChecksum().getValue().toLowerCase());
        } finally {
            source.close();
            if (s3Target != null) s3Target.close();
            deleteObjects(bucketName);
            s3.deleteBucket(bucketName);
        }
    }

    @Ignore // only perform this test on a co-located S3 store!
    @Test
    public void testVeryLargeUploadFile() throws Exception {
        String bucketName = "ecs-sync-s3-target-test-bucket";
        createBucket(bucketName, false);

        TestStorage source = new TestStorage();
        source.withConfig(new com.emc.ecs.sync.config.storage.TestConfig()).withOptions(new SyncOptions());
        AwsS3Storage s3Target = null;
        try {
            String key = "large-file-upload";
            long size = 512L * 1024 * 1024 + 10; // 512MB + 10 bytes
            InputStream stream = new RandomInputStream(size);

            // create temp file
            File tempFile = File.createTempFile(key, null);
            tempFile.deleteOnExit();
            StreamUtil.copy(stream, new FileOutputStream(tempFile), size);

            SyncObject object = new SyncObject(source, key, new ObjectMetadata().withContentLength(size), new FileInputStream(tempFile), null);
            object.setProperty(AbstractFilesystemStorage.PROP_FILE, tempFile);

            AwsS3Config s3Config = new AwsS3Config();
            if (endpointUri.getScheme() != null)
                s3Config.setProtocol(Protocol.valueOf(endpointUri.getScheme().toLowerCase()));
            s3Config.setHost(endpointUri.getHost());
            s3Config.setPort(endpointUri.getPort());
            s3Config.setAccessKey(accessKey);
            s3Config.setSecretKey(secretKey);
            s3Config.setLegacySignatures(true);
            s3Config.setDisableVHosts(true);
            s3Config.setBucketName(bucketName);

            s3Target = new AwsS3Storage();
            s3Target.withConfig(s3Config).withOptions(new SyncOptions());
            s3Target.configure(source, null, s3Target);

            String createdKey = s3Target.createObject(object);

            Assert.assertEquals(key, createdKey);

            // hyphen denotes an MPU
            Assert.assertTrue(s3.getObjectMetadata(bucketName, key).getETag().contains("-"));

            // verify bytes read from source
            // first wait a tick so the perf counter has at least one interval
            Thread.sleep(1000);
            Assert.assertEquals(size, object.getBytesRead());
            Assert.assertTrue(source.getReadRate() > 0);

            // need to read the entire object since we can't use the ETag
            InputStream objectStream = s3.getObject(bucketName, key).getObjectContent();
            ChecksummedInputStream md5Stream = new ChecksummedInputStream(objectStream, new RunningChecksum(ChecksumAlgorithm.MD5));
            byte[] buffer = new byte[128 * 1024];
            int c;
            do {
                c = md5Stream.read(buffer);
            } while (c >= 0);
            md5Stream.close();

            Assert.assertEquals(object.getMd5Hex(true).toLowerCase(), md5Stream.getChecksum().getValue().toLowerCase());
        } finally {
            source.close();
            if (s3Target != null) s3Target.close();
            deleteObjects(bucketName);
            s3.deleteBucket(bucketName);
        }
    }

    private void runSync(SyncConfig syncConfig, SyncStorage source) {
        EcsSync sync = new EcsSync();
        sync.setSyncConfig(syncConfig);
        sync.setSource(source);

        sync.run();

        Assert.assertEquals(0, sync.getStats().getObjectsFailed());
    }

    private void alterContent(TestStorage storage, String identifier, String version) throws Exception {
        Collection<TestStorage.TestSyncObject> children = identifier == null ? storage.getRootObjects() : storage.getChildren(identifier);
        for (TestStorage.TestSyncObject object : children) {
            if (object.getData() != null && object.getData().length > 0) object.getData()[0] += 1;
            object.getMetadata().setUserMetadataValue("version", version);
            object.getMetadata().setModificationTime(new Date());
            if (object.getMetadata().isDirectory()) {
                alterContent(storage, storage.getIdentifier(object.getRelativePath(), true), version);
            }
        }
    }

    private void verifyBuckets(String bucket1, String bucket2) {
        Map<String, SortedSet<S3VersionSummary>> map1 = getAllVersions(bucket1);
        Map<String, SortedSet<S3VersionSummary>> map2 = getAllVersions(bucket2);

        // should be sufficient to compare ETags since they should be valid
        Assert.assertEquals(map1.keySet(), map2.keySet());
        for (String key : map1.keySet()) {
            Assert.assertEquals(map1.get(key).size(), map2.get(key).size());
            Iterator<S3VersionSummary> summaries2 = map2.get(key).iterator();
            String lastEtag = null;
            for (S3VersionSummary summary1 : map1.get(key)) {
                S3VersionSummary summary2 = summaries2.next();
                Assert.assertEquals(summary1.isDeleteMarker(), summary2.isDeleteMarker());
                // probably sufficient to compare ETags
                Assert.assertEquals(summary1.getETag(), summary2.getETag());
                // also make sure it's different from the last ETag
                if (summary1.getSize() > 0) Assert.assertNotEquals(lastEtag, summary1.getETag());
                lastEtag = summary1.getETag();
            }
        }
    }

    private void verifyAcls(AccessControlList acl1, AccessControlList acl2) {
        Assert.assertEquals(acl1.getOwner(), acl2.getOwner());
        for (Grant grant : acl1.getGrantsAsList()) {
            Assert.assertTrue(acl2.getGrantsAsList().contains(grant));
        }
        for (Grant grant : acl2.getGrantsAsList()) {
            Assert.assertTrue(acl1.getGrantsAsList().contains(grant));
        }
    }

    private List<S3VersionSummary> getVersions(String bucket, String key) {
        List<S3VersionSummary> summaries = new ArrayList<>();

        VersionListing listing = null;
        do {
            if (listing == null) listing = s3.listVersions(bucket, key);
            else listing = s3.listNextBatchOfVersions(listing);

            for (S3VersionSummary summary : listing.getVersionSummaries()) {
                if (summary.getKey().equals(key)) summaries.add(summary);
            }
        } while (listing.isTruncated());

        Collections.sort(summaries, new VersionComparator());

        return summaries;
    }

    private Map<String, SortedSet<S3VersionSummary>> getAllVersions(String bucket) {
        // build comprehensive key -> versions map
        // must sort by keys and mtime for simple comparison
        Map<String, SortedSet<S3VersionSummary>> map = new TreeMap<>();

        VersionListing listing = null;
        do {
            if (listing == null) listing = s3.listVersions(bucket, null);
            else listing = s3.listNextBatchOfVersions(listing);

            for (S3VersionSummary summary : listing.getVersionSummaries()) {
                SortedSet<S3VersionSummary> versions = map.get(summary.getKey());
                if (versions == null) {
                    versions = new TreeSet<>(new VersionComparator());
                    map.put(summary.getKey(), versions);
                }
                versions.add(summary);
            }
        } while (listing.isTruncated());

        return map;
    }

    private void deleteObjects(final String bucket) {
        ExecutorService executor = Executors.newFixedThreadPool(32);
        List<Future> futures = new ArrayList<>();
        try {
            ObjectListing listing = null;
            do {
                if (listing == null) listing = s3.listObjects(bucket);
                else listing = s3.listNextBatchOfObjects(listing);

                for (final S3ObjectSummary summary : listing.getObjectSummaries()) {
                    futures.add(executor.submit(new Runnable() {
                        @Override
                        public void run() {
                            s3.deleteObject(bucket, summary.getKey());
                        }
                    }));
                }
            } while (listing.isTruncated());

            for (Future future : futures) {
                try {
                    future.get();
                } catch (Exception e) {
                    if (e instanceof RuntimeException) throw (RuntimeException) e;
                    else throw new RuntimeException(e);
                }
            }
        } finally {
            executor.shutdown();
        }
    }

    private void deleteVersionedBucket(final String bucket) {
        ExecutorService executor = Executors.newFixedThreadPool(32);
        List<Future> futures = new ArrayList<>();
        try {
            VersionListing listing = null;
            do {
                if (listing == null) listing = s3.listVersions(bucket, null);
                else listing = s3.listNextBatchOfVersions(listing);

                for (final S3VersionSummary summary : listing.getVersionSummaries()) {
                    futures.add(executor.submit(new Runnable() {
                        @Override
                        public void run() {
                            s3.deleteVersion(bucket, summary.getKey(), summary.getVersionId());
                        }
                    }));
                }
            } while (listing.isTruncated());

            for (Future future : futures) {
                try {
                    future.get();
                } catch (Throwable t) {
                    log.warn("error deleting version", t);
                }
            }

            s3.deleteBucket(bucket);
        } catch (RuntimeException e) {
            log.warn("could not delete bucket " + bucket, e);
        } finally {
            executor.shutdown();
        }
    }

    private void createBucket(String bucket, boolean withVersioning) {
        try {
            s3.createBucket(bucket);
        } catch (AmazonServiceException e) {
            if (!e.getErrorCode().equals("BucketAlreadyExists")) throw e;
        }

        if (withVersioning) {
            s3.setBucketVersioningConfiguration(new SetBucketVersioningConfigurationRequest(bucket,
                    new BucketVersioningConfiguration(BucketVersioningConfiguration.ENABLED)));
        }
    }

    private class VersionComparator implements Comparator<S3VersionSummary> {
        @Override
        public int compare(S3VersionSummary o1, S3VersionSummary o2) {
            int result = o1.getLastModified().compareTo(o2.getLastModified());
            if (result == 0) result = o1.getVersionId().compareTo(o2.getVersionId());
            return result;
        }
    }
}