S3Utils.java

package com.aol.micro.server.s3.data;

import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.function.Function;
import java.util.function.Supplier;

import cyclops.stream.ReactiveSeq;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Component;

import com.amazonaws.AmazonClientException;
import com.amazonaws.AmazonServiceException;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.model.DeleteObjectsRequest;
import com.amazonaws.services.s3.model.DeleteObjectsRequest.KeyVersion;
import com.amazonaws.services.s3.model.ListObjectsRequest;
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.amazonaws.services.s3.transfer.TransferManager;


@Component
public class S3Utils {

    private static final InputStream emptyInputStream = new EmptyInputStream();

    private final AmazonS3Client client;
    private final TransferManager transferManager;
    private final String tmpDirectory;
    private final ExecutorService uploaderService;
    private final boolean aes256Encryption;
    private final ReadUtils readUtils;

    @Autowired
    public S3Utils(AmazonS3Client client, TransferManager transferManager,
            @Value("${s3.tmp.dir:#{systemProperties['java.io.tmpdir']}}") String tmpDirectory,
            @Value("${s3.aes256.enabled:false}") boolean aes256Encryption,
            @Qualifier("s3UploadExecutorService") ExecutorService uploaderService) {
        this.client = client;
        this.transferManager = transferManager;
        this.tmpDirectory = tmpDirectory;
        this.uploaderService = uploaderService;
        this.aes256Encryption = aes256Encryption;
        this.readUtils = new ReadUtils(
                                       transferManager, tmpDirectory);
    }

    public S3Utils(AmazonS3Client client, TransferManager transferManager, String tmpDirectory,
            ExecutorService uploaderService) {
        this(
             client, transferManager, tmpDirectory, false, uploaderService);
    }

    public S3Reader reader(String bucket) {

        return new S3Reader(
                            readUtils, client, bucket);
    }

    public S3ObjectWriter writer(String bucket) {
        return new S3ObjectWriter(
                                  transferManager, bucket, new File(
                                                                    tmpDirectory),
                                  aes256Encryption);
    }

    public S3StringWriter stringWriter(String bucket) {
        return new S3StringWriter(
                                  client, bucket, uploaderService, aes256Encryption);
    }

    public S3Deleter deleter(String bucket) {
        return new S3Deleter(
                             bucket, client);
    }

    /**
     * Method returns list of all <b>S3ObjectSummary</b> objects, subject to
     * <i>req</i> parameters. Multiple S3 calls will be performed if there are
     * more than 1000 elements there
     * 
     * @param req
     *            - ListObjectRequest to be used.
     * @return List of S3ObjectSummary from bucket,
     */
    public List<S3ObjectSummary> getAllSummaries(ListObjectsRequest req) {
        List<S3ObjectSummary> result = new ArrayList<>();
        String marker = null;
        ListObjectsRequest req2 = (ListObjectsRequest) req.clone();
        ObjectListing listing;
        do {
            listing = client.listObjects(req2.withMarker(marker));
            marker = listing.getNextMarker();
            result.addAll(listing.getObjectSummaries());
        } while (listing.isTruncated());

        return result;
    }

    /**
     * Method return stream of S3ObjectSummary objects, subject to <i>req</i>
     * parameters Method will perform one query for every 1000 elements (current
     * s3 limitation). It is lazy, so there would be no unnecesarry calls
     * 
     * @param req
     *            - ListObjectRequest to be used.
     * @param processor
     *            - Function that convert S3ObjectSummary to any object
     * @return ReactiveSeq of converted S3Object summary elements.
     */
    public <T> ReactiveSeq<T> getSummariesStream(ListObjectsRequest req, Function<S3ObjectSummary, T> processor) {
        return ReactiveSeq.fromIterator(new S3ObjectSummaryIterator(
                                                                    client, req))
                          .map(processor);
    }

    /**
     * Method delete all <i>objects</i> from <i>bucketName</i> in groups by 1000
     * elements
     * 
     * @param bucketName
     * @param objects
     */
    public void delete(String bucketName, List<KeyVersion> objects) {
        ReactiveSeq.fromList(objects)
                   .grouped(1000)
                   .forEach(l -> {
                       DeleteObjectsRequest req = new DeleteObjectsRequest(
                                                                           bucketName);
                       req.setKeys(l.toList());
                       client.deleteObjects(req);
                   });
    }

    /**
     * Provide empty InputStream.
     * <p>
     * This implementation can be convenient if you need to place some empty
     * value to s3 bucket.
     * 
     * @return empty InputStream
     */
    public static InputStream emptyInputStream() {
        return emptyInputStream;
    }
    
    /**
     * Method returns InputStream from S3Object. Multi-part download is used to
     * get file. s3.tmp.dir property used to store temporary files.
     * 
     * @param bucketName
     * @param key
     * @return
     * @throws AmazonServiceException
     * @throws AmazonClientException
     * @throws InterruptedException
     * @throws IOException
     * 
     * @deprecated see ReadUtils
     */
    @Deprecated
    public InputStream getInputStream(String bucketName, String key)
            throws AmazonServiceException, AmazonClientException, InterruptedException, IOException{
        return readUtils.getInputStream(bucketName, key);
    }
    
    /**
     * Method returns InputStream from S3Object. Multi-part download is used to
     * get file. s3.tmp.dir property used to store temporary files. You can
     * specify temporary file name by using tempFileSupplier object.
     * 
     * @param bucketName
     * @param key
     *            -
     * @param tempFileSupplier
     *            - Supplier providing temporary filenames
     * @return InputStream of
     * @throws AmazonServiceException
     * @throws AmazonClientException
     * @throws InterruptedException
     * @throws IOException
     * 
     * @deprecated see ReadUtils
     */
    @Deprecated
    public InputStream getInputStream(String bucketName, String key, Supplier<File> tempFileSupplier) throws AmazonServiceException, AmazonClientException, InterruptedException, IOException{
        return readUtils.getInputStream(bucketName, key, tempFileSupplier);
    }

    static class EmptyInputStream extends InputStream {

        @Override
        public int available() {
            return 0;
        }

        @Override
        public int read() throws IOException {
            throw new IOException(
                                  "Nothing to read here");
        }

    }

    static class S3ObjectSummaryIterator implements Iterator<S3ObjectSummary> {

        private final AmazonS3Client client;
        private ListObjectsRequest req;
        private Iterator<S3ObjectSummary> iterator;
        private boolean empty = true;

        public S3ObjectSummaryIterator(AmazonS3Client client, ListObjectsRequest req) {
            this.client = client;
            this.req = req;
            updateIterator();
        }

        @Override
        public boolean hasNext() {
            if (iterator.hasNext()) {
                return true;
            } else if (!empty) {
                updateIterator();
                return iterator.hasNext();
            } else {
                return false;
            }
        }

        private void updateIterator() {
            if (iterator == null || !iterator.hasNext()) {
                ObjectListing listing = client.listObjects(req);
                req = req.withMarker(listing.getNextMarker());
                empty = !listing.isTruncated();
                iterator = listing.getObjectSummaries()
                                  .iterator();
            }
        }

        @Override
        public S3ObjectSummary next() {
            updateIterator();
            return iterator.next();
        }

    }

}