AmazonKinesisFirehoseToS3Sample.java

/*
 * Copyright 2012-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License").
 * You may not use this file except in compliance with the License.
 * A copy of the License is located at
 *
 *  http://aws.amazon.com/apache2.0
 *
 * or in the "license" file accompanying this file. This file is distributed
 * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
 * express or implied. See the License for the specific language governing
 * permissions and limitations under the License.
 */
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.util.List;
import java.util.Properties;
import java.util.concurrent.TimeUnit;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;

import com.amazonaws.AmazonClientException;
import com.amazonaws.AmazonServiceException;
import com.amazonaws.services.kinesisfirehose.model.BufferingHints;
import com.amazonaws.services.kinesisfirehose.model.CompressionFormat;
import com.amazonaws.services.kinesisfirehose.model.CreateDeliveryStreamRequest;
import com.amazonaws.services.kinesisfirehose.model.DeliveryStreamDescription;
import com.amazonaws.services.kinesisfirehose.model.EncryptionConfiguration;
import com.amazonaws.services.kinesisfirehose.model.KMSEncryptionConfig;
import com.amazonaws.services.kinesisfirehose.model.NoEncryptionConfig;
import com.amazonaws.services.kinesisfirehose.model.S3DestinationConfiguration;
import com.amazonaws.services.kinesisfirehose.model.S3DestinationUpdate;
import com.amazonaws.services.kinesisfirehose.model.UpdateDestinationRequest;
import com.amazonaws.util.StringUtils;

/**
 * Amazon Kinesis Firehose is a fully managed service for real-time streaming data delivery
 * to destinations such as Amazon S3 and Amazon Redshift. Firehose is part of the Amazon Kinesis
 * streaming data family, along with Amazon Kinesis Streams. With Firehose, you do not need to
 * write any applications or manage any resources. You configure your data producers to send data
 * to Firehose and it automatically delivers the data to the destination that you specified.
 *
 * Detailed Amazon Kinesis Firehose documentation can be found here:
 * https://aws.amazon.com/documentation/firehose/
 *
 * This is a sample java application to deliver data to Amazon S3 destination.
 */
public class AmazonKinesisFirehoseToS3Sample extends AbstractAmazonKinesisFirehoseDelivery {

    /*
     * Before running the code:
     *
     * Step 1: Please check you have AWS access credentials set under
     * (~/.aws/credentials). If not, fill in your AWS access credentials in the
     * provided credentials file template, and be sure to move the file to the
     * default location (~/.aws/credentials) where the sample code will load the
     * credentials from.
     * https://console.aws.amazon.com/iam/home?#security_credential
     *
     * WARNING: To avoid accidental leakage of your credentials, DO NOT keep the
     * credentials file in your source directory.
     *
     * Step 2: Update the firehosetos3sample.properties file with the required parameters.
     */

    // DeliveryStream properties
    private static String updateS3ObjectPrefix;
    private static Integer updateSizeInMBs;
    private static Integer updateIntervalInSeconds;

    // Properties File
    private static final String CONFIG_FILE = "firehosetos3sample.properties";

    // Logger
    private static final Log LOG = LogFactory.getLog(AmazonKinesisFirehoseToS3Sample.class);

    /**
     * Initialize the parameters.
     *
     * @throws Exception
     */
    private static void init() throws Exception {
        // Load the parameters from properties file
        loadConfig();

        // Initialize the clients
        initClients();

        // Validate AccountId parameter is set
        if (StringUtils.isNullOrEmpty(accountId)) {
            throw new IllegalArgumentException("AccountId is empty. Please enter the accountId in "
                    + CONFIG_FILE + " file");
        }
    }

    /**
     * Load the input parameters from properties file.
     *
     * @throws FileNotFoundException
     * @throws IOException
     */
    private static void loadConfig() throws FileNotFoundException, IOException {
        try (InputStream configStream = Thread.currentThread().getContextClassLoader().getResourceAsStream(CONFIG_FILE)) {
            if (configStream == null) {
                throw new FileNotFoundException();
            }

            properties = new Properties();
            properties.load(configStream);
        }

        // Read properties
        accountId = properties.getProperty("customerAccountId");
        createS3Bucket = Boolean.valueOf(properties.getProperty("createS3Bucket"));
        s3RegionName = properties.getProperty("s3RegionName");
        s3BucketName = properties.getProperty("s3BucketName").trim();
        s3BucketARN = getBucketARN(s3BucketName);
        s3ObjectPrefix = properties.getProperty("s3ObjectPrefix").trim();

        String sizeInMBsProperty = properties.getProperty("destinationSizeInMBs");
        s3DestinationSizeInMBs = StringUtils.isNullOrEmpty(sizeInMBsProperty) ? null : Integer.parseInt(sizeInMBsProperty.trim());
        String intervalInSecondsProperty = properties.getProperty("destinationIntervalInSeconds");
        s3DestinationIntervalInSeconds = StringUtils.isNullOrEmpty(intervalInSecondsProperty) ? null : Integer.parseInt(intervalInSecondsProperty.trim());

        deliveryStreamName = properties.getProperty("deliveryStreamName");
        s3DestinationAWSKMSKeyId = properties.getProperty("destinationAWSKMSKeyId");
        firehoseRegion = properties.getProperty("firehoseRegion");
        iamRoleName = properties.getProperty("iamRoleName");
        iamRegion = properties.getProperty("iamRegion");

        // Update Delivery Stream Destination related properties
        enableUpdateDestination = Boolean.valueOf(properties.getProperty("updateDestination"));
        updateS3ObjectPrefix = properties.getProperty("updateS3ObjectPrefix").trim();
        String updateSizeInMBsProperty = properties.getProperty("updateSizeInMBs");
        updateSizeInMBs = StringUtils.isNullOrEmpty(updateSizeInMBsProperty) ? null : Integer.parseInt(updateSizeInMBsProperty.trim());
        String updateIntervalInSecondsProperty = properties.getProperty("updateIntervalInSeconds");
        updateIntervalInSeconds = StringUtils.isNullOrEmpty(updateIntervalInSecondsProperty) ? null : Integer.parseInt(updateIntervalInSecondsProperty.trim());
    }

    public static void main(String[] args) throws Exception {
        init();

        try {
            // Create S3 bucket for deliveryStream to deliver data
            createS3Bucket();

            // Create the DeliveryStream
            createDeliveryStream();

            // Print the list of delivery streams
            printDeliveryStreams();

            // Put records into deliveryStream
            LOG.info("Putting records in deliveryStream : " + deliveryStreamName + " via Put Record method.");
            putRecordIntoDeliveryStream();

            // Batch Put records into deliveryStream
            LOG.info("Putting records in deliveryStream : " + deliveryStreamName
                    + " via Put Record Batch method. Now you can check your S3 bucket " + s3BucketName
                    + " for the data delivered by deliveryStream.");
            putRecordBatchIntoDeliveryStream();

            // Wait for some interval for the firehose to write data to the S3 bucket
            int waitTimeSecs = s3DestinationIntervalInSeconds == null ? DEFAULT_WAIT_INTERVAL_FOR_DATA_DELIVERY_SECS : s3DestinationIntervalInSeconds;
            waitForDataDelivery(waitTimeSecs);

            // Update the deliveryStream and Put records into updated deliveryStream, only if the flag is set
            if (enableUpdateDestination) {
                // Update the deliveryStream
                updateDeliveryStream();

                // Wait for some interval to propagate the updated configuration options before ingesting data
                LOG.info("Waiting for few seconds to propagate the updated configuration options.");
                TimeUnit.SECONDS.sleep(60);

                // Put records into updated deliveryStream. Records will be delivered to new S3 prefix location
                LOG.info("Putting records in updated deliveryStream : " + deliveryStreamName + " via Put Record method.");
                putRecordIntoDeliveryStream();

                // Batch Put records into updated deliveryStream. Records will be delivered to new S3 prefix location
                LOG.info("Putting records in updated deliveryStream : " + deliveryStreamName + " via Put Record Batch method.");
                putRecordBatchIntoDeliveryStream();

                // Wait for some interval for the deliveryStream to write data to new prefix location in S3 bucket
                waitTimeSecs = updateIntervalInSeconds == null ? waitTimeSecs : updateIntervalInSeconds;
                waitForDataDelivery(waitTimeSecs);
            }
        } catch (AmazonServiceException ase) {
            LOG.error("Caught Amazon Service Exception");
            LOG.error("Status Code " + ase.getErrorCode());
            LOG.error("Message: " + ase.getErrorMessage(), ase);
        } catch (AmazonClientException ace) {
            LOG.error("Caught Amazon Client Exception");
            LOG.error("Exception Message " + ace.getMessage(), ace);
        }
    }

    /**
     * Method to create delivery stream for S3 destination configuration.
     *
     * @throws Exception
     */
    private static void createDeliveryStream() throws Exception {

        boolean deliveryStreamExists = false;

        LOG.info("Checking if " + deliveryStreamName + " already exits");
        List<String> deliveryStreamNames = listDeliveryStreams();
        if (deliveryStreamNames != null && deliveryStreamNames.contains(deliveryStreamName)) {
            deliveryStreamExists = true;
            LOG.info("DeliveryStream " + deliveryStreamName + " already exists. Not creating the new delivery stream");
        } else {
            LOG.info("DeliveryStream " + deliveryStreamName + " does not exist");
        }

        if (!deliveryStreamExists) {
            // Create deliveryStream
            CreateDeliveryStreamRequest createDeliveryStreamRequest = new CreateDeliveryStreamRequest();
            createDeliveryStreamRequest.setDeliveryStreamName(deliveryStreamName);

            S3DestinationConfiguration s3DestinationConfiguration = new S3DestinationConfiguration();
            s3DestinationConfiguration.setBucketARN(s3BucketARN);
            s3DestinationConfiguration.setPrefix(s3ObjectPrefix);
            // Could also specify GZIP or ZIP
            s3DestinationConfiguration.setCompressionFormat(CompressionFormat.UNCOMPRESSED);

            // Encryption configuration is optional
            EncryptionConfiguration encryptionConfiguration = new EncryptionConfiguration();
            if (!StringUtils.isNullOrEmpty(s3DestinationAWSKMSKeyId)) {
                encryptionConfiguration.setKMSEncryptionConfig(new KMSEncryptionConfig()
                        .withAWSKMSKeyARN(s3DestinationAWSKMSKeyId));
            } else {
                encryptionConfiguration.setNoEncryptionConfig(NoEncryptionConfig.NoEncryption);
            }
            s3DestinationConfiguration.setEncryptionConfiguration(encryptionConfiguration);

            BufferingHints bufferingHints = null;
            if (s3DestinationSizeInMBs != null || s3DestinationIntervalInSeconds != null) {
                bufferingHints = new BufferingHints();
                bufferingHints.setSizeInMBs(s3DestinationSizeInMBs);
                bufferingHints.setIntervalInSeconds(s3DestinationIntervalInSeconds);
            }
            s3DestinationConfiguration.setBufferingHints(bufferingHints);

            // Create and set IAM role so that firehose service has access to the S3Buckets to put data 
            // and KMS keys (if provided) to encrypt data. Please check the trustPolicyDocument.json and 
            // permissionsPolicyDocument.json files for the trust and permissions policies set for the role.
            String iamRoleArn = createIamRole(s3ObjectPrefix);
            s3DestinationConfiguration.setRoleARN(iamRoleArn);
            
            createDeliveryStreamRequest.setS3DestinationConfiguration(s3DestinationConfiguration);

            firehoseClient.createDeliveryStream(createDeliveryStreamRequest);

            // The Delivery Stream is now being created.
            LOG.info("Creating DeliveryStream : " + deliveryStreamName);
            waitForDeliveryStreamToBecomeAvailable(deliveryStreamName);
        }
    }

    /**
     * Method to update s3 destination with updated s3Prefix, and buffering hints values.
     *
     * @throws Exception
     */
    private static void updateDeliveryStream() throws Exception {
        DeliveryStreamDescription deliveryStreamDescription = describeDeliveryStream(deliveryStreamName);
        
        LOG.info("Updating DeliveryStream Destination: " + deliveryStreamName + " with new configuration options");
        // get(0) -> DeliveryStream currently supports only one destination per DeliveryStream
        UpdateDestinationRequest updateDestinationRequest = 
                new UpdateDestinationRequest()
                    .withDeliveryStreamName(deliveryStreamName)
                    .withCurrentDeliveryStreamVersionId(deliveryStreamDescription.getVersionId())
                    .withDestinationId(deliveryStreamDescription.getDestinations().get(0).getDestinationId());
        
        S3DestinationUpdate s3DestinationUpdate = new S3DestinationUpdate();
        s3DestinationUpdate.withPrefix(updateS3ObjectPrefix);

        BufferingHints bufferingHints = null;
        if (updateSizeInMBs != null || updateIntervalInSeconds != null) {
            bufferingHints = new BufferingHints();
            bufferingHints.setSizeInMBs(updateSizeInMBs);
            bufferingHints.setIntervalInSeconds(updateIntervalInSeconds);
        }
        s3DestinationUpdate.setBufferingHints(bufferingHints);

        // Update the role policy with new s3Prefix configuration
        putRolePolicy(updateS3ObjectPrefix);

        updateDestinationRequest.setS3DestinationUpdate(s3DestinationUpdate);

        // Update deliveryStream destination with new configuration options such as s3Prefix and Buffering Hints.
        // Can also update Compression format, KMS key values and IAM Role.
        firehoseClient.updateDestination(updateDestinationRequest);
    }
}