/*
* Copyright 2010-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://aws.amazon.com/apache2.0
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/
package com.amazonaws.mobileconnectors.kinesis.kinesisrecorder;
import android.util.Log;
import com.amazonaws.AmazonClientException;
import com.amazonaws.mobileconnectors.kinesis.kinesisrecorder.FileRecordStore.RecordIterator;
import com.amazonaws.util.StringUtils;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
/**
* An abstract class for Amazon Kinesis recorders. It manages local file store
* that temporarily saves records and sends these records later.
*/
public abstract class AbstractKinesisRecorder {
private static final String TAG = "AbstractKinesisRecorder";
/**
* Maximum number of records per batch. Note that Kinesis Stream and Kinesis
* Firehose have much higher limits. The limits are lowered for performance
* consideration.
*/
private static final int MAX_RECORDS_PER_BATCH = 128;
/**
* Maximum size in bytes of records in PutRecordBatch.
*/
private static final int MAX_BATCH_RECORDS_SIZE_BYTES = 512 * 1024;
/**
* The configurable options for Kinesis Recorder, includes the
* ClientConfiguration of the low level client.
*/
protected KinesisRecorderConfig config;
protected FileRecordStore recordStore;
/**
* Gets the sender to send saved records.
*
* @return a {@link RecordSender}
*/
protected abstract RecordSender getRecordSender();
/**
* Creates a {@link AbstractKinesisRecorder}.
*
* @param recordStore local file store that keeps Kinesis records
* @param config configuration
*/
protected AbstractKinesisRecorder(FileRecordStore recordStore, KinesisRecorderConfig config) {
if (recordStore == null) {
throw new IllegalArgumentException("Record store can't be null");
}
this.recordStore = recordStore;
this.config = config;
}
/**
* Saves a string to local storage to be sent later. It's a convenient
* method to save the UTF-8 encoded bytes of the string.
*
* @param data A string to submit to the stream
* @param streamName The stream to submit the data to.
*/
public void saveRecord(String data, String streamName) {
saveRecord(data.getBytes(StringUtils.UTF8), streamName);
}
/**
* Saves a record to local storage to be sent later. The record will be
* submitted to the streamName provided with a randomly generated partition
* key to ensure equal distribution across shards. Note: Since operation
* involves file I/O it is recommended not to call this method on the main
* thread to ensure responsive applications.
*
* @param data The data to submit to the stream
* @param streamName The stream to submit the data to.
*/
public void saveRecord(byte[] data, String streamName) {
try {
recordStore.put(FileRecordParser.asString(streamName, data));
} catch (IOException e) {
throw new AmazonClientException("Error saving record", e);
}
}
/**
* Submits all requests saved to Amazon Kinesis. Requests that are
* successfully sent will be deleted from the device. Requests that fail due
* to the device being offline will stop the submission process and be kept.
* Requests that fail due to other reasons (such as the request being
* invalid) will be deleted. Note: Since KinesisRecorder uses synchronous
* methods to make calls to Amazon Kinesis, do not call submitAll() on the
* main thread of your application.
*
* @throws AmazonClientException Thrown if there was an unrecoverable error
* during submission. Note: If the request appears to be
* invalid, the record will be deleted. If the request appears
* to be valid, it will be kept.
*/
public synchronized void submitAllRecords() {
RecordSender sender = getRecordSender();
RecordIterator iterator = recordStore.iterator();
List<byte[]> data = new ArrayList<byte[]>(MAX_RECORDS_PER_BATCH);
int retry = 0;
int count = 0;
try {
while (iterator.hasNext() && retry < 3) {
String streamName = nextBatch(iterator, data, MAX_RECORDS_PER_BATCH,
MAX_BATCH_RECORDS_SIZE_BYTES);
if (streamName == null || data.isEmpty()) {
break;
}
try {
iterator.removeReadRecords();
} catch (IOException e) {
throw new AmazonClientException("Failed to removed records.", e);
}
try {
List<byte[]> failures = sender.sendBatch(streamName, data);
int successCount = data.size() - failures.size();
count += successCount;
if (successCount == 0) {
// no record went through, increase retry count.
retry++;
}
if (!failures.isEmpty()) {
for (byte[] bytes : failures) {
saveRecord(bytes, streamName);
}
}
} catch (AmazonClientException ace) {
if (sender.isRecoverable(ace)) {
for (byte[] bytes : data) {
saveRecord(bytes, streamName);
}
Log.e(TAG,
"ServiceException in submit all, the values of the data inside the requests appears valid. The request will be kept",
ace);
} else {
// We have reason to believe the values in the request
// is invalid and cannot be sent or recovered.
Log.e(TAG,
"ServiceException in submit all, the last request is presumed to be the cause and will be dropped",
ace);
}
throw ace;
}
}
} finally {
Log.d(TAG, String.format("submitAllRecords sent %d records", count));
try {
iterator.close();
} catch (IOException e) {
throw new AmazonClientException("Failed to close record file", e);
}
}
}
/**
* Reads a batch of records belong to the same stream into a list. If data
* is read successfully, the stream name is returned.
*
* @param iterator record iterator
* @param data a list to hold data.
* @param maxCount maximum number of records in a batch
* @param maxSize a threshold that concludes a batch. It allows one extra
* record that brings the total size over this threshold.
* @return the stream name that the batch belongs to
*/
protected String nextBatch(RecordIterator iterator, List<byte[]> data, int maxCount,
int maxSize) {
data.clear();
String lastStreamName = null;
int size = 0;
int count = 0;
FileRecordParser frp = new FileRecordParser();
while (iterator.hasNext() && count < maxCount && size < maxSize) {
String line = iterator.peek();
if (line == null || line.isEmpty()) {
continue;
}
// parse a line. Skip in case of corrupted data
try {
frp.parse(line);
} catch (Exception e) {
Log.w(TAG, "Failed to read line. Skip.", e);
continue;
}
// check whether it belongs to previous batch
if (lastStreamName == null || lastStreamName.equals(frp.streamName)) {
data.add(frp.bytes);
// update counter
count++;
size += frp.bytes.length;
lastStreamName = frp.streamName;
iterator.next();
} else {
break;
}
}
return lastStreamName;
}
/**
* Returns the KinesisRecorderConfig this Kinesis Recorder is using. This is
* either the config passed into the constructor or the default one if one
* was not specified.
*
* @return The KinesisRecorderConfig
*/
public KinesisRecorderConfig getKinesisRecorderConfig() {
return config;
}
/**
* Returns the number of bytes KinesisRecorder currently has stored in the
* directory passed in the constructor.
*
* @return long The number of bytes used
*/
public long getDiskBytesUsed() {
return recordStore.getFileSize();
}
/**
* Returns the max number of bytes that this Kinesis Recorder will store on
* disk. This is the same as specified in getMaxStorageSize() in the
* KinesisRecorderConfig, either the one passed into the constructor or the
* default one that was constructed.
*
* @return The number of bytes allowed
*/
public long getDiskByteLimit() {
return config.getMaxStorageSize();
}
/**
* Removes all requests saved to disk in the directory provided this
* KinesisRecorder
*/
public synchronized void deleteAllRecords() {
try {
recordStore.iterator().removeAllRecords();
} catch (IOException e) {
throw new AmazonClientException("Error deleting events", e);
}
}
}