/*
* Copyright 2010-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Portions copyright 2006-2009 James Murty. Please see LICENSE.txt
* for applicable license terms and NOTICE.txt for applicable notices.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://aws.amazon.com/apache2.0
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/
package com.amazonaws.services.s3.internal;
import com.amazonaws.Request;
import com.amazonaws.SdkClientException;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.model.GetObjectMetadataRequest;
import com.amazonaws.services.s3.model.GetObjectRequest;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.transfer.exception.FileLockException;
import com.amazonaws.util.BinaryUtils;
import com.amazonaws.util.DateUtils;
import com.amazonaws.util.Md5Utils;
import com.amazonaws.util.SdkHttpUtils;
import com.amazonaws.util.StringUtils;
import com.amazonaws.util.ValidationUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.OutputStream;
import java.net.MalformedURLException;
import java.net.SocketException;
import java.net.URL;
import java.nio.channels.FileChannel;
import java.util.Arrays;
import java.util.Date;
import java.util.List;
import java.util.Map;
import javax.net.ssl.SSLProtocolException;
import static com.amazonaws.services.s3.internal.Constants.KB;
import static com.amazonaws.services.s3.internal.Constants.MB;
import static com.amazonaws.util.IOUtils.closeQuietly;
import static com.amazonaws.util.StringUtils.UTF8;
/**
* General utility methods used throughout the AWS S3 Java client.
*/
public class ServiceUtils {
private static final Log LOG = LogFactory.getLog(ServiceUtils.class);
public static final boolean APPEND_MODE = true;
public static final boolean OVERWRITE_MODE = false;
private static final SkipMd5CheckStrategy skipMd5CheckStrategy = SkipMd5CheckStrategy.INSTANCE;
@Deprecated
protected static final DateUtils dateUtils = new DateUtils();
public static Date parseIso8601Date(String dateString) {
return DateUtils.parseISO8601Date(dateString);
}
public static String formatIso8601Date(Date date) {
return DateUtils.formatISO8601Date(date);
}
public static Date parseRfc822Date(String dateString) {
if (StringUtils.isNullOrEmpty(dateString)) {
return null;
}
return DateUtils.parseRFC822Date(dateString);
}
public static String formatRfc822Date(Date date) {
return DateUtils.formatRFC822Date(date);
}
/**
* Safely converts a string to a byte array, first attempting to explicitly
* use our preferred encoding (UTF-8), and then falling back to the
* platform's default encoding if for some reason our preferred encoding
* isn't supported.
*
* @param s
* The string to convert to a byte array.
*
* @return The byte array contents of the specified string.
*/
public static byte[] toByteArray(String s) {
return s.getBytes(UTF8);
}
/**
* Removes any surrounding quotes from the specified string and returns a
* new string.
*
* @param s
* The string to check for surrounding quotes.
*
* @return A new string created from the specified string, minus any
* surrounding quotes.
*/
public static String removeQuotes(String s) {
if (s == null) return null;
s = s.trim();
if (s.startsWith("\"")) s = s.substring(1);
if (s.endsWith("\"")) s = s.substring(0, s.length() - 1);
return s;
}
/**
* Converts the specified request object into a URL, containing all the
* specified parameters, the specified request endpoint, etc.
*
* @param request
* The request to convert into a URL.
* @return A new URL representing the specified request.
*
* @throws SdkClientException
* If the request cannot be converted to a well formed URL.
* @deprecated No longer used. May be removed in a future major version.
*/
@Deprecated
public static URL convertRequestToUrl(Request<?> request) {
// To be backward compatible, this method by default does not
// remove the leading slash in the request resource-path.
return convertRequestToUrl(request, false);
}
/**
* Converts the specified request object into a URL, containing all the
* specified parameters, the specified request endpoint, etc.
*
* @param request
* The request to convert into a URL.
* @param removeLeadingSlashInResourcePath
* Whether the leading slash in resource-path should be removed
* before appending to the endpoint.
* @return A new URL representing the specified request.
*
* @throws SdkClientException
* If the request cannot be converted to a well formed URL.
* @deprecated No longer used. May be removed in a future major version.
*/
@Deprecated
public static URL convertRequestToUrl(Request<?> request, boolean removeLeadingSlashInResourcePath) {
return convertRequestToUrl(request, removeLeadingSlashInResourcePath, true);
}
/**
* Converts the specified request object into a URL, containing all the
* specified parameters, the specified request endpoint, etc.
*
* @param request
* The request to convert into a URL.
* @param removeLeadingSlashInResourcePath
* Whether the leading slash in resource-path should be removed
* before appending to the endpoint.
* @param urlEncode True if request resource path should be URL encoded
* @return A new URL representing the specified request.
*
* @throws SdkClientException
* If the request cannot be converted to a well formed URL.
*/
public static URL convertRequestToUrl(Request<?> request, boolean removeLeadingSlashInResourcePath,
boolean urlEncode) {
String resourcePath = urlEncode ?
SdkHttpUtils.urlEncode(request.getResourcePath(), true)
: request.getResourcePath();
// Removed the padding "/" that was already added into the request's resource path.
if (removeLeadingSlashInResourcePath
&& resourcePath.startsWith("/")) {
resourcePath = resourcePath.substring(1);
}
// Some http client libraries (e.g. Apache HttpClient) cannot handle
// consecutive "/"s between URL authority and path components.
// So we escape "////..." into "/%2F%2F%2F...", in the same way as how
// we treat consecutive "/"s in AmazonS3Client#presignRequest(...)
String urlPath = "/" + resourcePath;
urlPath = urlPath.replaceAll("(?<=/)/", "%2F");
StringBuilder url = new StringBuilder(request.getEndpoint().toString());
url.append(urlPath);
StringBuilder queryParams = new StringBuilder();
Map<String, List<String>> requestParams = request.getParameters();
for (Map.Entry<String, List<String>> entry : requestParams.entrySet()) {
for (String value : entry.getValue()) {
queryParams = queryParams.length() > 0 ? queryParams
.append("&") : queryParams.append("?");
queryParams.append(entry.getKey())
.append("=")
.append(SdkHttpUtils.urlEncode(value, false));
}
}
url.append(queryParams.toString());
try {
return new URL(url.toString());
} catch (MalformedURLException e) {
throw new SdkClientException(
"Unable to convert request to well formed URL: " + e.getMessage(), e);
}
}
/**
* Returns a new string created by joining each of the strings in the
* specified list together, with a comma between them.
*
* @param strings
* The list of strings to join into a single, comma delimited
* string list.
* @return A new string created by joining each of the strings in the
* specified list together, with a comma between strings.
*/
public static String join(List<String> strings) {
StringBuilder result = new StringBuilder();
boolean first = true;
for (String s : strings) {
if (!first) result.append(", ");
result.append(s);
first = false;
}
return result.toString();
}
/**
* Downloads an S3Object, as returned from
* {@link AmazonS3Client#getObject(com.amazonaws.services.s3.model.GetObjectRequest)},
* to the specified file.
*
* @param s3Object
* The S3Object containing a reference to an InputStream
* containing the object's data.
* @param destinationFile
* The file to store the object's data in.
* @param performIntegrityCheck
* Boolean valuable to indicate whether to perform integrity check
* @param appendData
* appends the data to end of the file.
*/
public static void downloadObjectToFile(S3Object s3Object,
final File destinationFile, boolean performIntegrityCheck,
boolean appendData) {
downloadToFile(s3Object, destinationFile, performIntegrityCheck, appendData, -1);
}
/**
* Same as {@link #downloadObjectToFile(S3Object, File, boolean, boolean)}
* but has an additional expected file length parameter for integrity
* checking purposes.
*
* @param expectedFileLength
* applicable only when appendData is true; the expected length
* of the file to append to.
*/
public static void downloadToFile(S3Object s3Object,
final File dstfile, boolean performIntegrityCheck,
final boolean appendData,
final long expectedFileLength)
{
createParentDirectoryIfNecessary(dstfile);
if (!FileLocks.lock(dstfile)) {
throw new FileLockException("Fail to lock " + dstfile
+ " for appendData=" + appendData);
}
OutputStream outputStream = null;
try {
final long actualLen = dstfile.length();
if (appendData && actualLen != expectedFileLength) {
// Fail fast to prevent data corruption
throw new IllegalStateException(
"Expected file length to append is "
+ expectedFileLength + " but actual length is "
+ actualLen + " for file " + dstfile);
}
outputStream = new BufferedOutputStream(new FileOutputStream(
dstfile, appendData));
byte[] buffer = new byte[1024*10];
int bytesRead;
while ((bytesRead = s3Object.getObjectContent().read(buffer)) > -1) {
outputStream.write(buffer, 0, bytesRead);
}
} catch (IOException e) {
s3Object.getObjectContent().abort();
throw new SdkClientException(
"Unable to store object contents to disk: " + e.getMessage(), e);
} finally {
closeQuietly(outputStream, LOG);
FileLocks.unlock(dstfile);
closeQuietly(s3Object.getObjectContent(), LOG);
}
if (performIntegrityCheck) {
byte[] clientSideHash = null;
byte[] serverSideHash = null;
try {
final ObjectMetadata metadata = s3Object.getObjectMetadata();
if (!skipMd5CheckStrategy.skipClientSideValidationPerGetResponse(metadata)) {
clientSideHash = Md5Utils.computeMD5Hash(new FileInputStream(dstfile));
serverSideHash = BinaryUtils.fromHex(metadata.getETag());
}
} catch (Exception e) {
LOG.warn("Unable to calculate MD5 hash to validate download: " + e.getMessage(), e);
}
if (clientSideHash != null && serverSideHash != null && !Arrays.equals(clientSideHash, serverSideHash)) {
throw new SdkClientException("Unable to verify integrity of data download. " +
"Client calculated content hash didn't match hash calculated by Amazon S3. " +
"The data stored in '" + dstfile.getAbsolutePath() + "' may be corrupt.");
}
}
}
/**
* Creates the parent directory for a file if it doesn't already exist.
* @param file
* @throws SdkClientException when creation of parent directory failed.
*/
public static void createParentDirectoryIfNecessary(final File file) {
final File parentDirectory = file.getParentFile();
if (parentDirectory == null || parentDirectory.mkdirs() || parentDirectory.exists()) {
return;
}
throw new SdkClientException("Unable to create directory in the path: " + parentDirectory.getAbsolutePath());
}
/**
* Interface for the task of downloading object from S3 to a specific file,
* enabling one-time retry mechanism after integrity check failure
* on the downloaded file.
*/
public interface RetryableS3DownloadTask {
/**
* User defines how to get the S3Object from S3 for this RetryableS3DownloadTask.
*
* @return
* The S3Object containing a reference to an InputStream
* containing the object's data.
*/
public S3Object getS3ObjectStream ();
/**
* User defines whether integrity check is needed for this RetryableS3DownloadTask.
*
* @return
* Boolean value indicating whether this task requires integrity check
* after downloading the S3 object to file.
*/
public boolean needIntegrityCheck ();
}
/**
* Gets an object stored in S3 and downloads it into the specified file.
* This method includes the one-time retry mechanism after integrity check failure
* on the downloaded file. It will also return immediately after getting null valued
* S3Object (when getObject request does not meet the specified constraints).
*
* @param file
* The file to store the object's data in.
* @param retryableS3DownloadTask
* The implementation of SafeS3DownloadTask interface which allows user to
* get access to all the visible variables at the calling site of this method.
*/
public static S3Object retryableDownloadS3ObjectToFile(File file,
RetryableS3DownloadTask retryableS3DownloadTask, boolean appendData) {
boolean hasRetried = false;
boolean needRetry;
S3Object s3Object;
do {
needRetry = false;
s3Object = retryableS3DownloadTask.getS3ObjectStream();
if ( s3Object == null )
return null;
try {
ServiceUtils.downloadObjectToFile(s3Object, file,
retryableS3DownloadTask.needIntegrityCheck(),
appendData);
} catch (SdkClientException ace) {
if (!ace.isRetryable()) {
s3Object.getObjectContent().abort();
throw ace;
}
// Determine whether an immediate retry is needed according to the captured SdkClientException.
// (There are three cases when downloadObjectToFile() throws SdkClientException:
// 1) SocketException or SSLProtocolException when writing to disk (e.g. when user aborts the download)
// 2) Other IOException when writing to disk
// 3) MD5 hashes don't match
// The current code will retry the download only when case 2) or 3) happens.
if (ace.getCause() instanceof SocketException || ace.getCause() instanceof SSLProtocolException) {
throw ace;
} else {
needRetry = true;
if ( hasRetried ) {
s3Object.getObjectContent().abort();
throw ace;
} else {
LOG.info("Retry the download of object " + s3Object.getKey() + " (bucket " + s3Object.getBucketName() + ")", ace);
hasRetried = true;
}
}
}
} while ( needRetry );
return s3Object;
}
/**
* Append the data in sourceFile to destinationFile.
*
* Note that the sourceFile is deleted after appending the data.
*
* @param sourceFile
* The file that is to be appended.
* @param destinationFile
* The file to append to.
*/
public static void appendFile(File sourceFile, File destinationFile) {
ValidationUtils.assertNotNull(destinationFile, "destFile");
ValidationUtils.assertNotNull(sourceFile, "sourceFile");
if (!FileLocks.lock(sourceFile)) {
throw new FileLockException("Fail to lock " + sourceFile);
}
if (!FileLocks.lock(destinationFile)) {
throw new FileLockException("Fail to lock " + destinationFile);
}
FileChannel in = null;
FileChannel out = null;
try {
in = new FileInputStream(sourceFile).getChannel();
out = new FileOutputStream(destinationFile, true).getChannel();
final long size = in.size();
// In some Windows platforms, copying large files fail due to insufficient system resources.
// Limit copy size to 32 MB in each transfer
final long count = 32 * MB;
long position = 0;
while (position < size) {
position += in.transferTo(position, count, out);
}
} catch (IOException e) {
throw new SdkClientException("Unable to append file " + sourceFile.getAbsolutePath()
+ "to destination file " + destinationFile.getAbsolutePath() + "\n" + e.getMessage(), e);
} finally {
closeQuietly(out, LOG);
closeQuietly(in, LOG);
FileLocks.unlock(sourceFile);
FileLocks.unlock(destinationFile);
try {
if (!sourceFile.delete()) {
LOG.warn("Failed to delete file " + sourceFile.getAbsolutePath());
}
} catch (SecurityException exception) {
LOG.warn("Security manager denied delete access to file " + sourceFile.getAbsolutePath());
}
}
}
public static boolean isS3USStandardEndpoint(String endpoint) {
return endpoint.endsWith(Constants.S3_HOSTNAME);
}
/**
* @return true if the given endpoint is known to be at the region us-east-1.
* (currently this includes S3 standard, S3 external-1 endpoints).
*/
public static boolean isS3USEastEndpiont(String endpoint) {
return isS3USStandardEndpoint(endpoint) ||
endpoint.endsWith(Constants.S3_EXTERNAL_1_HOSTNAME);
}
public static boolean isS3AccelerateEndpoint(String endpoint) {
return endpoint.endsWith(Constants.S3_ACCELERATE_HOSTNAME) ||
endpoint.endsWith(Constants.S3_ACCELERATE_DUALSTACK_HOSTNAME);
}
/**
* Returns the part count of the object represented by the getObjectRequest.
*
* @param getObjectRequest
* The request to check.
* @param s3
* The Amazon s3 client.
*
* @return The number of parts in the object if it is multipart object, otherwise returns null.
*/
public static Integer getPartCount(GetObjectRequest getObjectRequest, AmazonS3 s3) {
ValidationUtils.assertNotNull(s3, "S3 client");
ValidationUtils.assertNotNull(getObjectRequest, "GetObjectRequest");
ObjectMetadata metadata = s3.getObjectMetadata(new GetObjectMetadataRequest(getObjectRequest.getBucketName(), getObjectRequest.getKey(), getObjectRequest.getVersionId())
.withSSECustomerKey(getObjectRequest.getSSECustomerKey())
.withPartNumber(1));
return metadata.getPartCount();
}
/**
* Returns the last byte number in a part of an object.
*
* @param s3
* The Amazon s3 client.
* @param getObjectRequest
* The request to check.
* @param partNumber
* The part in which we need the last byte number.
* @return
* The last byte number in the part.
*/
public static long getLastByteInPart(AmazonS3 s3, GetObjectRequest getObjectRequest, Integer partNumber) {
ValidationUtils.assertNotNull(s3, "S3 client");
ValidationUtils.assertNotNull(getObjectRequest, "GetObjectRequest");
ValidationUtils.assertNotNull(partNumber, "partNumber");
ObjectMetadata metadata = s3.getObjectMetadata(new GetObjectMetadataRequest(getObjectRequest.getBucketName(), getObjectRequest.getKey(), getObjectRequest.getVersionId())
.withSSECustomerKey(getObjectRequest.getSSECustomerKey())
.withPartNumber(partNumber));
return metadata.getContentRange()[1];
}
}