/*
* Copyright 2010 Cloud.com, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.cloud.bridge.service.controller.s3;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Date;
import java.util.Enumeration;
import java.util.List;
import java.util.UUID;
import javax.activation.DataHandler;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import javax.xml.namespace.QName;
import javax.xml.parsers.DocumentBuilderFactory;
import javax.xml.stream.XMLOutputFactory;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamWriter;
import org.apache.axiom.om.OMAbstractFactory;
import org.apache.axiom.om.OMFactory;
import org.apache.axis2.databinding.utils.writer.MTOMAwareXMLSerializer;
import org.apache.log4j.Logger;
import org.apache.commons.fileupload.MultipartStream;
import org.hibernate.LockMode;
import com.amazon.s3.GetBucketAccessControlPolicyResponse;
import com.amazon.s3.GetObjectAccessControlPolicyResponse;
import com.cloud.bridge.model.SBucket;
import com.cloud.bridge.model.SHost;
import com.cloud.bridge.model.SObject;
import com.cloud.bridge.model.SObjectItem;
import com.cloud.bridge.persist.PersistContext;
import com.cloud.bridge.persist.dao.SBucketDao;
import com.cloud.bridge.persist.dao.SObjectDao;
import com.cloud.bridge.service.S3BucketAdapter;
import com.cloud.bridge.service.S3Constants;
import com.cloud.bridge.service.S3RestServlet;
import com.cloud.bridge.service.S3SoapServiceImpl;
import com.cloud.bridge.service.ServiceProvider;
import com.cloud.bridge.service.ServletAction;
import com.cloud.bridge.service.UserContext;
import com.cloud.bridge.service.core.ec2.EC2Volume;
import com.cloud.bridge.service.core.s3.S3AccessControlPolicy;
import com.cloud.bridge.service.core.s3.S3AuthParams;
import com.cloud.bridge.service.core.s3.S3ConditionalHeaders;
import com.cloud.bridge.service.core.s3.S3DeleteObjectRequest;
import com.cloud.bridge.service.core.s3.S3Engine;
import com.cloud.bridge.service.core.s3.S3GetBucketAccessControlPolicyRequest;
import com.cloud.bridge.service.core.s3.S3GetObjectAccessControlPolicyRequest;
import com.cloud.bridge.service.core.s3.S3GetObjectRequest;
import com.cloud.bridge.service.core.s3.S3GetObjectResponse;
import com.cloud.bridge.service.core.s3.S3MetaDataEntry;
import com.cloud.bridge.service.core.s3.S3PutObjectInlineRequest;
import com.cloud.bridge.service.core.s3.S3PutObjectInlineResponse;
import com.cloud.bridge.service.core.s3.S3PutObjectRequest;
import com.cloud.bridge.service.core.s3.S3Response;
import com.cloud.bridge.service.core.s3.S3SetObjectAccessControlPolicyRequest;
import com.cloud.bridge.util.Converter;
import com.cloud.bridge.util.DateHelper;
import com.cloud.bridge.util.HeaderParam;
import com.cloud.bridge.util.ServletRequestDataSource;
/**
* @author Kelven Yang
*/
public class S3ObjectAction implements ServletAction {
protected final static Logger logger = Logger.getLogger(S3ObjectAction.class);
private DocumentBuilderFactory dbf = null;
private OMFactory factory = OMAbstractFactory.getOMFactory();
private XMLOutputFactory xmlOutFactory = XMLOutputFactory.newInstance();
public S3ObjectAction() {
dbf = DocumentBuilderFactory.newInstance();
dbf.setNamespaceAware( true );
}
public void execute(HttpServletRequest request, HttpServletResponse response) throws IOException
{
String method = request.getMethod();
String queryString = request.getQueryString();
response.addHeader( "x-amz-request-id", UUID.randomUUID().toString());
if (method.equalsIgnoreCase( "GET" )) {
if ( queryString != null && queryString.length() > 0 )
{
if ( queryString.equalsIgnoreCase("acl")) executeGetObjectAcl(request, response);
}
else executeGetObject(request, response);
}
else if (method.equalsIgnoreCase( "PUT" )) {
if ( queryString != null && queryString.length() > 0 )
{
if ( queryString.equalsIgnoreCase("acl")) executePutObjectAcl(request, response);
}
else executePutObject(request, response);
}
else if (method.equalsIgnoreCase( "DELETE" )) {
executeDeleteObject(request, response);
}
else if (method.equalsIgnoreCase( "HEAD" )) {
executeHeadObject(request, response);
}
else if (method.equalsIgnoreCase( "POST" )) {
executePostObject(request, response);
}
else throw new IllegalArgumentException( "Unsupported method in REST request");
}
private void executeGetObjectAcl(HttpServletRequest request, HttpServletResponse response) throws IOException
{
String bucketName = (String)request.getAttribute(S3Constants.BUCKET_ATTR_KEY);
String key = (String)request.getAttribute(S3Constants.OBJECT_ATTR_KEY);
S3GetObjectAccessControlPolicyRequest engineRequest = new S3GetObjectAccessControlPolicyRequest();
engineRequest.setBucketName( bucketName );
engineRequest.setKey( key );
S3AccessControlPolicy engineResponse = ServiceProvider.getInstance().getS3Engine().handleRequest(engineRequest);
// -> serialize using the apache's Axiom classes
GetObjectAccessControlPolicyResponse onePolicy = S3SoapServiceImpl.toGetObjectAccessControlPolicyResponse( engineResponse );
try {
OutputStream os = response.getOutputStream();
response.setStatus(200);
response.setContentType("text/xml; charset=UTF-8");
XMLStreamWriter xmlWriter = xmlOutFactory.createXMLStreamWriter( os );
String documentStart = new String( "<?xml version=\"1.0\" encoding=\"UTF-8\"?>" );
os.write( documentStart.getBytes());
MTOMAwareXMLSerializer MTOMWriter = new MTOMAwareXMLSerializer( xmlWriter );
onePolicy.serialize( new QName( "http://s3.amazonaws.com/doc/2006-03-01/", "GetObjectAccessControlPolicyResponse", "ns1" ), factory, MTOMWriter );
xmlWriter.flush();
xmlWriter.close();
os.close();
}
catch( XMLStreamException e ) {
throw new IOException( e.toString());
}
}
private void executePutObjectAcl(HttpServletRequest request, HttpServletResponse response) throws IOException
{
S3PutObjectRequest putRequest = null;
// -> reuse the Access Control List parsing code that was added to support DIME
String bucketName = (String)request.getAttribute(S3Constants.BUCKET_ATTR_KEY);
String key = (String) request.getAttribute(S3Constants.OBJECT_ATTR_KEY);
try {
putRequest = S3RestServlet.toEnginePutObjectRequest( request.getInputStream());
}
catch( Exception e ) {
throw new IOException( e.toString());
}
// -> reuse the SOAP code to save the passed in ACLs
S3SetObjectAccessControlPolicyRequest engineRequest = new S3SetObjectAccessControlPolicyRequest();
engineRequest.setBucketName( bucketName );
engineRequest.setKey( key );
engineRequest.setAcl( putRequest.getAcl());
S3Response engineResponse = ServiceProvider.getInstance().getS3Engine().handleRequest(engineRequest);
response.setStatus( engineResponse.getResultCode());
}
private void executeGetObject(HttpServletRequest request, HttpServletResponse response) throws IOException
{
String bucket = (String) request.getAttribute(S3Constants.BUCKET_ATTR_KEY);
String key = (String) request.getAttribute(S3Constants.OBJECT_ATTR_KEY);
String[] paramList = null;
S3GetObjectRequest engineRequest = new S3GetObjectRequest();
engineRequest.setBucketName(bucket);
engineRequest.setKey(key);
engineRequest.setInlineData(true);
engineRequest.setReturnData(true);
//engineRequest.setReturnMetadata(true);
engineRequest = setRequestByteRange( request, engineRequest );
// -> is this a request for a specific version of the object? look for "versionId=" in the query string
String queryString = request.getQueryString();
if (null != queryString) {
paramList = queryString.split( "[&=]" );
if (null != paramList) engineRequest.setVersion( returnParameter( paramList, "versionId" ));
}
S3GetObjectResponse engineResponse = ServiceProvider.getInstance().getS3Engine().handleRequest( engineRequest );
response.setStatus( engineResponse.getResultCode());
String deleteMarker = engineResponse.getDeleteMarker();
if ( null != deleteMarker ) {
response.addHeader( "x-amz-delete-marker", "true" );
response.addHeader( "x-amz-version-id", deleteMarker );
}
else {
String version = engineResponse.getVersion();
if (null != version) response.addHeader( "x-amz-version-id", version );
}
// -> was the get conditional?
if (!conditionPassed( request, response, engineResponse.getLastModified().getTime(), engineResponse.getETag()))
return;
// -> is there data to return
// -> from the Amazon REST documentation it appears that Meta data is only returned as part of a HEAD request
//returnMetaData( engineResponse, response );
DataHandler dataHandler = engineResponse.getData();
if (dataHandler != null) {
response.addHeader("ETag", engineResponse.getETag());
response.addHeader("Last-Modified", DateHelper.getDateDisplayString(
DateHelper.GMT_TIMEZONE, engineResponse.getLastModified().getTime(), "E, d MMM yyyy HH:mm:ss z"));
response.setContentLength((int)engineResponse.getContentLength());
S3RestServlet.writeResponse(response, dataHandler.getInputStream());
}
}
private void executePutObject(HttpServletRequest request, HttpServletResponse response) throws IOException
{
String continueHeader = request.getHeader( "Expect" );
if (continueHeader != null && continueHeader.equalsIgnoreCase("100-continue")) {
S3RestServlet.writeResponse(response, "HTTP/1.1 100 Continue\r\n");
}
String contentType = request.getHeader( "Content-Type" );
long contentLength = Converter.toLong(request.getHeader("Content-Length"), 0);
String bucket = (String) request.getAttribute(S3Constants.BUCKET_ATTR_KEY);
String key = (String) request.getAttribute(S3Constants.OBJECT_ATTR_KEY);
S3PutObjectInlineRequest engineRequest = new S3PutObjectInlineRequest();
engineRequest.setBucketName(bucket);
engineRequest.setKey(key);
engineRequest.setContentLength(contentLength);
engineRequest.setMetaEntries( extractMetaData( request ));
engineRequest.setCannedAccess( request.getHeader( "x-amz-acl" ));
DataHandler dataHandler = new DataHandler(new ServletRequestDataSource(request));
engineRequest.setData(dataHandler);
S3PutObjectInlineResponse engineResponse = ServiceProvider.getInstance().getS3Engine().handleRequest(engineRequest);
response.setHeader("ETag", engineResponse.getETag());
String version = engineResponse.getVersion();
if (null != version) response.addHeader( "x-amz-version-id", version );
}
/**
* Once versioining is turned on then to delete an object requires specifying a version
* parameter. A deletion marker is set once versioning is turned on in a bucket.
*/
private void executeDeleteObject(HttpServletRequest request, HttpServletResponse response) throws IOException
{
String bucket = (String) request.getAttribute(S3Constants.BUCKET_ATTR_KEY);
String key = (String) request.getAttribute(S3Constants.OBJECT_ATTR_KEY);
String[] paramList = null;
S3DeleteObjectRequest engineRequest = new S3DeleteObjectRequest();
engineRequest.setBucketName(bucket);
engineRequest.setKey(key);
// -> is this a request for a specific version of the object? look for "versionId=" in the query string
String queryString = request.getQueryString();
if (null != queryString) {
paramList = queryString.split( "[&=]" );
if (null != paramList) engineRequest.setVersion( returnParameter( paramList, "versionId" ));
}
S3Response engineResponse = ServiceProvider.getInstance().getS3Engine().handleRequest( engineRequest );
response.setStatus( engineResponse.getResultCode());
String version = engineRequest.getVersion();
if (null != version) response.addHeader( "x-amz-version-id", version );
}
private void executeHeadObject(HttpServletRequest request, HttpServletResponse response) throws IOException
{
String bucket = (String) request.getAttribute(S3Constants.BUCKET_ATTR_KEY);
String key = (String) request.getAttribute(S3Constants.OBJECT_ATTR_KEY);
String[] paramList = null;
S3GetObjectRequest engineRequest = new S3GetObjectRequest();
engineRequest.setBucketName(bucket);
engineRequest.setKey(key);
engineRequest.setInlineData(true); // -> need to set so we get ETag etc returned
engineRequest.setReturnData(true);
engineRequest.setReturnMetadata(true);
engineRequest = setRequestByteRange( request, engineRequest );
// -> is this a request for a specific version of the object? look for "versionId=" in the query string
String queryString = request.getQueryString();
if (null != queryString) {
paramList = queryString.split( "[&=]" );
if (null != paramList) engineRequest.setVersion( returnParameter( paramList, "versionId" ));
}
S3GetObjectResponse engineResponse = ServiceProvider.getInstance().getS3Engine().handleRequest( engineRequest );
response.setStatus( engineResponse.getResultCode());
String deleteMarker = engineResponse.getDeleteMarker();
if ( null != deleteMarker ) {
response.addHeader( "x-amz-delete-marker", "true" );
response.addHeader( "x-amz-version-id", deleteMarker );
}
else {
String version = engineResponse.getVersion();
if (null != version) response.addHeader( "x-amz-version-id", version );
}
// -> was the head request conditional?
if (!conditionPassed( request, response, engineResponse.getLastModified().getTime(), engineResponse.getETag()))
return;
// -> for a head request we return everything except the data
returnMetaData( engineResponse, response );
DataHandler dataHandler = engineResponse.getData();
if (dataHandler != null) {
response.addHeader("ETag", engineResponse.getETag());
response.addHeader("Last-Modified", DateHelper.getDateDisplayString(
DateHelper.GMT_TIMEZONE, engineResponse.getLastModified().getTime(), "E, d MMM yyyy HH:mm:ss z"));
response.setContentLength((int)engineResponse.getContentLength());
}
}
// There is a problem with POST since the 'Signature' and 'AccessKey' parameters are not
// determined until we hit this function (i.e., they are encoded in the body of the message
// they are not HTTP request headers). All the values we used to get in the request headers
// are not encoded in the request body.
//
public void executePostObject( HttpServletRequest request, HttpServletResponse response ) throws IOException
{
String bucket = (String) request.getAttribute(S3Constants.BUCKET_ATTR_KEY);
String contentType = request.getHeader( "Content-Type" );
int boundaryIndex = contentType.indexOf( "boundary=" );
String boundary = "--" + (contentType.substring( boundaryIndex + 9 ));
String lastBoundary = boundary + "--";
InputStreamReader isr = new InputStreamReader( request.getInputStream());
BufferedReader br = new BufferedReader( isr );
StringBuffer temp = new StringBuffer();
String oneLine = null;
String name = null;
String value = null;
String metaName = null; // -> after stripped off the x-amz-meta-
boolean isMetaTag = false;
int countMeta = 0;
int state = 0;
// [A] First parse all the parts out of the POST request and message body
// -> bucket name is still encoded in a Host header
S3AuthParams params = new S3AuthParams();
List<S3MetaDataEntry> metaSet = new ArrayList<S3MetaDataEntry>();
S3PutObjectInlineRequest engineRequest = new S3PutObjectInlineRequest();
engineRequest.setBucketName( bucket );
// -> the last body part contains the content that is used to write the S3 object, all
// other body parts are header values
while( null != (oneLine = br.readLine()))
{
if ( oneLine.startsWith( lastBoundary ))
{
// -> this is the data of the object to put
if (0 < temp.length())
{
value = temp.toString();
temp.setLength( 0 );
engineRequest.setContentLength( value.length());
engineRequest.setDataAsString( value );
}
break;
}
else if ( oneLine.startsWith( boundary ))
{
// -> this is the header data
if (0 < temp.length())
{
value = temp.toString().trim();
temp.setLength( 0 );
//System.out.println( "param: " + name + " = " + value );
if (name.equalsIgnoreCase( "key" )) {
engineRequest.setKey( value );
}
else if (name.equalsIgnoreCase( "x-amz-acl" )) {
engineRequest.setCannedAccess( value );
}
else if (isMetaTag) {
S3MetaDataEntry oneMeta = new S3MetaDataEntry();
oneMeta.setName( metaName );
oneMeta.setValue( value );
metaSet.add( oneMeta );
countMeta++;
metaName = null;
}
// -> build up the headers so we can do authentication on this POST
HeaderParam oneHeader = new HeaderParam();
oneHeader.setName( name );
oneHeader.setValue( value );
params.addHeader( oneHeader );
}
state = 1;
}
else if (1 == state && 0 == oneLine.length())
{
// -> data of a body part starts here
state = 2;
}
else if (1 == state)
{
// -> the name of the 'name-value' pair is encoded in the Content-Disposition header
if (oneLine.startsWith( "Content-Disposition: form-data;"))
{
isMetaTag = false;
int nameOffset = oneLine.indexOf( "name=" );
if (-1 != nameOffset)
{
name = oneLine.substring( nameOffset+5 );
if (name.startsWith( "\"" )) name = name.substring( 1 );
if (name.endsWith( "\"" )) name = name.substring( 0, name.length()-1 );
name = name.trim();
if (name.startsWith( "x-amz-meta-" )) {
metaName = name.substring( 11 );
isMetaTag = true;
}
}
}
}
else if (2 == state)
{
// -> the body parts data may take up multiple lines
//System.out.println( oneLine.length() + " body data: " + oneLine );
temp.append( oneLine );
}
// else System.out.println( oneLine.length() + " preamble: " + oneLine );
}
// [B] Authenticate the POST request after we have all the headers
try {
S3RestServlet.authenticateRequest( request, params );
}
catch( Exception e ) {
throw new IOException( e.toString());
}
// [C] Perform the request
if (0 < countMeta) engineRequest.setMetaEntries( metaSet.toArray(new S3MetaDataEntry[0]));
S3PutObjectInlineResponse engineResponse = ServiceProvider.getInstance().getS3Engine().handleRequest( engineRequest );
response.setHeader("ETag", engineResponse.getETag());
String version = engineResponse.getVersion();
if (null != version) response.addHeader( "x-amz-version-id", version );
}
/**
* Support the "Range: bytes=0-399" header with just one byte range.
* @param request
* @param engineRequest
* @return
*/
private S3GetObjectRequest setRequestByteRange(HttpServletRequest request, S3GetObjectRequest engineRequest )
{
String temp = request.getHeader( "Range" );
if (null == temp) return engineRequest;
int offset = temp.indexOf( "=" );
if (-1 != offset)
{
String range = temp.substring( offset+1 );
String[] parts = range.split( "-" );
if (2 >= parts.length) {
// -> the end byte is inclusive
engineRequest.setByteRangeStart( Long.parseLong(parts[0]));
engineRequest.setByteRangeEnd( Long.parseLong(parts[1])+1);
}
}
return engineRequest;
}
private S3ConditionalHeaders conditionalRequest( HttpServletRequest request )
{
S3ConditionalHeaders headers = new S3ConditionalHeaders();
headers.setModifiedSince( request.getHeader( "If-Modified-Since" ));
headers.setUnModifiedSince( request.getHeader( "If-Unmodified-Since" ));
headers.setMatch( request.getHeader( "If-Match" ));
headers.setNoneMatch( request.getHeader( "If-None-Match" ));
return headers;
}
private boolean conditionPassed( HttpServletRequest request, HttpServletResponse response, Date lastModified, String ETag )
{
S3ConditionalHeaders ifCond = conditionalRequest( request );
if (0 > ifCond.ifModifiedSince( lastModified )) {
response.setStatus( 304 );
return false;
}
if (0 > ifCond.ifUnmodifiedSince( lastModified )) {
response.setStatus( 412 );
return false;
}
if (0 > ifCond.ifMatchEtag( ETag )) {
response.setStatus( 412 );
return false;
}
if (0 > ifCond.ifNoneMatchEtag( ETag )) {
response.setStatus( 412 );
return false;
}
return true;
}
/**
* Return the saved object's meta data back to the client as HTTP "x-amz-meta-" headers.
* This function is constructing an HTTP header and these headers have a defined syntax
* as defined in rfc2616. Any characters that could cause an invalid HTTP header will
* prevent that meta data from being returned via the REST call (as is defined in the Amazon
* spec). These characters can be defined if using the SOAP API as well as the REST API.
*
* @param engineResponse
* @param response
*/
private void returnMetaData( S3GetObjectResponse engineResponse, HttpServletResponse response )
{
boolean ignoreMeta = false;
int ignoredCount = 0;
S3MetaDataEntry[] metaSet = engineResponse.getMetaEntries();
for( int i=0; null != metaSet && i < metaSet.length; i++ )
{
String name = metaSet[i].getName();
String value = metaSet[i].getValue();
byte[] nameBytes = name.getBytes();
ignoreMeta = false;
// -> cannot have control characters (octets 0 - 31) and DEL (127), in an HTTP header
for( int j=0; j < name.length(); j++ ) {
if ((0 <= nameBytes[j] && 31 >= nameBytes[j]) || 127 == nameBytes[j]) {
ignoreMeta = true;
break;
}
}
// -> cannot have HTTP separators in an HTTP header
if (-1 != name.indexOf('(') || -1 != name.indexOf(')') || -1 != name.indexOf('@') ||
-1 != name.indexOf('<') || -1 != name.indexOf('>') || -1 != name.indexOf('\"') ||
-1 != name.indexOf('[') || -1 != name.indexOf(']') || -1 != name.indexOf('=') ||
-1 != name.indexOf(',') || -1 != name.indexOf(';') || -1 != name.indexOf(':') ||
-1 != name.indexOf('\\') || -1 != name.indexOf('/') || -1 != name.indexOf(' ') ||
-1 != name.indexOf('{') || -1 != name.indexOf('}') || -1 != name.indexOf('?') ||
-1 != name.indexOf('\t')
) ignoreMeta = true;
if ( ignoreMeta )
ignoredCount++;
else response.addHeader( "x-amz-meta-" + name, value );
}
if (0 < ignoredCount) response.addHeader( "x-amz-missing-meta", new String( "" + ignoredCount ));
}
/**
* Extract the name and value of all meta data so it can be written with the
* object that is being 'PUT'.
*
* @param request
* @return
*/
private S3MetaDataEntry[] extractMetaData( HttpServletRequest request )
{
List<S3MetaDataEntry> metaSet = new ArrayList<S3MetaDataEntry>();
int count = 0;
Enumeration headers = request.getHeaderNames();
while( headers.hasMoreElements())
{
String key = (String)headers.nextElement();
if (key.startsWith( "x-amz-meta-" ))
{
String name = key.substring( 11 );
String value = request.getHeader( key );
if (null != value) {
S3MetaDataEntry oneMeta = new S3MetaDataEntry();
oneMeta.setName( name );
oneMeta.setValue( value );
metaSet.add( oneMeta );
count++;
}
}
}
if ( 0 < count )
return metaSet.toArray(new S3MetaDataEntry[0]);
else return null;
}
/**
* @param paramList - name - value pairs with name at odd indexes
* @param find - name string to return first found
* @return the value matching the found name
*/
private String returnParameter( String[] paramList, String find )
{
int i=0;
if (paramList == null) return null;
while( i+2 <= paramList.length ) {
if (paramList[i].equalsIgnoreCase( find )) return paramList[ i+1 ];
i += 2;
}
return null;
}
}