/*
* Copyright 2010 Cloud.com, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.cloud.bridge.service.core.s3;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.ListIterator;
import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.UUID;
import javax.activation.DataHandler;
import org.apache.log4j.Logger;
import org.hibernate.LockMode;
import org.hibernate.Session;
import com.cloud.bridge.model.MHost;
import com.cloud.bridge.model.MHostMount;
import com.cloud.bridge.model.SAcl;
import com.cloud.bridge.model.SBucket;
import com.cloud.bridge.model.SHost;
import com.cloud.bridge.model.SMeta;
import com.cloud.bridge.model.SObject;
import com.cloud.bridge.model.SObjectItem;
import com.cloud.bridge.persist.PersistContext;
import com.cloud.bridge.persist.dao.MHostDao;
import com.cloud.bridge.persist.dao.MHostMountDao;
import com.cloud.bridge.persist.dao.SAclDao;
import com.cloud.bridge.persist.dao.SBucketDao;
import com.cloud.bridge.persist.dao.SHostDao;
import com.cloud.bridge.persist.dao.SMetaDao;
import com.cloud.bridge.persist.dao.SObjectDao;
import com.cloud.bridge.persist.dao.SObjectItemDao;
import com.cloud.bridge.service.S3BucketAdapter;
import com.cloud.bridge.service.S3FileSystemBucketAdapter;
import com.cloud.bridge.service.ServiceProvider;
import com.cloud.bridge.service.UserContext;
import com.cloud.bridge.service.exception.HostNotMountedException;
import com.cloud.bridge.service.exception.InternalErrorException;
import com.cloud.bridge.service.exception.NoSuchObjectException;
import com.cloud.bridge.service.exception.ObjectAlreadyExistsException;
import com.cloud.bridge.service.exception.OutOfServiceException;
import com.cloud.bridge.service.exception.OutOfStorageException;
import com.cloud.bridge.service.exception.PermissionDeniedException;
import com.cloud.bridge.service.exception.UnsupportedException;
import com.cloud.bridge.util.DateHelper;
import com.cloud.bridge.util.StringHelper;
import com.cloud.bridge.util.Tuple;
/**
* @author Kelven Yang
*/
public class S3Engine {
protected final static Logger logger = Logger.getLogger(S3Engine.class);
private final int LOCK_ACQUIRING_TIMEOUT_SECONDS = 10; // ten seconds
private final Map<Integer, S3BucketAdapter> bucketAdapters = new HashMap<Integer, S3BucketAdapter>();
public S3Engine() {
bucketAdapters.put(SHost.STORAGE_HOST_TYPE_LOCAL, new S3FileSystemBucketAdapter());
}
public S3CreateBucketResponse handleRequest(S3CreateBucketRequest request)
{
S3CreateBucketResponse response = new S3CreateBucketResponse();
response.setBucketName(request.getBucketName());
if (PersistContext.acquireNamedLock("bucket.creation", LOCK_ACQUIRING_TIMEOUT_SECONDS))
{
Tuple<SHost, String> shostTuple = null;
boolean success = false;
try {
SBucketDao bucketDao = new SBucketDao();
if(bucketDao.getByName(request.getBucketName()) != null)
throw new ObjectAlreadyExistsException("Bucket already exists");
shostTuple = allocBucketStorageHost(request.getBucketName());
SBucket sbucket = new SBucket();
sbucket.setName(request.getBucketName());
sbucket.setCreateTime(DateHelper.currentGMTTime());
sbucket.setOwnerCanonicalId( UserContext.current().getCanonicalUserId());
sbucket.setShost(shostTuple.getFirst());
shostTuple.getFirst().getBuckets().add(sbucket);
bucketDao.save(sbucket);
String cannedAccessPolicy = request.getCannedAccess();
if ( null != cannedAccessPolicy )
setCannedAccessControls( cannedAccessPolicy, "SBucket", sbucket.getId(), sbucket );
else setSingleAcl( "SBucket", sbucket.getId(), SAcl.PERMISSION_FULL );
// explicitly commit the transaction
PersistContext.commitTransaction();
success = true;
}
finally
{
if(!success && shostTuple != null) {
S3BucketAdapter bucketAdapter = getStorageHostBucketAdapter(shostTuple.getFirst());
bucketAdapter.deleteContainer(shostTuple.getSecond(), request.getBucketName());
}
PersistContext.releaseNamedLock("bucket.creation");
}
} else {
throw new OutOfServiceException("Unable to acquire synchronization lock");
}
return response;
}
public S3Response handleRequest( S3DeleteBucketRequest request )
{
S3Response response = new S3Response();
SBucketDao bucketDao = new SBucketDao();
SBucket sbucket = bucketDao.getByName(request.getBucketName());
if ( sbucket != null )
{
// -> does not matter what the ACLs say only the owner can delete a bucket
String client = UserContext.current().getCanonicalUserId();
if (!client.equals( sbucket.getOwnerCanonicalId()))
throw new PermissionDeniedException( "Access Denied - only the owner can delete a bucket" );
// -> delete the file
Tuple<SHost, String> tupleBucketHost = getBucketStorageHost(sbucket);
S3BucketAdapter bucketAdapter = getStorageHostBucketAdapter(tupleBucketHost.getFirst());
bucketAdapter.deleteContainer(tupleBucketHost.getSecond(), request.getBucketName());
// -> cascade-deleting can delete related SObject/SObjectItem objects, but not SAcl and SMeta objects. We
// need to perform deletion of these objects related to bucket manually.
// Delete SMeta & SAcl objects: (1)Get all the objects in the bucket, (2)then all the items in each object, (3) then all meta & acl data for each item
Set<SObject> objectsInBucket = sbucket.getObjectsInBucket();
Iterator it = objectsInBucket.iterator();
while( it.hasNext())
{
SObject oneObject = (SObject)it.next();
Set<SObjectItem> itemsInObject = oneObject.getItems();
Iterator is = itemsInObject.iterator();
while( is.hasNext())
{
SObjectItem oneItem = (SObjectItem)is.next();
deleteMetaData( oneItem.getId());
deleteObjectAcls( oneItem.getId());
}
}
deleteBucketAcls( sbucket.getId());
bucketDao.delete( sbucket );
response.setResultCode(204);
response.setResultDescription("OK");
}
else
{ response.setResultCode(404);
response.setResultDescription("Bucket does not exist");
}
return response;
}
public S3ListBucketResponse handleRequest(S3ListBucketRequest request)
{
S3ListBucketResponse response = new S3ListBucketResponse();
String bucketName = request.getBucketName();
String prefix = request.getPrefix();
if (prefix == null) prefix = StringHelper.EMPTY_STRING;
String marker = request.getMarker();
if (marker == null) marker = StringHelper.EMPTY_STRING;
String delimiter = request.getDelimiter();
int maxKeys = request.getMaxKeys();
if(maxKeys <= 0) maxKeys = 1000;
SBucketDao bucketDao = new SBucketDao();
SBucket sbucket = bucketDao.getByName(bucketName);
if (sbucket == null) throw new NoSuchObjectException("Bucket " + bucketName + " does not exist");
accessAllowed( "SBucket", sbucket.getId(), SAcl.PERMISSION_READ );
// when we query, request one more item so that we know how to set isTruncated flag
SObjectDao sobjectDao = new SObjectDao();
List<SObject> l = sobjectDao.listBucketObjects(sbucket, prefix, marker, maxKeys + 1);
response.setBucketName(bucketName);
response.setMarker(marker);
response.setMaxKeys(maxKeys);
response.setPrefix(prefix);
response.setDelimiter(delimiter);
response.setTruncated(l.size() > maxKeys);
if(l.size() > maxKeys) {
response.setNextMarker(l.get(l.size() - 1).getNameKey());
}
// SOAP response does not support versioning
response.setContents(composeListBucketContentEntries(l, prefix, delimiter, maxKeys, false));
response.setCommonPrefixes(composeListBucketPrefixEntries(l, prefix, delimiter, maxKeys));
return response;
}
public S3ListAllMyBucketsResponse handleRequest(S3ListAllMyBucketsRequest request)
{
S3ListAllMyBucketsResponse response = new S3ListAllMyBucketsResponse();
SBucketDao bucketDao = new SBucketDao();
List<SBucket> buckets = bucketDao.listBuckets(UserContext.current().getCanonicalUserId());
S3CanonicalUser owner = new S3CanonicalUser();
owner.setID(UserContext.current().getCanonicalUserId());
owner.setDisplayName("");
response.setOwner(owner);
if (buckets != null)
{
S3ListAllMyBucketsEntry[] entries = new S3ListAllMyBucketsEntry[buckets.size()];
int i = 0;
for(SBucket bucket : buckets) {
entries[i] = new S3ListAllMyBucketsEntry();
entries[i].setName(bucket.getName());
entries[i].setCreationDate(DateHelper.toCalendar(bucket.getCreateTime()));
i++;
}
response.setBuckets(entries);
}
return response;
}
public S3Response handleRequest(S3SetBucketAccessControlPolicyRequest request)
{
S3Response response = new S3Response();
SBucketDao bucketDao = new SBucketDao();
SBucket sbucket = bucketDao.getByName(request.getBucketName());
if(sbucket == null) {
response.setResultCode(404);
response.setResultDescription("Bucket does not exist");
return response;
}
accessAllowed( "SBucket", sbucket.getId(), SAcl.PERMISSION_WRITE_ACL );
SAclDao aclDao = new SAclDao();
aclDao.save("SBucket", sbucket.getId(), request.getAcl());
response.setResultCode(200);
response.setResultDescription("OK");
return response;
}
public S3AccessControlPolicy handleRequest(S3GetBucketAccessControlPolicyRequest request)
{
S3AccessControlPolicy policy = new S3AccessControlPolicy();
SBucketDao bucketDao = new SBucketDao();
SBucket sbucket = bucketDao.getByName(request.getBucketName());
if (sbucket == null)
throw new NoSuchObjectException("Bucket " + request.getBucketName() + " does not exist");
S3CanonicalUser owner = new S3CanonicalUser();
owner.setID(sbucket.getOwnerCanonicalId());
owner.setDisplayName("");
policy.setOwner(owner);
accessAllowed( "SBucket", sbucket.getId(), SAcl.PERMISSION_READ_ACL );
SAclDao aclDao = new SAclDao();
List<SAcl> grants = aclDao.listGrants("SBucket", sbucket.getId());
policy.setGrants(S3Grant.toGrants(grants));
return policy;
}
public S3PutObjectInlineResponse handleRequest(S3PutObjectInlineRequest request)
{
S3PutObjectInlineResponse response = new S3PutObjectInlineResponse();
String bucketName = request.getBucketName();
String key = request.getKey();
long contentLength = request.getContentLength();
S3MetaDataEntry[] meta = request.getMetaEntries();
S3AccessControlList acl = request.getAcl();
SBucketDao bucketDao = new SBucketDao();
SBucket bucket = bucketDao.getByName(bucketName);
if (bucket == null) throw new NoSuchObjectException("Bucket " + bucketName + " does not exist");
// -> is the caller allowed to write the object?
Tuple<SObject, SObjectItem> tupleObjectItem = allocObjectItem(bucket, key, meta, acl, request.getCannedAccess());
Tuple<SHost, String> tupleBucketHost = getBucketStorageHost(bucket);
S3BucketAdapter bucketAdapter = getStorageHostBucketAdapter(tupleBucketHost.getFirst());
String itemFileName = tupleObjectItem.getSecond().getStoredPath();
InputStream is = null;
try {
// explicit transaction control to avoid holding transaction during file-copy process
PersistContext.commitTransaction();
is = request.getDataInputStream();
String md5Checksum = bucketAdapter.saveObject(is, tupleBucketHost.getSecond(), bucket.getName(), itemFileName);
response.setETag(md5Checksum);
response.setLastModified(DateHelper.toCalendar( tupleObjectItem.getSecond().getLastModifiedTime()));
response.setVersion( tupleObjectItem.getSecond().getVersion());
SObjectItemDao itemDao = new SObjectItemDao();
SObjectItem item = itemDao.get( tupleObjectItem.getSecond().getId());
item.setMd5(md5Checksum);
item.setStoredSize(contentLength);
PersistContext.getSession().save(item);
} catch (IOException e) {
logger.error("PutObjectInline failed due to " + e.getMessage(), e);
} catch (OutOfStorageException e) {
logger.error("PutObjectInline failed due to " + e.getMessage(), e);
} finally {
if(is != null) {
try {
is.close();
} catch (IOException e) {
logger.error("Unable to close stream from data handler.", e);
}
}
}
return response;
}
public S3PutObjectResponse handleRequest(S3PutObjectRequest request)
{
S3PutObjectResponse response = new S3PutObjectResponse();
String bucketName = request.getBucketName();
String key = request.getKey();
long contentLength = request.getContentLength();
S3MetaDataEntry[] meta = request.getMetaEntries();
S3AccessControlList acl = request.getAcl();
SBucketDao bucketDao = new SBucketDao();
SBucket bucket = bucketDao.getByName(bucketName);
if(bucket == null) throw new NoSuchObjectException("Bucket " + bucketName + " does not exist");
// -> is the caller allowed to write the object?
Tuple<SObject, SObjectItem> tupleObjectItem = allocObjectItem(bucket, key, meta, acl, null);
Tuple<SHost, String> tupleBucketHost = getBucketStorageHost(bucket);
S3BucketAdapter bucketAdapter = getStorageHostBucketAdapter(tupleBucketHost.getFirst());
String itemFileName = tupleObjectItem.getSecond().getStoredPath();
InputStream is = null;
try {
// explicit transaction control to avoid holding transaction during file-copy process
PersistContext.commitTransaction();
is = request.getInputStream();
String md5Checksum = bucketAdapter.saveObject(is, tupleBucketHost.getSecond(), bucket.getName(), itemFileName);
response.setETag(md5Checksum);
response.setLastModified(DateHelper.toCalendar( tupleObjectItem.getSecond().getLastModifiedTime()));
SObjectItemDao itemDao = new SObjectItemDao();
SObjectItem item = itemDao.get( tupleObjectItem.getSecond().getId());
item.setMd5(md5Checksum);
item.setStoredSize(contentLength);
PersistContext.getSession().save(item);
} catch (OutOfStorageException e) {
logger.error("PutObject failed due to " + e.getMessage(), e);
} finally {
if(is != null) {
try {
is.close();
} catch (IOException e) {
logger.error("Unable to close stream from data handler.", e);
}
}
}
return response;
}
public S3Response handleRequest(S3SetObjectAccessControlPolicyRequest request)
{
S3Response response = new S3Response();
SBucketDao bucketDao = new SBucketDao();
SBucket sbucket = bucketDao.getByName(request.getBucketName());
if(sbucket == null) {
response.setResultCode(404);
response.setResultDescription("Bucket " + request.getBucketName() + "does not exist");
return response;
}
SObjectDao sobjectDao = new SObjectDao();
SObject sobject = sobjectDao.getByNameKey(sbucket, request.getKey());
if(sobject == null) {
response.setResultCode(404);
response.setResultDescription("Object " + request.getKey() + " in bucket " + request.getBucketName() + " does not exist");
return response;
}
accessAllowed( "SObject", sobject.getId(), SAcl.PERMISSION_WRITE_ACL );
SAclDao aclDao = new SAclDao();
aclDao.save("SObject", sobject.getId(), request.getAcl());
response.setResultCode(200);
response.setResultDescription("OK");
return response;
}
public S3AccessControlPolicy handleRequest(S3GetObjectAccessControlPolicyRequest request)
{
S3AccessControlPolicy policy = new S3AccessControlPolicy();
SBucketDao bucketDao = new SBucketDao();
SBucket sbucket = bucketDao.getByName(request.getBucketName());
if (sbucket == null)
throw new NoSuchObjectException("Bucket " + request.getBucketName() + " does not exist");
SObjectDao sobjectDao = new SObjectDao();
SObject sobject = sobjectDao.getByNameKey(sbucket, request.getKey());
if (sobject == null)
throw new NoSuchObjectException("Object " + request.getKey() + " does not exist");
S3CanonicalUser owner = new S3CanonicalUser();
owner.setID(sobject.getOwnerCanonicalId());
owner.setDisplayName("");
policy.setOwner(owner);
accessAllowed( "SObject", sobject.getId(), SAcl.PERMISSION_READ_ACL );
SAclDao aclDao = new SAclDao();
List<SAcl> grants = aclDao.listGrants("SObject", sobject.getId());
policy.setGrants(S3Grant.toGrants(grants));
return policy;
}
public S3GetObjectResponse handleRequest(S3GetObjectRequest request)
{
S3GetObjectResponse response = new S3GetObjectResponse();
int resultCode = 200;
// [A] Verify that the bucket and the object exist
SBucketDao bucketDao = new SBucketDao();
SBucket sbucket = bucketDao.getByName(request.getBucketName());
if (sbucket == null) {
response.setResultCode(404);
response.setResultDescription("Bucket " + request.getBucketName() + " does not exist");
return response;
}
SObjectDao objectDao = new SObjectDao();
SObject sobject = objectDao.getByNameKey(sbucket, request.getKey());
if (sobject == null) {
response.setResultCode(404);
response.setResultDescription("Object " + request.getKey() + " does not exist in bucket " + request.getBucketName());
return response;
}
String deletionMark = sobject.getDeletionMark();
if (null != deletionMark) {
response.setDeleteMarker( deletionMark );
response.setResultCode(404);
response.setResultDescription("Object " + request.getKey() + " has been deleted (1)");
return response;
}
// [B] Versioning allow the client to ask for a specific version not just the latest
SObjectItem item = null;
int versioningStatus = sbucket.getVersioningStatus();
String wantVersion = request.getVersion();
if ( SBucket.VERSIONING_ENABLED == versioningStatus && null != wantVersion)
item = sobject.getVersion( wantVersion );
else item = sobject.getLatestVersion(( SBucket.VERSIONING_ENABLED != versioningStatus ));
if (item == null) {
response.setResultCode(404);
response.setResultDescription("Object " + request.getKey() + " has been deleted (2)");
return response;
}
accessAllowed( "SObject", item.getId(), SAcl.PERMISSION_READ );
// -> extract the meta data that corresponds the specific versioned item
SMetaDao metaDao = new SMetaDao();
List<SMeta> itemMetaData = metaDao.getByTarget( "SObjectItem", item.getId());
if (null != itemMetaData)
{
int i = 0;
S3MetaDataEntry[] metaEntries = new S3MetaDataEntry[ itemMetaData.size() ];
ListIterator it = itemMetaData.listIterator();
while( it.hasNext()) {
SMeta oneTag = (SMeta)it.next();
S3MetaDataEntry oneEntry = new S3MetaDataEntry();
oneEntry.setName( oneTag.getName());
oneEntry.setValue( oneTag.getValue());
metaEntries[i++] = oneEntry;
}
response.setMetaEntries( metaEntries );
}
// TODO logic of IfMatch IfNoneMatch, IfModifiedSince, IfUnmodifiedSince (already done in the REST half)
// [C] Return the contents of the object inline
// -> support a single byte range
long bytesStart = request.getByteRangeStart();
long bytesEnd = request.getByteRangeEnd();
if ( 0 <= bytesStart && 0 <= bytesEnd) {
response.setContentLength( bytesEnd - bytesStart );
resultCode = 206;
}
else response.setContentLength( item.getStoredSize());
if(request.isReturnData())
{
response.setETag( item.getMd5());
response.setLastModified(DateHelper.toCalendar( item.getLastModifiedTime()));
response.setVersion( item.getVersion());
if (request.isInlineData())
{
Tuple<SHost, String> tupleSHostInfo = getBucketStorageHost(sbucket);
S3BucketAdapter bucketAdapter = getStorageHostBucketAdapter(tupleSHostInfo.getFirst());
if ( request.getByteRangeStart() >= 0 && request.getByteRangeEnd() >= 0)
response.setData(bucketAdapter.loadObjectRange(tupleSHostInfo.getSecond(),
request.getBucketName(), item.getStoredPath(), bytesStart, bytesEnd ));
else response.setData(bucketAdapter.loadObject(tupleSHostInfo.getSecond(), request.getBucketName(), item.getStoredPath()));
}
}
response.setResultCode( resultCode );
response.setResultDescription("OK");
return response;
}
/**
* In one place we handle both versioning and non-versioning delete requests.
*/
//@SuppressWarnings("deprecation")
public S3Response handleRequest(S3DeleteObjectRequest request)
{
// -> verify that the bucket and object exist
S3Response response = new S3Response();
SBucketDao bucketDao = new SBucketDao();
String bucketName = request.getBucketName();
SBucket sbucket = bucketDao.getByName( bucketName );
if (sbucket == null) {
response.setResultCode(404);
response.setResultDescription("Bucket does not exist");
return response;
}
SObjectDao objectDao = new SObjectDao();
SObject sobject = objectDao.getByNameKey( sbucket, request.getKey());
if (sobject == null) {
response.setResultCode(404);
response.setResultDescription("Bucket does not exist");
return response;
}
accessAllowed( "SBucket", sbucket.getId(), SAcl.PERMISSION_WRITE );
// -> versioning controls what delete means
String storedPath = null;
SObjectItem item = null;
int versioningStatus = sbucket.getVersioningStatus();
if ( SBucket.VERSIONING_ENABLED == versioningStatus )
{
String wantVersion = request.getVersion();
if (null == wantVersion) {
// -> if versioning is on and no versionId is given then we just write a deletion marker
sobject.setDeletionMark( UUID.randomUUID().toString());
objectDao.update( sobject );
}
else {
// -> are we removing the delete marker?
String deletionMarker = sobject.getDeletionMark();
if (null != deletionMarker && wantVersion.equalsIgnoreCase( deletionMarker )) {
sobject.setDeletionMark( null );
objectDao.update( sobject );
response.setResultCode(204);
return response;
}
// -> if versioning is on and the versionId is given then we delete the object matching that version
if ( null == (item = sobject.getVersion( wantVersion ))) {
response.setResultCode(404);
return response;
}
else {
// -> just delete the one item that matches the versionId from the database
storedPath = item.getStoredPath();
sobject.deleteItem( item.getId());
objectDao.update( sobject );
}
}
}
else {
// -> if versioning if off then we do delete the null object
if ( null == (item = sobject.getLatestVersion( true ))) {
response.setResultCode(404);
return response;
}
else {
// -> if no item with a null version then we are done
if (null == item.getVersion()) {
// -> remove the entire object
// -> cascade-deleting can delete related SObject/SObjectItem objects, but not SAcl and SMeta objects.
storedPath = item.getStoredPath();
deleteMetaData( item.getId());
deleteObjectAcls( item.getId());
objectDao.delete( sobject );
}
}
}
// -> delete the file holding the object
if (null != storedPath)
{
Tuple<SHost, String> tupleBucketHost = getBucketStorageHost( sbucket );
S3BucketAdapter bucketAdapter = getStorageHostBucketAdapter( tupleBucketHost.getFirst());
bucketAdapter.deleteObject( tupleBucketHost.getSecond(), bucketName, storedPath );
}
response.setResultCode(204);
return response;
}
private void deleteMetaData( long itemId ) {
SMetaDao metaDao = new SMetaDao();
List<SMeta> itemMetaData = metaDao.getByTarget( "SObjectItem", itemId );
if (null != itemMetaData)
{
ListIterator it = itemMetaData.listIterator();
while( it.hasNext()) {
SMeta oneTag = (SMeta)it.next();
metaDao.delete( oneTag );
}
}
}
private void deleteObjectAcls( long itemId ) {
SAclDao aclDao = new SAclDao();
List<SAcl> itemAclData = aclDao.listGrants( "SObject", itemId );
if (null != itemAclData)
{
ListIterator it = itemAclData.listIterator();
while( it.hasNext()) {
SAcl oneTag = (SAcl)it.next();
aclDao.delete( oneTag );
}
}
}
private void deleteBucketAcls( long bucketId ) {
SAclDao aclDao = new SAclDao();
List<SAcl> bucketAclData = aclDao.listGrants( "SBucket", bucketId );
if (null != bucketAclData)
{
ListIterator it = bucketAclData.listIterator();
while( it.hasNext()) {
SAcl oneTag = (SAcl)it.next();
aclDao.delete( oneTag );
}
}
}
private S3ListBucketPrefixEntry[] composeListBucketPrefixEntries(List<SObject> l, String prefix, String delimiter, int maxKeys)
{
List<S3ListBucketPrefixEntry> entries = new ArrayList<S3ListBucketPrefixEntry>();
int count = 0;
for(SObject sobject : l)
{
if(delimiter != null && !delimiter.isEmpty())
{
String subName = StringHelper.substringInBetween(sobject.getNameKey(), prefix, delimiter);
if(subName != null)
{
S3ListBucketPrefixEntry entry = new S3ListBucketPrefixEntry();
if ( prefix != null && prefix.length() > 0)
entry.setPrefix(prefix + delimiter + subName);
else entry.setPrefix(subName);
}
}
count++;
if(count >= maxKeys) break;
}
if(entries.size() > 0) return entries.toArray(new S3ListBucketPrefixEntry[0]);
return null;
}
private S3ListBucketObjectEntry[] composeListBucketContentEntries(List<SObject> l, String prefix, String delimiter, int maxKeys, boolean enableVersion)
{
List<S3ListBucketObjectEntry> entries = new ArrayList<S3ListBucketObjectEntry>();
int count = 0;
for(SObject sobject : l)
{
if (delimiter != null && !delimiter.isEmpty())
{
if (StringHelper.substringInBetween(sobject.getNameKey(), prefix, delimiter) != null)
continue;
}
if (enableVersion)
{
Iterator<SObjectItem> it = sobject.getItems().iterator();
while(it.hasNext())
{
// TODO, should add version info
SObjectItem item = (SObjectItem)it.next();
entries.add(toListEntry(sobject, item));
}
}
else {
// this should be able to be optimized
Iterator<SObjectItem> it = sobject.getItems().iterator();
SObjectItem lastestItem = null;
int maxVersion = 0;
while(it.hasNext())
{
SObjectItem item = (SObjectItem)it.next();
int version = Integer.parseInt(item.getVersion());
if(version > maxVersion) {
maxVersion = version;
lastestItem = item;
}
}
if (lastestItem != null) {
entries.add(toListEntry(sobject, lastestItem));
}
}
count++;
if(count >= maxKeys) break;
}
if(entries.size() > 0) return entries.toArray(new S3ListBucketObjectEntry[0]);
return null;
}
private static S3ListBucketObjectEntry toListEntry(SObject sobject, SObjectItem item)
{
// TODO, should add version info
S3ListBucketObjectEntry entry = new S3ListBucketObjectEntry();
entry.setKey(sobject.getNameKey());
entry.setETag(item.getMd5());
entry.setSize(item.getStoredSize());
entry.setLastModified(DateHelper.toCalendar(item.getLastModifiedTime()));
entry.setOwnerCanonicalId(sobject.getOwnerCanonicalId());
entry.setOwnerDisplayName("");
return entry;
}
public Tuple<SHost, String> getBucketStorageHost(SBucket bucket)
{
MHostMountDao mountDao = new MHostMountDao();
SHost shost = bucket.getShost();
if(shost.getHostType() == SHost.STORAGE_HOST_TYPE_LOCAL) {
return new Tuple<SHost, String>(shost, shost.getExportRoot());
}
MHostMount mount = mountDao.getHostMount(ServiceProvider.getInstance().getManagementHostId(), shost.getId());
if(mount != null) {
return new Tuple<SHost, String>(shost, mount.getMountPath());
}
// need to redirect request to other node
throw new HostNotMountedException("Storage host " + shost.getHost() + " is not locally mounted");
}
private Tuple<SHost, String> allocBucketStorageHost(String bucketName)
{
MHostDao mhostDao = new MHostDao();
SHostDao shostDao = new SHostDao();
MHost mhost = mhostDao.get(ServiceProvider.getInstance().getManagementHostId());
if(mhost == null)
throw new OutOfServiceException("Temporarily out of service");
if(mhost.getMounts().size() > 0) {
Random random = new Random();
MHostMount[] mounts = (MHostMount[])mhost.getMounts().toArray();
MHostMount mount = mounts[random.nextInt(mounts.length)];
S3BucketAdapter bucketAdapter = getStorageHostBucketAdapter(mount.getShost());
bucketAdapter.createContainer(mount.getMountPath(), bucketName);
return new Tuple<SHost, String>(mount.getShost(), mount.getMountPath());
}
// To make things simple, only allow one local mounted storage root
String localStorageRoot = ServiceProvider.getInstance().getStartupProperties().getProperty("storage.root");
if(localStorageRoot != null) {
SHost localSHost = shostDao.getLocalStorageHost(mhost.getId(), localStorageRoot);
if(localSHost == null)
throw new InternalErrorException("storage.root is configured but not initialized");
S3BucketAdapter bucketAdapter = getStorageHostBucketAdapter(localSHost);
bucketAdapter.createContainer(localSHost.getExportRoot(), bucketName);
return new Tuple<SHost, String>(localSHost, localStorageRoot);
}
throw new OutOfStorageException("No storage host is available");
}
public S3BucketAdapter getStorageHostBucketAdapter(SHost shost)
{
S3BucketAdapter adapter = bucketAdapters.get(shost.getHostType());
if(adapter == null)
throw new InternalErrorException("Bucket adapter is not installed for host type: " + shost.getHostType());
return adapter;
}
/**
* If acl is set then the cannedAccessPolicy parameter should be null and is ignored.
* The cannedAccessPolicy parameter is for REST Put requests only where a simple set of ACLs can be
* created with a single header value. Note that we do not currently support "anonymous" un-authenticated
* access in our implementation.
*
* @throws IOException
*/
@SuppressWarnings("deprecation")
public Tuple<SObject, SObjectItem> allocObjectItem(SBucket bucket, String nameKey, S3MetaDataEntry[] meta, S3AccessControlList acl, String cannedAccessPolicy)
{
SObjectDao objectDao = new SObjectDao();
SObjectItemDao objectItemDao = new SObjectItemDao();
SMetaDao metaDao = new SMetaDao();
SAclDao aclDao = new SAclDao();
SObjectItem item = null;
boolean newVersion = false;
int versionSeq = 1;
int versioningStatus = bucket.getVersioningStatus();
Session session = PersistContext.getSession();
// [A] If versoning is off them we over write a null object item
SObject object = objectDao.getByNameKey(bucket, nameKey);
if ( object != null )
{
accessAllowed( "SObject", object.getId(), SAcl.PERMISSION_WRITE );
// -> if versioning is on create new object items
if ( SBucket.VERSIONING_ENABLED == versioningStatus )
{
session.lock(object, LockMode.UPGRADE);
versionSeq = object.getNextSequence();
object.setNextSequence(versionSeq + 1);
session.save(object);
item = new SObjectItem();
item.setTheObject(object);
object.getItems().add(item);
item.setVersion(String.valueOf(versionSeq));
Date ts = DateHelper.currentGMTTime();
item.setCreateTime(ts);
item.setLastAccessTime(ts);
item.setLastModifiedTime(ts);
session.save(item);
newVersion = true;
}
else
{ // -> find an object item with a null version, can be null
// if bucket started out with versioning enabled and was then suspended
item = objectItemDao.getByObjectIdNullVersion( object.getId());
if (item == null)
{
item = new SObjectItem();
item.setTheObject(object);
object.getItems().add(item);
Date ts = DateHelper.currentGMTTime();
item.setCreateTime(ts);
item.setLastAccessTime(ts);
item.setLastModifiedTime(ts);
session.save(item);
}
}
}
else
{ // -> there is no object nor an object item
// -> to create a new object in a bucket then we need write access to that bucket
accessAllowed( "SBucket", bucket.getId(), SAcl.PERMISSION_WRITE );
object = new SObject();
object.setBucket(bucket);
object.setNameKey(nameKey);
object.setNextSequence(2);
object.setCreateTime(DateHelper.currentGMTTime());
object.setOwnerCanonicalId(UserContext.current().getCanonicalUserId());
session.save(object);
item = new SObjectItem();
item.setTheObject(object);
object.getItems().add(item);
if (SBucket.VERSIONING_ENABLED == versioningStatus) item.setVersion(String.valueOf(versionSeq));
Date ts = DateHelper.currentGMTTime();
item.setCreateTime(ts);
item.setLastAccessTime(ts);
item.setLastModifiedTime(ts);
session.save(item);
}
// [B] We will use the item DB id as the file name, MD5/contentLength will be stored later
String suffix = null;
int dotPos = nameKey.lastIndexOf('.');
if (dotPos >= 0) suffix = nameKey.substring(dotPos);
if ( suffix != null )
item.setStoredPath(String.valueOf(item.getId()) + suffix);
else item.setStoredPath(String.valueOf(item.getId()));
metaDao.save("SObjectItem", item.getId(), meta);
// [C] Are we setting an ACL along with the object
if ( null != cannedAccessPolicy )
{
setCannedAccessControls( cannedAccessPolicy, "SObject", object.getId(), bucket );
}
else if ((null == acl || 0 == acl.size()) && !newVersion )
{
// -> this is termed the "private" or default ACL, "Owner gets FULL_CONTROL"
setSingleAcl( "SObject", object.getId(), SAcl.PERMISSION_FULL );
}
else if (null != acl) {
aclDao.save( "SObject", object.getId(), acl );
}
session.update(item);
return new Tuple<SObject, SObjectItem>(object, item);
}
/**
* Access controls that are specified via the "x-amz-acl:" headers in REST requests.
* Note that canned policies can be set when the object's contents are set
*/
private void setCannedAccessControls( String cannedAccessPolicy, String target, long objectId, SBucket bucket )
{
if ( cannedAccessPolicy.equalsIgnoreCase( "public-read" ))
{
// -> owner gets FULL_CONTROL and the anonymous principal (the 'A' symbol here) is granted READ access.
setDefaultAcls( target, objectId, SAcl.PERMISSION_FULL, SAcl.PERMISSION_READ, "A" );
}
else if (cannedAccessPolicy.equalsIgnoreCase( "public-read-write" ))
{
// -> owner gets FULL_CONTROL and the anonymous principal (the 'A' symbol here) is granted READ and WRITE access
setDefaultAcls( target, objectId, SAcl.PERMISSION_FULL, (SAcl.PERMISSION_READ | SAcl.PERMISSION_WRITE), "A" );
}
else if (cannedAccessPolicy.equalsIgnoreCase( "authenticated-read" ))
{
// -> Owner gets FULL_CONTROL and ANY principal authenticated as a registered S3 user (the '*' symbol here) is granted READ access
setDefaultAcls( target, objectId, SAcl.PERMISSION_FULL, SAcl.PERMISSION_READ, "*" );
}
else if (cannedAccessPolicy.equalsIgnoreCase( "private" ))
{
// -> this is termed the "private" or default ACL, "Owner gets FULL_CONTROL"
setSingleAcl( target, objectId, SAcl.PERMISSION_FULL );
}
else if (cannedAccessPolicy.equalsIgnoreCase( "bucket-owner-read" ))
{
// -> Object Owner gets FULL_CONTROL, Bucket Owner gets READ
// -> is equivalent to private when used with PUT Bucket
if ( target.equalsIgnoreCase( "SBucket" ))
setSingleAcl( target, objectId, SAcl.PERMISSION_FULL );
else setDefaultAcls( target, objectId, SAcl.PERMISSION_FULL, SAcl.PERMISSION_READ, bucket.getOwnerCanonicalId());
}
else if (cannedAccessPolicy.equalsIgnoreCase( "bucket-owner-full-control" ))
{
// -> Object Owner gets FULL_CONTROL, Bucket Owner gets FULL_CONTROL
// -> is equivalent to private when used with PUT Bucket
if ( target.equalsIgnoreCase( "SBucket" ))
setSingleAcl( target, objectId, SAcl.PERMISSION_FULL );
else setDefaultAcls( target, objectId, SAcl.PERMISSION_FULL, SAcl.PERMISSION_FULL, bucket.getOwnerCanonicalId());
}
else throw new UnsupportedException( "Unknown Canned Access Policy: " + cannedAccessPolicy + " is not supported" );
}
private void setSingleAcl( String target, long targetId, int permission )
{
SAclDao aclDao = new SAclDao();
S3AccessControlList defaultAcl = new S3AccessControlList();
// -> if an annoymous request, then do not rewrite the ACL
String userId = UserContext.current().getCanonicalUserId();
if (0 < userId.length())
{
S3Grant defaultGrant = new S3Grant();
defaultGrant.setGrantee(SAcl.GRANTEE_USER);
defaultGrant.setCanonicalUserID( userId);
defaultGrant.setPermission( permission );
defaultAcl.addGrant( defaultGrant );
aclDao.save( target, targetId, defaultAcl );
}
}
/**
* Note that we use the Cloud Stack API Access key for the Canonical User Id everywhere
* (i.e., for buckets, and objects).
*
* @param owner - this can be the Cloud Access Key for a bucket owner or one of the
* following special symbols:
* (a) '*' - any principal authenticated user (i.e., any user with a registered Cloud Access Key)
* (b) 'A' - any anonymous principal (i.e., S3 request without an Authorization header)
*/
private void setDefaultAcls( String target, long objectId, int permission1, int permission2, String owner )
{
SAclDao aclDao = new SAclDao();
S3AccessControlList defaultAcl = new S3AccessControlList();
// -> object owner
S3Grant defaultGrant = new S3Grant();
defaultGrant.setGrantee(SAcl.GRANTEE_USER);
defaultGrant.setCanonicalUserID( UserContext.current().getCanonicalUserId());
defaultGrant.setPermission( permission1 );
defaultAcl.addGrant( defaultGrant );
// -> bucket owner
defaultGrant = new S3Grant();
defaultGrant.setGrantee(SAcl.GRANTEE_USER);
defaultGrant.setCanonicalUserID( owner );
defaultGrant.setPermission( permission2 );
defaultAcl.addGrant( defaultGrant );
aclDao.save( target, objectId, defaultAcl );
}
/**
* This function verifies that the accessing client has the requested
* premission on the object/bucket/Acl represented by the tuble: <target, targetId>
* For cases where an ACL is meant for any authenticated user we place a "*" for the
* Canonical User Id ("*" is not a legal Cloud Stack Access key). For cases where
* an ACL is meant for any anonymous user we place a "A" for the Canonical User Id ("A"
* is not a legal Cloud Stack Access key).
*/
public void accessAllowed( String target, long targetId, int requestedPermission ) {
SAclDao aclDao = new SAclDao();
// -> if an annoymous request, then canonicalUserId is an empty string
String userId = UserContext.current().getCanonicalUserId();
if ( 0 == userId.length())
{
// -> is an anonymous principal ACL set for this <target, targetId>?
if (hasPermission( aclDao.listGrants( target, targetId, "A" ), requestedPermission )) return;
}
else
{ // -> no priviledges means no access allowed
if (hasPermission( aclDao.listGrants( target, targetId, userId ), requestedPermission )) return;
// -> or maybe there is any principal authenticated ACL set for this <target, targetId>?
if (hasPermission( aclDao.listGrants( target, targetId, "*" ), requestedPermission )) return;
}
throw new PermissionDeniedException( "Access Denied - user does not have the required permission" );
}
private boolean hasPermission( List<SAcl> priviledges, int requestedPermission )
{
ListIterator it = priviledges.listIterator();
while( it.hasNext())
{
// -> is the requested permission "contained" in one or the granted rights for this user
SAcl rights = (SAcl)it.next();
int permission = rights.getPermission();
if (requestedPermission == (permission & requestedPermission)) return true;
}
return false;
}
}