/*
* Copyright 2015 Bounce Storage, Inc. <info@bouncestorage.com>
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.bouncestorage.swiftproxy.v1;
import static java.util.Objects.requireNonNull;
import static com.bouncestorage.swiftproxy.Utils.eTagsEqual;
import static com.google.common.base.Throwables.propagate;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
import java.net.URLDecoder;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.Spliterator;
import java.util.Spliterators;
import java.util.StringTokenizer;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import java.util.stream.StreamSupport;
import javax.validation.constraints.NotNull;
import javax.ws.rs.BadRequestException;
import javax.ws.rs.ClientErrorException;
import javax.ws.rs.Consumes;
import javax.ws.rs.DELETE;
import javax.ws.rs.Encoded;
import javax.ws.rs.GET;
import javax.ws.rs.HEAD;
import javax.ws.rs.HeaderParam;
import javax.ws.rs.POST;
import javax.ws.rs.PUT;
import javax.ws.rs.Path;
import javax.ws.rs.PathParam;
import javax.ws.rs.QueryParam;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.HttpHeaders;
import javax.ws.rs.core.MediaType;
import javax.ws.rs.core.Response;
import com.bouncestorage.swiftproxy.BlobStoreResource;
import com.bouncestorage.swiftproxy.COPY;
import com.bouncestorage.swiftproxy.Utils;
import com.fasterxml.jackson.annotation.JsonProperty;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Joiner;
import com.google.common.base.Objects;
import com.google.common.base.Strings;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterators;
import com.google.common.collect.Multimap;
import com.google.common.collect.PeekingIterator;
import com.google.common.hash.HashCode;
import com.google.common.hash.Hasher;
import com.google.common.hash.Hashing;
import com.google.common.io.BaseEncoding;
import org.apache.commons.io.input.TeeInputStream;
import org.glassfish.grizzly.http.server.Request;
import org.glassfish.grizzly.utils.Pair;
import org.jclouds.blobstore.BlobStore;
import org.jclouds.blobstore.ContainerNotFoundException;
import org.jclouds.blobstore.domain.Blob;
import org.jclouds.blobstore.domain.BlobBuilder;
import org.jclouds.blobstore.domain.BlobMetadata;
import org.jclouds.blobstore.domain.StorageMetadata;
import org.jclouds.blobstore.options.CopyOptions;
import org.jclouds.blobstore.options.GetOptions;
import org.jclouds.blobstore.options.ListContainerOptions;
import org.jclouds.http.HttpResponse;
import org.jclouds.http.HttpResponseException;
import org.jclouds.io.ContentMetadata;
import org.jclouds.io.ContentMetadataBuilder;
import org.jclouds.io.MutableContentMetadata;
import org.jclouds.io.payloads.InputStreamPayload;
import org.jclouds.openstack.swift.v1.reference.SwiftHeaders;
@Path("/v1/{account}/{container}/{object:.*}")
public final class ObjectResource extends BlobStoreResource {
private static final String META_HEADER_PREFIX = "X-Object-Meta-";
private static final String DYNAMIC_OBJECT_MANIFEST = "x-object-manifest";
private static final String STATIC_OBJECT_MANIFEST = "x-static-large-object";
private static final Set<String> RESERVED_METADATA = ImmutableSet.of(
DYNAMIC_OBJECT_MANIFEST,
STATIC_OBJECT_MANIFEST
);
private static final MediaType MANIFEST_CONTENT_TYPE = MediaType.APPLICATION_JSON_TYPE.withCharset("utf-8");
private static final Set<String> STD_BLOB_HEADERS = ImmutableSet.of(
"Content-Range"
);
private List<Pair<Long, Long>> parseRange(String range) {
range = range.replaceAll(" ", "").toLowerCase();
String bytesUnit = "bytes=";
int idx = range.indexOf(bytesUnit);
if (idx == 0) {
String byteRangeSet = range.substring(bytesUnit.length());
Iterator<Object> iter = Iterators.forEnumeration(new StringTokenizer(byteRangeSet, ","));
return StreamSupport.stream(Spliterators.spliteratorUnknownSize(iter, Spliterator.ORDERED), false)
.map(rangeSpec -> (String) rangeSpec)
.map(rangeSpec -> {
int dash = rangeSpec.indexOf("-");
if (dash == -1) {
throw new BadRequestException("Range");
}
String firstBytePos = rangeSpec.substring(0, dash);
String lastBytePos = rangeSpec.substring(dash + 1);
Long firstByte = firstBytePos.isEmpty() ? null : Long.parseLong(firstBytePos);
Long lastByte = lastBytePos.isEmpty() ? null : Long.parseLong(lastBytePos);
return new Pair<>(firstByte, lastByte);
})
.peek(r -> logger.debug("parsed range {} {}", r.getFirst(), r.getSecond()))
.collect(Collectors.toList());
} else {
return null;
}
}
private static GetOptions addRanges(GetOptions options, List<Pair<Long, Long>> ranges) {
ranges.forEach(rangeSpec -> {
if (rangeSpec.getFirst() == null) {
if (rangeSpec.getSecond() == 0) {
throw new ClientErrorException(Response.Status.REQUESTED_RANGE_NOT_SATISFIABLE);
}
options.tail(rangeSpec.getSecond());
} else if (rangeSpec.getSecond() == null) {
options.startAt(rangeSpec.getFirst());
} else {
options.range(rangeSpec.getFirst(), rangeSpec.getSecond());
}
});
return options;
}
private static String maybeUnquote(String quoted) {
if (quoted.startsWith("\"") && quoted.endsWith("\"")) {
return quoted.substring(1, quoted.length() - 1);
}
return quoted;
}
@GET
public Response getObject(@NotNull @PathParam("container") String container,
@NotNull @Encoded @PathParam("object") String object,
@NotNull @PathParam("account") String account,
@HeaderParam("X-Auth-Token") String authToken,
@HeaderParam("X-Newest") boolean newest,
@QueryParam("signature") String signature,
@QueryParam("expires") String expires,
@QueryParam("multipart-manifest") String multiPartManifest,
@HeaderParam("Range") String range,
@HeaderParam("If-Match") String ifMatch,
@HeaderParam("If-None-Match") String ifNoneMatch,
@HeaderParam("If-Modified-Since") Date ifModifiedSince,
@HeaderParam("If-Unmodified-Since") Date ifUnmodifiedSince) {
logger.debug("GET account={} container={} object={}", account, container, object);
BlobStore blobStore = getBlobStore(authToken).get(container, object);
if (!blobStore.containerExists(container)) {
return notFound();
}
GetOptions options = new GetOptions();
List<Pair<Long, Long>> ranges = null;
if (range != null) {
ranges = parseRange(range);
if (ranges != null) {
options = addRanges(options, ranges);
}
}
if (ifMatch != null) {
options.ifETagMatches(maybeUnquote(ifMatch));
}
if (ifNoneMatch != null) {
options.ifETagDoesntMatch(maybeUnquote(ifNoneMatch));
}
if (ifModifiedSince != null) {
options.ifModifiedSince(ifModifiedSince);
}
if (ifUnmodifiedSince != null) {
options.ifUnmodifiedSince(ifUnmodifiedSince);
}
return getObject(blobStore, container, object, options, ranges, "get".equals(multiPartManifest));
}
private Map<String, Object> blobGetStandardHeaders(Blob blob) {
ImmutableMap.Builder<String, Object> builder = ImmutableMap.builder();
Multimap<String, String> headers = blob.getAllHeaders();
for (String h : STD_BLOB_HEADERS) {
if (headers.containsKey(h)) {
builder.put(h, headers.get(h).iterator().next());
}
}
return builder.build();
}
private Response conditionalGetSatisified(GetOptions options, String etag, Date mtime) {
if (options.getIfMatch() != null && !eTagsEqual(etag, options.getIfMatch())) {
return Response.status(Response.Status.PRECONDITION_FAILED).build();
}
if (options.getIfNoneMatch() != null && eTagsEqual(etag, options.getIfNoneMatch())) {
return Response.notModified().build();
}
if (options.getIfModifiedSince() != null && mtime.compareTo(options.getIfModifiedSince()) <= 0) {
return Response.notModified().build();
}
if (options.getIfUnmodifiedSince() != null && mtime.compareTo(options.getIfUnmodifiedSince()) > 0) {
return Response.status(Response.Status.PRECONDITION_FAILED).build();
}
return null;
}
private Response getObject(BlobStore blobStore, String container, String object,
GetOptions options, List<Pair<Long, Long>> ranges, boolean multiPartManifest) {
Blob blob = null;
BlobMetadata meta;
if (GetOptions.NONE.equals(options) || multiPartManifest) {
blob = blobStore.getBlob(container, object, options);
if (blob == null) {
return Response.status(Response.Status.NOT_FOUND).build();
}
meta = blob.getMetadata();
} else {
logger.debug("range get, check to see if object is a large object");
meta = blobStore.blobMetadata(container, object);
if (meta == null) {
return Response.status(Response.Status.NOT_FOUND).build();
}
}
boolean isMultiPartManifest = false;
if (meta.getUserMetadata().containsKey(DYNAMIC_OBJECT_MANIFEST)) {
isMultiPartManifest = true;
if (!multiPartManifest) {
return getDloObject(blobStore, meta, options, ranges);
}
} else if (meta.getUserMetadata().containsKey(STATIC_OBJECT_MANIFEST)) {
isMultiPartManifest = true;
if (!multiPartManifest) {
if (blob == null) {
String sloData = meta.getUserMetadata().get(STATIC_OBJECT_MANIFEST);
String[] data = sloData.split(" ", 2);
Response cond = conditionalGetSatisified(options,
data[1], meta.getLastModified());
if (cond != null) {
return cond;
}
}
if (blob == null) {
blob = blobStore.getBlob(container, object);
}
return getSloObject(blobStore, blob, options, ranges);
}
} else if (blob == null) {
// this is just a normal blob
try {
blob = blobStore.getBlob(container, object, options);
} catch (IllegalArgumentException e) {
if (ranges != null) {
throw requestRangeNotSatisfiable();
} else {
throw e;
}
}
meta = blob.getMetadata();
}
try {
return addObjectHeaders(Response.ok(blob.getPayload().openStream()), meta,
isMultiPartManifest ?
Optional.of(ImmutableMap.of(HttpHeaders.CONTENT_TYPE, MANIFEST_CONTENT_TYPE)) :
Optional.of(blobGetStandardHeaders(blob)))
.build();
} catch (IOException e) {
throw propagate(e);
}
}
private ClientErrorException requestRangeNotSatisfiable() {
throw new ClientErrorException(Response.Status.REQUESTED_RANGE_NOT_SATISFIABLE);
}
private long getTotalRangesLength(List<Pair<Long, Long>> ranges, long totalSize) {
return ranges.stream().mapToLong(r -> {
if (r.getFirst() == null) {
if (r.getSecond() > totalSize) {
throw requestRangeNotSatisfiable();
}
return r.getSecond();
} else {
if (r.getFirst() >= totalSize) {
throw requestRangeNotSatisfiable();
}
if (r.getSecond() == null) {
return totalSize - r.getFirst();
} else {
return r.getSecond() - r.getFirst() + 1;
}
}
}).sum();
}
private Response getSloObject(BlobStore blobStore, Blob blob, GetOptions options, List<Pair<Long, Long>> ranges) {
try {
Iterable<ManifestEntry> entries = Arrays.asList(readSLOManifest(blob.getPayload().openStream()));
Pair<Long, String> sizeAndEtag = getManifestTotalSizeAndETag(entries);
logger.debug("getting SLO object: {}", sizeAndEtag);
entries.forEach(e -> logger.debug("sub-object: {}", e));
InputStream combined = new ManifestObjectInputStream(blobStore, entries);
long size = sizeAndEtag.getFirst();
if (ranges != null) {
combined = new HttpRangeInputStream(combined, sizeAndEtag.getFirst(), ranges);
size = getTotalRangesLength(ranges, size);
logger.debug("range request for {} bytes", size);
}
return addObjectHeaders(Response.ok(combined), blob.getMetadata(),
Optional.of(overwriteSizeAndETag(size, sizeAndEtag.getSecond())))
.build();
} catch (IOException e) {
throw propagate(e);
}
}
private Response getDloObject(BlobStore blobStore, BlobMetadata meta, GetOptions options, List<Pair<Long, Long>> ranges) {
String manifest = meta.getUserMetadata().get(DYNAMIC_OBJECT_MANIFEST);
Pair<String, String> param = validateCopyParam(manifest);
String dloContainer = param.getFirst();
String objectsPrefix = param.getSecond();
List<ManifestEntry> segments = getDLOSegments(blobStore, dloContainer, objectsPrefix);
Pair<Long, String> sizeAndEtag = getManifestTotalSizeAndETag(segments);
Response cond = conditionalGetSatisified(options,
sizeAndEtag.getSecond(), meta.getLastModified());
if (cond != null) {
return cond;
}
logger.debug("getting DLO object: {}", sizeAndEtag);
InputStream combined = new ManifestObjectInputStream(blobStore, segments);
long size = sizeAndEtag.getFirst();
if (ranges != null) {
combined = new HttpRangeInputStream(combined, sizeAndEtag.getFirst(), ranges);
size = getTotalRangesLength(ranges, size);
logger.debug("range request for {} bytes", size);
}
return addObjectHeaders(Response.ok(combined), meta,
Optional.of(overwriteSizeAndETag(size, sizeAndEtag.getSecond())))
.build();
}
private List<ManifestEntry> getDLOSegments(BlobStore blobStore, String container, String objectsPrefix) {
ListContainerOptions listOptions = new ListContainerOptions()
.recursive()
.prefix(objectsPrefix);
logger.debug("dlo prefix: {}", objectsPrefix);
Iterable<StorageMetadata> res = Utils.crawlBlobStore(blobStore, container, listOptions);
List<ManifestEntry> segments = new ArrayList<>();
for (StorageMetadata sm : res) {
if (sm.getName().startsWith(objectsPrefix)) {
ManifestEntry entry = new ManifestEntry();
entry.container = container;
entry.object = sm.getName();
entry.size_bytes = sm.getSize();
entry.etag = sm.getETag();
segments.add(entry);
} else {
throw new IllegalStateException(
String.format("list object %s from prefix %s", sm.getName(), objectsPrefix));
}
}
segments.forEach(e -> logger.debug("sub-object: {}", e));
return segments;
}
private static String normalizePath(String pathName) {
String objectName = Joiner.on("/").join(Iterators.forEnumeration(new StringTokenizer(pathName, "/")));
if (pathName.endsWith("/")) {
objectName += "/";
}
return objectName;
}
private Map<String, String> getUserMetadata(Request request) {
return StreamSupport.stream(request.getHeaderNames().spliterator(), false)
.filter(name -> name.toLowerCase().startsWith(META_HEADER_PREFIX.toLowerCase()))
.filter(name -> {
if (name.equalsIgnoreCase(META_HEADER_PREFIX) || RESERVED_METADATA.contains(name)) {
throw new BadRequestException();
}
if (name.length() - META_HEADER_PREFIX.length() > InfoResource.CONFIG.swift.max_meta_name_length ||
request.getHeader(name).length() > InfoResource.CONFIG.swift.max_meta_value_length) {
throw new BadRequestException();
}
return true;
})
.collect(Collectors.toMap(
name -> name.substring(META_HEADER_PREFIX.length()),
name -> request.getHeader(name)));
}
private static Pair<String, String> validateCopyParam(String destination) {
if (destination == null) {
return null;
}
Pair<String, String> res;
if (destination.charAt(0) == '/') {
String[] tokens = destination.split("/", 3);
if (tokens.length != 3) {
return null;
}
res = new Pair<>(tokens[1], tokens[2]);
} else {
String[] tokens = destination.split("/", 2);
if (tokens.length != 2) {
return null;
}
res = new Pair<>(tokens[0], tokens[1]);
}
return res;
}
private String emulateCopyBlob(BlobStore blobStore, Response resp, BlobMetadata meta,
String destContainer, String destObject, CopyOptions options) {
Response.StatusType statusInfo = resp.getStatusInfo();
if (statusInfo.equals(Response.Status.OK)) {
ContentMetadata contentMetadata = meta.getContentMetadata();
Map<String, String> newMetadata = new HashMap<>();
newMetadata.putAll(meta.getUserMetadata());
newMetadata.putAll(options.userMetadata());
RESERVED_METADATA.forEach(s -> newMetadata.remove(s));
Blob blob = blobStore.blobBuilder(destObject)
.userMetadata(newMetadata)
.payload(new InputStreamPayload((InputStream) resp.getEntity()))
.contentLength(resp.getLength())
.contentDisposition(contentMetadata.getContentDisposition())
.contentEncoding(contentMetadata.getContentEncoding())
.contentType(contentMetadata.getContentType())
.contentLanguage(contentMetadata.getContentLanguage())
.build();
return blobStore.putBlob(destContainer, blob);
} else {
throw new ClientErrorException(statusInfo.getReasonPhrase(), statusInfo.getStatusCode());
}
}
private String serverCopyBlob(BlobStore blobStore, String container, String objectName, String destContainer,
String destObject, CopyOptions options) {
try {
return blobStore.copyBlob(container, objectName, destContainer, destObject, options);
} catch (RuntimeException e) {
if (e.getCause() instanceof HttpResponseException) {
throw (HttpResponseException) e.getCause();
} else {
throw e;
}
}
}
private void validateUserMetadata(Map<String, String> userMetadata) {
if (userMetadata != null) {
if (userMetadata.size() > InfoResource.CONFIG.swift.max_meta_count) {
throw new BadRequestException();
}
}
}
@COPY
@Consumes(" ")
public Response copyObject(@NotNull @PathParam("container") String container,
@NotNull @Encoded @PathParam("object") String objectName,
@NotNull @PathParam("account") String account,
@HeaderParam("X-Auth-Token") String authToken,
@NotNull @HeaderParam("Destination") String destination,
@NotNull @HeaderParam("Destination-Account") String destAccount,
@QueryParam("multipart-manifest") String multiPartManifest,
@HeaderParam(HttpHeaders.CONTENT_TYPE) String contentType,
@HeaderParam(HttpHeaders.CONTENT_ENCODING) String contentEncoding,
@HeaderParam(HttpHeaders.CONTENT_DISPOSITION) String contentDisposition,
@HeaderParam(HttpHeaders.IF_MATCH) String ifMatch,
@HeaderParam(HttpHeaders.IF_MODIFIED_SINCE) Date ifModifiedSince,
@HeaderParam(HttpHeaders.IF_UNMODIFIED_SINCE) Date ifUnmodifiedSince,
@HeaderParam(SwiftHeaders.OBJECT_COPY_FRESH_METADATA) boolean freshMetadata,
@Context Request request) {
if (objectName.length() > InfoResource.CONFIG.swift.max_object_name_length) {
return badRequest();
}
Pair<String, String> dest = validateCopyParam(destination);
if (dest == null) {
return Response.status(Response.Status.PRECONDITION_FAILED).build();
}
String destContainer = dest.getFirst();
String destObject = dest.getSecond();
// TODO: unused
if (destAccount == null) {
destAccount = account;
}
if (destObject.length() > InfoResource.CONFIG.swift.max_object_name_length) {
return badRequest();
}
logger.info("copy {}/{} to {}/{}", container, objectName, destContainer, destObject);
Map<String, String> additionalUserMeta = getUserMetadata(request);
BlobStore blobStore = getBlobStore(authToken).get(container, objectName);
if (!blobStore.containerExists(container) || !blobStore.containerExists(destContainer)) {
return notFound();
}
String copiedFrom;
try {
copiedFrom = container + "/" + URLDecoder.decode(objectName, "UTF-8");
} catch (UnsupportedEncodingException e) {
throw propagate(e);
}
BlobMetadata meta = blobStore.blobMetadata(container, objectName);
if (meta == null) {
return notFound();
}
CopyOptions.Builder builder = CopyOptions.builder();
ContentMetadataBuilder contentMetadata = meta.getContentMetadata().toBuilder();
if (contentDisposition != null) {
contentMetadata.contentDisposition(contentDisposition);
}
if (contentEncoding != null) {
contentMetadata.contentEncoding(contentEncoding);
}
if (contentType != null) {
contentMetadata.contentType(contentType);
}
builder.contentMetadata(contentMetadata.build());
if (freshMetadata) {
builder.userMetadata(additionalUserMeta);
} else {
if (!additionalUserMeta.isEmpty()) {
Map<String, String> newMetadata = new HashMap<>();
newMetadata.putAll(meta.getUserMetadata());
newMetadata.putAll(additionalUserMeta);
builder.userMetadata(newMetadata);
}
}
if (ifMatch != null) {
builder.ifMatch(ifMatch);
}
if (ifModifiedSince != null) {
builder.ifModifiedSince(ifModifiedSince);
}
if (ifUnmodifiedSince != null) {
builder.ifUnmodifiedSince(ifUnmodifiedSince);
}
CopyOptions options = builder.build();
validateUserMetadata(options.userMetadata());
Map<String, String> userMetadata = meta.getUserMetadata();
String etag = null;
if (!"get".equals(multiPartManifest)) {
// copy is supposed to flatten the large object, which we have to emulate
Response resp = null;
if (userMetadata.containsKey(DYNAMIC_OBJECT_MANIFEST)) {
resp = getDloObject(blobStore, meta, GetOptions.NONE, null);
} else if (userMetadata.containsKey(STATIC_OBJECT_MANIFEST)) {
resp = getSloObject(blobStore, blobStore.getBlob(container, objectName), GetOptions.NONE, null);
}
if (resp != null) {
try {
etag = emulateCopyBlob(blobStore, resp, meta, destContainer, destObject, options);
} finally {
resp.close();
}
}
}
if (etag == null) {
etag = serverCopyBlob(blobStore, container, objectName, destContainer, destObject, options);
}
return Response.status(Response.Status.CREATED)
.header(HttpHeaders.ETAG, etag)
.header(HttpHeaders.CONTENT_LENGTH, 0)
.header(HttpHeaders.CONTENT_TYPE, contentType)
.header(HttpHeaders.DATE, new Date())
.header("X-Copied-From", copiedFrom)
.build();
}
// TODO: actually handle this, jclouds doesn't support metadata update yet
@POST
@Consumes(" ")
public Response postObject(@NotNull @PathParam("container") String container,
@NotNull @Encoded @PathParam("object") String objectName,
@NotNull @PathParam("account") String account,
@HeaderParam("X-Auth-Token") String authToken,
@HeaderParam("X-Delete-At") long deleteAt,
@HeaderParam(HttpHeaders.CONTENT_DISPOSITION) String contentDisposition,
@HeaderParam(HttpHeaders.CONTENT_ENCODING) String contentEncoding,
@HeaderParam("X-Delete-After") long deleteAfter,
@HeaderParam(HttpHeaders.CONTENT_TYPE) String contentType,
@HeaderParam("X-Detect-Content-Type") boolean detectContentType,
@Context Request request) {
if (objectName.length() > InfoResource.CONFIG.swift.max_object_name_length) {
return badRequest();
}
BlobStore blobStore = getBlobStore(authToken).get(container, objectName);
if (!blobStore.containerExists(container)) {
return notFound();
}
BlobMetadata meta = blobStore.blobMetadata(container, objectName);
if (meta == null) {
return notFound();
}
Map<String, String> newMetadata = getUserMetadata(request);
validateUserMetadata(newMetadata);
Map<String, String> originalMetadata = meta.getUserMetadata();
// copy the dlo/slo headers
RESERVED_METADATA.stream()
.filter(k -> originalMetadata.containsKey(k))
.forEach(k -> newMetadata.put(k, originalMetadata.get(k)));
CopyOptions options = CopyOptions.builder().userMetadata(newMetadata).build();
String etag = serverCopyBlob(blobStore, container, objectName, container, objectName, options);
if (etag == null) {
return notFound();
}
return Response.accepted()
.header(HttpHeaders.DATE, new Date())
.build();
}
private static void copyContentHeaders(Blob blob, String contentDisposition, String contentEncoding, String contentType) {
MutableContentMetadata contentMetadata = blob.getMetadata().getContentMetadata();
if (contentDisposition != null) {
contentMetadata.setContentDisposition(contentDisposition);
}
if (contentType != null) {
contentMetadata.setContentType(contentType);
}
if (contentEncoding != null) {
contentMetadata.setContentEncoding(contentEncoding);
}
}
private ManifestEntry[] readSLOManifest(InputStream in) throws IOException {
ObjectMapper mapper = new ObjectMapper();
ManifestEntry[] res = mapper.readValue(in, ManifestEntry[].class);
if (res.length > 1000) {
throw new ClientErrorException(Response.Status.BAD_REQUEST);
}
return res;
}
private Pair<Long, String> getManifestTotalSizeAndETag(Iterable<ManifestEntry> entries) {
Hasher hash = Hashing.md5().newHasher();
long segmentsTotalLength = 0;
for (ManifestEntry entry : entries) {
hash.putString(entry.etag, StandardCharsets.UTF_8);
segmentsTotalLength += entry.size_bytes;
}
return new Pair<>(segmentsTotalLength, '"' + hash.hash().toString() + '"');
}
private void validateManifest(ManifestEntry[] res, BlobStore blobStore, String authToken) {
Arrays.stream(res).parallel()
.forEach(s -> {
Response r = null;
try {
r = headObject(blobStore, authToken, s.container, s.object, null);
if (!r.getStatusInfo().getFamily().equals(Response.Status.Family.SUCCESSFUL)) {
throw new ClientErrorException(Response.Status.CONFLICT);
}
long size = Long.parseLong(r.getHeaderString(HttpHeaders.CONTENT_LENGTH));
String etag = r.getHeaderString(HttpHeaders.ETAG);
if (s.size_bytes != size || !eTagsEqual(s.etag, etag)) {
logger.error("400 bad request: {}/{} {} {} != {} {}",
s.container, s.object, s.etag, s.size_bytes, etag, size);
throw new ClientErrorException(Response.Status.BAD_REQUEST);
}
} finally {
if (r != null) {
r.close();
}
}
});
}
@PUT
public Response putObject(@NotNull @PathParam("container") String container,
@NotNull @Encoded @PathParam("object") String objectName,
@NotNull @PathParam("account") String account,
@QueryParam("multipart-manifest") String multiPartManifest,
@QueryParam("signature") String signature,
@QueryParam("expires") String expires,
@HeaderParam(DYNAMIC_OBJECT_MANIFEST) String objectManifest,
@HeaderParam("X-Auth-Token") String authToken,
@HeaderParam(HttpHeaders.CONTENT_LENGTH) String contentLengthParam,
@HeaderParam("Transfer-Encoding") String transferEncoding,
@HeaderParam(HttpHeaders.CONTENT_TYPE) MediaType contentType,
@HeaderParam("X-Detect-Content-Type") boolean detectContentType,
@HeaderParam("X-Copy-From") String copyFrom,
@HeaderParam("X-Copy-From-Account") String copyFromAccount,
@HeaderParam(HttpHeaders.ETAG) String eTag,
@HeaderParam(HttpHeaders.CONTENT_DISPOSITION) String contentDisposition,
@HeaderParam(HttpHeaders.CONTENT_ENCODING) String contentEncoding,
@HeaderParam("X-Delete-At") long deleteAt,
@HeaderParam("X-Delete-After") long deleteAfter,
@HeaderParam(HttpHeaders.IF_MATCH) String ifMatch,
@HeaderParam(HttpHeaders.IF_NONE_MATCH) String ifNoneMatch,
@HeaderParam(HttpHeaders.IF_MODIFIED_SINCE) Date ifModifiedSince,
@HeaderParam(HttpHeaders.IF_UNMODIFIED_SINCE) Date ifUnmodifiedSince,
@HeaderParam(SwiftHeaders.OBJECT_COPY_FRESH_METADATA) boolean freshMetadata,
@Context Request request) {
//objectName = normalizePath(objectName);
if (objectName.length() > InfoResource.CONFIG.swift.max_object_name_length) {
return badRequest();
}
if (transferEncoding != null && !"chunked".equals(transferEncoding)) {
return Response.status(Response.Status.NOT_IMPLEMENTED).build();
}
if (contentLengthParam == null && !"chunked".equals(transferEncoding)) {
return Response.status(Response.Status.LENGTH_REQUIRED).build();
}
long contentLength = contentLengthParam == null ? 0 : Long.parseLong(contentLengthParam);
logger.info("PUT {}", objectName);
if (copyFromAccount == null) {
copyFromAccount = account;
}
if (copyFrom != null) {
Pair<String, String> copy = validateCopyParam(copyFrom);
return copyObject(copy.getFirst(), copy.getSecond(), copyFromAccount, authToken,
container + "/" + objectName, account, null, contentType.toString(),
contentEncoding, contentDisposition,
ifMatch, ifModifiedSince, ifUnmodifiedSince, freshMetadata,
request);
}
Map<String, String> metadata = getUserMetadata(request);
validateUserMetadata(metadata);
InputStream copiedStream = null;
BlobStore blobStore = getBlobStore(authToken).get(container, objectName);
if ("put".equals(multiPartManifest)) {
ByteArrayOutputStream buffer = new ByteArrayOutputStream();
try (TeeInputStream tee = new TeeInputStream(request.getInputStream(), buffer, true)) {
ManifestEntry[] manifest = readSLOManifest(tee);
validateManifest(manifest, blobStore, authToken);
Pair<Long, String> sizeAndEtag = getManifestTotalSizeAndETag(Arrays.asList(manifest));
metadata.put(STATIC_OBJECT_MANIFEST, sizeAndEtag.getFirst() + " " + sizeAndEtag.getSecond());
copiedStream = new ByteArrayInputStream(buffer.toByteArray());
} catch (IOException e) {
throw propagate(e);
}
} else if (objectManifest != null) {
metadata.put(DYNAMIC_OBJECT_MANIFEST, objectManifest);
}
if (!blobStore.containerExists(container)) {
return notFound();
}
HashCode contentMD5 = null;
if (eTag != null) {
try {
contentMD5 = HashCode.fromBytes(BaseEncoding.base16().lowerCase().decode(eTag));
} catch (IllegalArgumentException iae) {
throw new ClientErrorException(422, iae); // Unprocessable Entity
}
if (contentMD5.bits() != Hashing.md5().bits()) {
// Unprocessable Entity
throw new ClientErrorException(contentMD5.bits() + " != " + Hashing.md5().bits(), 422);
}
}
try (InputStream is = copiedStream != null ? copiedStream : request.getInputStream()) {
BlobBuilder.PayloadBlobBuilder builder = blobStore.blobBuilder(objectName)
.userMetadata(metadata)
.payload(is);
if (contentDisposition != null) {
builder.contentDisposition(contentDisposition);
}
if (contentEncoding != null) {
builder.contentEncoding(contentEncoding);
}
if (contentType != null) {
builder.contentType(contentType.toString());
}
if (contentLengthParam != null) {
builder.contentLength(contentLength);
}
if (contentMD5 != null) {
builder.contentMD5(contentMD5);
}
try {
String remoteETag;
try {
remoteETag = blobStore.putBlob(container, builder.build());
} catch (HttpResponseException e) {
HttpResponse response = e.getResponse();
if (response == null) {
throw e;
}
int code = response.getStatusCode();
if (code == 400 && !"openstack-swift".equals(blobStore.getContext().unwrap().getId())) {
// swift expects 422 for md5 mismatch
throw new ClientErrorException(response.getStatusLine(), 422, e.getCause());
} else {
throw new ClientErrorException(response.getStatusLine(), code, e.getCause());
}
}
BlobMetadata meta = blobStore.blobMetadata(container, objectName);
return Response.status(Response.Status.CREATED).header(HttpHeaders.ETAG, remoteETag)
.header(HttpHeaders.LAST_MODIFIED, meta.getLastModified())
.header(HttpHeaders.CONTENT_LENGTH, 0)
.header(HttpHeaders.CONTENT_TYPE, contentType)
.header(HttpHeaders.DATE, new Date()).build();
} catch (ContainerNotFoundException e) {
return notFound();
}
} catch (IOException e) {
return Response.status(Response.Status.INTERNAL_SERVER_ERROR).build();
}
}
@HEAD
public Response headObject(@NotNull @PathParam("container") String container,
@NotNull @Encoded @PathParam("object") String objectName,
@NotNull @PathParam("account") String account,
@HeaderParam("X-Auth-Token") String authToken,
@QueryParam("multipart-manifest") String multiPartManifest) {
if (objectName.length() > InfoResource.CONFIG.swift.max_object_name_length) {
return badRequest();
}
BlobStore blobStore = getBlobStore(authToken).get(container, objectName);
return headObject(blobStore, authToken, container, objectName, multiPartManifest);
}
private Response headObject(BlobStore blobStore, String authToken,
String container, String objectName, String multiPartManifest) {
BlobMetadata meta = blobStore.blobMetadata(container, objectName);
if (meta == null) {
return notFound();
}
boolean isMultiPartManifest = false;
if (meta.getUserMetadata().containsKey(STATIC_OBJECT_MANIFEST)) {
isMultiPartManifest = true;
if (!"get".equals(multiPartManifest)) {
String sloData = meta.getUserMetadata().get(STATIC_OBJECT_MANIFEST);
String[] data = sloData.split(" ", 2);
return addObjectHeaders(Response.ok(), meta,
Optional.of(overwriteSizeAndETag(Long.parseLong(data[0]), data[1])))
.build();
}
} else {
String manifest = meta.getUserMetadata().get(DYNAMIC_OBJECT_MANIFEST);
if (manifest != null) {
isMultiPartManifest = true;
if (!"get".equals(multiPartManifest)) {
Pair<String, String> param = validateCopyParam(manifest);
String dloContainer = param.getFirst();
String objectsPrefix = param.getSecond();
blobStore = getBlobStore(authToken).get(container);
List<ManifestEntry> segments = getDLOSegments(blobStore, dloContainer, objectsPrefix);
Pair<Long, String> sizeAndEtag = getManifestTotalSizeAndETag(segments);
return addObjectHeaders(Response.ok(), meta,
Optional.of(overwriteSizeAndETag(sizeAndEtag.getFirst(), sizeAndEtag.getSecond())))
.build();
}
}
}
// TODO: We should be sending a 204 (No Content), but cannot due to https://java.net/jira/browse/JERSEY-2822
return addObjectHeaders(Response.ok(), meta,
isMultiPartManifest ?
Optional.of(ImmutableMap.of(HttpHeaders.CONTENT_TYPE, MANIFEST_CONTENT_TYPE)) :
Optional.empty())
.build();
}
@DELETE
public Response deleteObject(@NotNull @PathParam("account") String account,
@NotNull @PathParam("container") String container,
@NotNull @Encoded @PathParam("object") String objectName,
@QueryParam("multipart-manifest") String multipartManifest,
@HeaderParam("X-Auth-Token") String authToken) throws IOException {
if (objectName.length() > InfoResource.CONFIG.swift.max_object_name_length) {
return badRequest();
}
BlobStore store = getBlobStore(authToken).get(container, objectName);
if ("delete".equals(multipartManifest)) {
Blob blob;
try {
blob = store.getBlob(container, objectName);
} catch (ContainerNotFoundException cnfe) {
blob = null;
}
if (blob == null) {
return Response.status(Response.Status.OK)
.entity("{\"Number Not Found\": 1" +
", \"Response Status\": \"404 Not Found\"" +
// TODO: JSON encode container and objectName
", \"Errors\": [[\"/" + container + "/" + objectName + "\", \"Not found\"]]" +
", \"Number Deleted\": 0" +
", \"Response Body\": \"\"}")
.build();
}
if (!blob.getMetadata().getUserMetadata().containsKey(STATIC_OBJECT_MANIFEST)) {
return Response.status(Response.Status.OK)
.entity("{\"Number Not Found\": 0" +
", \"Response Status\": \"400 Bad Request\"" +
// TODO: JSON encode container and objectName
", \"Errors\": [[\"/" + container + "/" + objectName + "\", \"Not an SLO manifest\"]]" +
", \"Number Deleted\": 0" +
", \"Response Body\": \"\"}")
.build();
}
ManifestEntry[] entries = readSLOManifest(blob.getPayload().openStream());
Arrays.stream(entries).parallel().forEach(e -> store.removeBlob(e.container, e.object));
store.removeBlob(container, objectName);
return Response.status(Response.Status.OK)
.entity("{\"Number Not Found\": 0" +
", \"Response Status\": \"200 OK\"" +
", \"Errors\": [[]]" +
", \"Number Deleted\": " + entries.length +
", \"Response Body\": \"\"}")
.build();
}
if (!store.containerExists(container)) {
return Response.status(Response.Status.NOT_FOUND).build();
}
BlobMetadata meta = store.blobMetadata(container, objectName);
if (meta == null) {
return Response.status(Response.Status.NOT_FOUND).build();
}
store.removeBlob(container, objectName);
return Response.noContent()
.type(meta.getContentMetadata().getContentType())
.build();
}
private Map<String, Object> overwriteSizeAndETag(long size, String etag) {
return ImmutableMap.of(HttpHeaders.CONTENT_LENGTH, size,
HttpHeaders.ETAG, etag);
}
private Response.ResponseBuilder addObjectHeaders(Response.ResponseBuilder responseBuilder, BlobMetadata metaData,
Optional<Map<String, Object>> overwrites) {
Map<String, String> userMetadata = metaData.getUserMetadata();
userMetadata.entrySet().stream()
.filter(entry -> !RESERVED_METADATA.contains(entry.getKey()))
.forEach(entry -> responseBuilder.header(META_HEADER_PREFIX + entry.getKey(), entry.getValue()));
if (userMetadata.containsKey(DYNAMIC_OBJECT_MANIFEST)) {
responseBuilder.header(DYNAMIC_OBJECT_MANIFEST, userMetadata.get(DYNAMIC_OBJECT_MANIFEST));
}
String contentType = Strings.isNullOrEmpty(metaData.getContentMetadata().getContentType()) ?
MediaType.APPLICATION_OCTET_STREAM : metaData.getContentMetadata().getContentType();
Map<String, Supplier<Object>> defaultHeaders = ImmutableMap.<String, Supplier<Object>>builder()
.put(HttpHeaders.CONTENT_DISPOSITION, () -> metaData.getContentMetadata().getContentDisposition())
.put(HttpHeaders.CONTENT_ENCODING, () -> metaData.getContentMetadata().getContentEncoding())
.put(HttpHeaders.CONTENT_LENGTH, metaData::getSize)
.put(HttpHeaders.LAST_MODIFIED, metaData::getLastModified)
.put(HttpHeaders.ETAG, metaData::getETag)
.put(STATIC_OBJECT_MANIFEST, () -> userMetadata.containsKey(STATIC_OBJECT_MANIFEST))
.put(HttpHeaders.DATE, Date::new)
.put(HttpHeaders.CONTENT_TYPE, () -> contentType)
.build();
overwrites.ifPresent(headers ->
headers.forEach((k, v) -> responseBuilder.header(k, v)));
defaultHeaders.forEach((k, v) -> {
if (!overwrites.isPresent() || !overwrites.get().containsKey(k)) {
responseBuilder.header(k, v.get());
}
});
return responseBuilder;
}
private class HttpRangeInputStream extends InputStream {
private final InputStream in;
private final Iterator<Range> ranges;
private final long size;
private Range currentRange;
private long offset = -1;
HttpRangeInputStream(InputStream in, long size, List<Pair<Long, Long>> ranges) {
this.in = requireNonNull(in);
this.size = size;
if (ranges != null) {
this.ranges = ranges.stream().map(r -> new Range(r)).collect(Collectors.toList()).iterator();
currentRange = this.ranges.next();
} else {
this.ranges = Iterators.emptyIterator();
currentRange = new Range(new Pair<>(0L, Long.MAX_VALUE));
}
}
private void seek() throws IOException {
long skipped = 0;
logger.debug("seeking to {} from {}", currentRange, offset);
if (currentRange.start > 0) {
skipped = in.skip(currentRange.start - offset);
} else if (currentRange.start == -1) {
// suffix range
skipped = in.skip(size - currentRange.available - offset);
}
offset += skipped;
}
@Override
public int read(byte[] b, int off, int len) throws IOException {
if (!maybeSeek()) {
return -1;
}
int nread = in.read(b, off, len);
if (nread != -1) {
currentRange.available -= nread;
offset += nread;
}
return nread;
}
private boolean maybeSeek() throws IOException {
if (currentRange.available == 0) {
if (ranges.hasNext()) {
currentRange = ranges.next();
seek();
} else {
return false;
}
}
if (offset == -1) {
offset = 0;
seek();
}
return true;
}
@Override
public int read() throws IOException {
if (!maybeSeek()) {
return -1;
}
int val = in.read();
if (val != -1) {
currentRange.available--;
offset++;
}
return val;
}
private class Range {
long start;
long available;
Range(Pair<Long, Long> rangeSpec) {
if (rangeSpec.getFirst() == null) {
start = -1;
available = rangeSpec.getSecond();
} else {
start = rangeSpec.getFirst();
if (rangeSpec.getSecond() == null) {
available = Long.MAX_VALUE;
} else {
available = rangeSpec.getSecond() - start + 1;
}
}
}
@Override
public String toString() {
return Objects.toStringHelper(this)
.add("start", start)
.add("available", available)
.toString();
}
}
}
private class ManifestObjectInputStream extends InputStream {
private final PeekingIterator<ManifestEntry> entries;
private final BlobStore blobStore;
private Response currentResp;
private InputStream currentStream;
private long availableBytes;
ManifestObjectInputStream(BlobStore blobStore, Iterable<ManifestEntry> entries) {
this.blobStore = requireNonNull(blobStore);
this.entries = Iterators.peekingIterator(requireNonNull(entries).iterator());
}
@Override
public long skip(long requestSkip) throws IOException {
long remainingSkip = requestSkip;
if (remainingSkip <= 0) {
return 0;
}
if (availableBytes >= remainingSkip) {
long skipped = currentStream.skip(remainingSkip);
availableBytes -= skipped;
remainingSkip -= skipped;
} else {
remainingSkip -= availableBytes;
while (entries.hasNext()) {
ManifestEntry e = entries.peek();
if (remainingSkip > e.size_bytes) {
entries.next();
remainingSkip -= e.size_bytes;
} else {
break;
}
}
if (remainingSkip > 0) {
openNextStream();
if (currentStream != null) {
long skipped = currentStream.skip(remainingSkip);
availableBytes -= skipped;
remainingSkip -= skipped;
}
}
}
return requestSkip - remainingSkip;
}
void openNextStream() throws IOException {
if (currentStream != null) {
currentResp.close();
currentResp = null;
currentStream = null;
availableBytes = 0;
}
if (!entries.hasNext()) {
return;
}
ManifestEntry entry = entries.next();
logger.info("opening {}/{}", entry.container, entry.object);
Response resp = getObject(blobStore, entry.container, entry.object, GetOptions.NONE, null, false);
if (!resp.getStatusInfo().getFamily().equals(Response.Status.Family.SUCCESSFUL)) {
resp.close();
throw new ClientErrorException(Response.Status.CONFLICT);
}
availableBytes = Long.parseLong(resp.getHeaderString(HttpHeaders.CONTENT_LENGTH));
currentStream = (InputStream) resp.getEntity();
currentResp = resp;
String etag = resp.getHeaderString(HttpHeaders.ETAG);
if (entry.size_bytes != availableBytes || !eTagsEqual(entry.etag, etag)) {
logger.error("409 conflict: {}/{} {} {} != {} {}",
entry.container, entry.object, entry.etag, entry.size_bytes,
etag, availableBytes);
throw new ClientErrorException(Response.Status.CONFLICT);
}
}
@Override
public int read(byte[] b, int off, int len) throws IOException {
if (currentStream == null || availableBytes == 0) {
openNextStream();
}
do {
if (currentStream == null) {
return -1;
}
try {
int res = currentStream.read(b, off, len);
if (res == -1) {
openNextStream();
} else {
availableBytes -= res;
return res;
}
} catch (EOFException e) {
if (availableBytes != 0) {
logger.error("error with {} bytes left", availableBytes);
throw e;
}
openNextStream();
} catch (IOException e) {
logger.error("error with {} bytes left", availableBytes);
throw e;
}
} while (true);
}
@Override
public int read() throws IOException {
if (currentStream == null || availableBytes == 0) {
openNextStream();
}
do {
if (currentStream == null) {
return -1;
}
try {
int res = currentStream.read();
if (res == -1) {
openNextStream();
} else {
availableBytes--;
return res;
}
} catch (EOFException e) {
if (availableBytes != 0) {
logger.error("error with {} bytes left", availableBytes);
throw e;
}
openNextStream();
} catch (IOException e) {
logger.error("error with {} bytes left", availableBytes);
throw e;
}
} while (true);
}
}
private static class ManifestEntry {
@JsonProperty String etag;
@JsonProperty long size_bytes;
String container;
String object;
@JsonProperty void setPath(String path) {
Pair<String, String> param = validateCopyParam(path);
container = param.getFirst();
object = param.getSecond();
}
@Override
public String toString() {
return Objects.toStringHelper(this)
.add("object", container + "/" + object)
.add("size", size_bytes)
.toString();
}
}
}