/*
* Copyright 2016 The Simple File Server Authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sfs.nodes.compute.container;
import com.google.common.base.Joiner;
import com.google.common.base.Optional;
import com.google.protobuf.InvalidProtocolBufferException;
import io.vertx.core.Handler;
import io.vertx.core.MultiMap;
import io.vertx.core.http.HttpServerResponse;
import io.vertx.core.json.JsonObject;
import io.vertx.core.logging.Logger;
import org.sfs.Server;
import org.sfs.SfsRequest;
import org.sfs.SfsVertx;
import org.sfs.VertxContext;
import org.sfs.auth.Authenticate;
import org.sfs.elasticsearch.container.LoadAccountAndContainer;
import org.sfs.elasticsearch.object.LoadObject;
import org.sfs.elasticsearch.object.PersistOrUpdateVersion;
import org.sfs.elasticsearch.object.UpdateObject;
import org.sfs.encryption.Algorithm;
import org.sfs.encryption.AlgorithmDef;
import org.sfs.filesystem.JournalFile;
import org.sfs.io.BufferEndableWriteStream;
import org.sfs.io.InflaterEndableWriteStream;
import org.sfs.io.PipedEndableWriteStream;
import org.sfs.io.PipedReadStream;
import org.sfs.nodes.all.segment.AcknowledgeSegment;
import org.sfs.nodes.compute.object.WriteNewSegment;
import org.sfs.rx.ConnectionCloseTerminus;
import org.sfs.rx.ObservableFuture;
import org.sfs.rx.RxHelper;
import org.sfs.rx.ToVoid;
import org.sfs.util.HttpRequestValidationException;
import org.sfs.validate.ValidateActionAdmin;
import org.sfs.validate.ValidateContainerIsEmpty;
import org.sfs.validate.ValidateContainerPath;
import org.sfs.validate.ValidateHeaderBetweenLong;
import org.sfs.validate.ValidateHeaderExists;
import org.sfs.validate.ValidateHeaderIsBase64Encoded;
import org.sfs.validate.ValidateObjectPath;
import org.sfs.validate.ValidateOptimisticObjectLock;
import org.sfs.validate.ValidatePath;
import org.sfs.vo.ObjectPath;
import org.sfs.vo.PersistentObject;
import org.sfs.vo.TransientObject;
import org.sfs.vo.TransientSegment;
import org.sfs.vo.XObject;
import rx.Observable;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.util.HashSet;
import java.util.List;
import java.util.Objects;
import java.util.Set;
import java.util.zip.InflaterOutputStream;
import static com.google.common.base.Charsets.UTF_8;
import static com.google.common.base.Preconditions.checkState;
import static com.google.common.base.Predicates.notNull;
import static com.google.common.base.Splitter.on;
import static com.google.common.base.Strings.isNullOrEmpty;
import static com.google.common.collect.FluentIterable.from;
import static com.google.common.io.BaseEncoding.base64;
import static com.google.common.primitives.Longs.tryParse;
import static io.vertx.core.logging.LoggerFactory.getLogger;
import static java.lang.Boolean.TRUE;
import static java.lang.Long.parseLong;
import static java.lang.String.format;
import static java.net.HttpURLConnection.HTTP_BAD_REQUEST;
import static java.net.HttpURLConnection.HTTP_INTERNAL_ERROR;
import static java.net.HttpURLConnection.HTTP_OK;
import static java.nio.file.Paths.get;
import static java.util.Calendar.getInstance;
import static org.sfs.encryption.AlgorithmDef.fromNameIfExists;
import static org.sfs.filesystem.containerdump.DumpFileWriter.DUMP_FILE_NAME;
import static org.sfs.protobuf.XVolume.XDumpFile.CompressionType;
import static org.sfs.protobuf.XVolume.XDumpFile.CompressionType.DEFLATE;
import static org.sfs.protobuf.XVolume.XDumpFile.CompressionType.NONE;
import static org.sfs.protobuf.XVolume.XDumpFile.FirstHeader.parseFrom;
import static org.sfs.protobuf.XVolume.XDumpFile.Header;
import static org.sfs.protobuf.XVolume.XDumpFile.Header.Type;
import static org.sfs.protobuf.XVolume.XDumpFile.Header.Type.VERSION_01;
import static org.sfs.protobuf.XVolume.XDumpFile.Version01;
import static org.sfs.rx.Defer.aVoid;
import static org.sfs.rx.Defer.just;
import static org.sfs.rx.RxHelper.combineSinglesDelayError;
import static org.sfs.util.ExceptionHelper.unwrapCause;
import static org.sfs.util.KeepAliveHttpServerResponse.DELIMITER_BUFFER;
import static org.sfs.util.SfsHttpHeaders.X_SFS_IMPORT_SKIP_POSITIONS;
import static org.sfs.util.SfsHttpHeaders.X_SFS_KEEP_ALIVE_TIMEOUT;
import static org.sfs.util.SfsHttpHeaders.X_SFS_SECRET;
import static org.sfs.util.SfsHttpHeaders.X_SFS_SRC_DIRECTORY;
import static org.sfs.vo.ObjectPath.DELIMITER;
import static org.sfs.vo.ObjectPath.fromPaths;
import static org.sfs.vo.ObjectPath.fromSfsRequest;
import static rx.Observable.error;
public class ImportContainer implements Handler<SfsRequest> {
private static final Logger LOGGER = getLogger(ImportContainer.class);
@Override
public void handle(final SfsRequest httpServerRequest) {
VertxContext<Server> vertxContext = httpServerRequest.vertxContext();
aVoid()
.flatMap(new Authenticate(httpServerRequest))
.flatMap(new ValidateActionAdmin(httpServerRequest))
.map(aVoid -> httpServerRequest)
.map(new ValidateHeaderExists(X_SFS_SRC_DIRECTORY))
.map(new ValidateHeaderBetweenLong(X_SFS_KEEP_ALIVE_TIMEOUT, 10000, 300000))
.map(aVoid -> fromSfsRequest(httpServerRequest))
.map(new ValidateContainerPath())
.flatMap(new LoadAccountAndContainer(vertxContext))
.flatMap(new ValidateContainerIsEmpty(vertxContext))
.flatMap(targetPersistentContainer -> {
MultiMap headers = httpServerRequest.headers();
String importDirectory = headers.get(X_SFS_SRC_DIRECTORY);
String unparsedSkipPositions = headers.get(X_SFS_IMPORT_SKIP_POSITIONS);
Set<Long> skipPositions;
if (!isNullOrEmpty(unparsedSkipPositions)) {
skipPositions =
from(on(',').trimResults().split(unparsedSkipPositions))
.transform(input -> tryParse(input))
.filter(notNull())
.toSet();
} else {
skipPositions = new HashSet<>(0);
}
return aVoid()
.flatMap(aVoid -> {
ObservableFuture<Boolean> handler = RxHelper.observableFuture();
vertxContext.vertx().fileSystem().exists(importDirectory, handler.toHandler());
return handler
.map(destDirectoryExists -> {
if (!TRUE.equals(destDirectoryExists)) {
JsonObject jsonObject = new JsonObject()
.put("message", format("%s does not exist", importDirectory));
throw new HttpRequestValidationException(HTTP_BAD_REQUEST, jsonObject);
} else {
return (Void) null;
}
});
})
.flatMap(oVoid -> {
ObservableFuture<List<String>> handler = RxHelper.observableFuture();
vertxContext.vertx().fileSystem().readDir(importDirectory, handler.toHandler());
return handler
.map(listing -> {
if (listing.size() <= 0) {
JsonObject jsonObject = new JsonObject()
.put("message", format("%s is empty", importDirectory));
throw new HttpRequestValidationException(HTTP_BAD_REQUEST, jsonObject);
} else {
return (Void) null;
}
});
})
.flatMap(aVoid -> {
LOGGER.info("Importing into container " + targetPersistentContainer.getId() + " from " + importDirectory);
JournalFile journalFile = new JournalFile(get(importDirectory).resolve(DUMP_FILE_NAME));
return journalFile.open(vertxContext.vertx())
.map(aVoid1 -> journalFile);
})
.flatMap(journalFile -> {
SfsVertx sfsVertx = vertxContext.vertx();
return journalFile.getFirstEntry(sfsVertx)
.map(entryOptional -> {
checkState(entryOptional.isPresent(), "First dump file entry is corrupt");
return entryOptional.get();
})
.flatMap(entry ->
entry.getMetadata(sfsVertx)
.map(buffer -> {
try {
return parseFrom(buffer.getBytes());
} catch (InvalidProtocolBufferException e) {
throw new RuntimeException(e);
}
})
.flatMap(firstHeader -> {
if (firstHeader.getEncrypted()) {
return just(httpServerRequest)
.map(new ValidateHeaderExists(X_SFS_SECRET))
.map(new ValidateHeaderIsBase64Encoded(X_SFS_SECRET))
.map(new ToVoid<>())
.map(aVoid -> {
String cipherName = firstHeader.getCipherName();
checkState(!isNullOrEmpty(cipherName), "Encryption is enabled by cipher name is not specified");
AlgorithmDef algorithmDef = fromNameIfExists(cipherName);
checkState(algorithmDef != null, "Algorithm %s not found", cipherName);
return new ImportStartState(
journalFile,
entry.getNextHeaderPosition(),
algorithmDef,
base64().decode(headers.get(X_SFS_SECRET)));
});
} else {
return just(new ImportStartState(
journalFile,
entry.getNextHeaderPosition(),
null,
null));
}
}));
})
.flatMap(importStartState -> {
JournalFile journalFile = importStartState.getJournalFile();
long startPosition = importStartState.getStartPosition();
boolean encrypted = importStartState.getAlgorithmDef() != null;
byte[] secret = importStartState.getSecret();
AlgorithmDef algorithmDef = importStartState.getAlgorithmDef();
httpServerRequest.startProxyKeepAlive();
SfsVertx sfsVertx = vertxContext.vertx();
return journalFile.scan(sfsVertx, startPosition, entry -> {
// skip over any positions that should be skipped
if (skipPositions.contains(entry.getHeaderPosition())) {
return just(true);
}
return entry.getMetadata(sfsVertx)
.flatMap(buffer -> {
try {
Header header = Header.parseFrom(buffer.getBytes());
Type type = header.getType();
checkState(VERSION_01.equals(type), "Type was %s, expected %s", type, VERSION_01);
byte[] cipherDataSalt = header.getCipherDataSalt() != null ? header.getCipherDataSalt().toByteArray() : null;
byte[] cipherMetadataSalt = header.getCipherMetadataSalt() != null ? header.getCipherMetadataSalt().toByteArray() : null;
CompressionType metadataCompressionType = header.getMetadataCompressionType();
checkState(NONE.equals(metadataCompressionType) || DEFLATE.equals(metadataCompressionType), "Metadata compression type was %s, expected %s", metadataCompressionType, DEFLATE);
CompressionType dataCompressionType = header.getDataCompressionType();
checkState(NONE.equals(dataCompressionType) || DEFLATE.equals(dataCompressionType), "Data compression type was %s, expected %s", dataCompressionType, DEFLATE);
byte[] marshaledExportObject = header.getData().toByteArray();
if (encrypted) {
checkState(cipherMetadataSalt != null && cipherMetadataSalt.length > 0);
Algorithm algorithm = algorithmDef.create(secret, cipherMetadataSalt);
marshaledExportObject = algorithm.decrypt(marshaledExportObject);
}
if (DEFLATE.equals(metadataCompressionType)) {
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
try (InflaterOutputStream inflaterOutputStream = new InflaterOutputStream(byteArrayOutputStream)) {
inflaterOutputStream.write(marshaledExportObject);
} catch (IOException e) {
throw new RuntimeException(e);
}
marshaledExportObject = byteArrayOutputStream.toByteArray();
}
Version01 exportObject = Version01.parseFrom(marshaledExportObject);
ObjectPath originalObjectPath = fromPaths(exportObject.getObjectId());
String originalAccountName = originalObjectPath.accountName().get();
String originalContainerName = originalObjectPath.containerName().get();
String originalObjectName = originalObjectPath.objectName().get();
ObjectPath targetObjectPath = fromPaths(targetPersistentContainer.getId(), originalObjectName);
ValidatePath validatePath = new ValidateObjectPath();
validatePath.call(targetObjectPath);
String targetObjectId = targetObjectPath.objectPath().get();
String targetAccountName = targetObjectPath.accountName().get();
String targetContainerName = targetObjectPath.containerName().get();
return just(targetObjectId)
.flatMap(new LoadObject(vertxContext, targetPersistentContainer))
.map(oPersistentObject -> {
if (oPersistentObject.isPresent()) {
PersistentObject persistentObject = oPersistentObject.get();
return persistentObject.newVersion().merge(exportObject);
} else {
final TransientObject transientObject = new TransientObject(targetPersistentContainer, targetObjectId)
.setOwnerGuid(exportObject.getOwnerGuid());
return transientObject
.newVersion()
.merge(exportObject);
}
})
.flatMap(transientVersion -> {
long length = transientVersion.getContentLength().get();
if (length > 0 && !transientVersion.isDeleted()) {
return aVoid()
.flatMap(aVoid -> {
PipedReadStream pipedReadStream = new PipedReadStream();
BufferEndableWriteStream bufferStreamConsumer = new PipedEndableWriteStream(pipedReadStream);
if (DEFLATE.equals(dataCompressionType)) {
bufferStreamConsumer = new InflaterEndableWriteStream(bufferStreamConsumer);
}
if (encrypted) {
checkState(cipherDataSalt != null && cipherDataSalt.length > 0);
Algorithm algorithm = algorithmDef.create(secret, cipherDataSalt);
bufferStreamConsumer = algorithm.decrypt(bufferStreamConsumer);
}
Observable<Void> oProducer = entry.produceData(sfsVertx, bufferStreamConsumer);
Observable<TransientSegment> oConsumer =
just(transientVersion)
.flatMap(new WriteNewSegment(vertxContext, pipedReadStream));
return combineSinglesDelayError(oProducer, oConsumer, (aVoid1, transientSegment) -> transientSegment);
})
.map(transientSegment -> transientSegment.getParent());
} else {
return just(transientVersion);
}
})
.doOnNext(transientVersion -> {
Optional<String> oObjectManifest = transientVersion.getObjectManifest();
if (oObjectManifest.isPresent()) {
String objectManifest = oObjectManifest.get();
int indexOfObjectName = objectManifest.indexOf(DELIMITER);
if (indexOfObjectName > 0) {
String containerName = objectManifest.substring(0, indexOfObjectName);
// only adjust the object manifest if the manifest references objects
// in the container that was exported
if (Objects.equals(containerName, originalContainerName)) {
objectManifest = targetContainerName + DELIMITER + objectManifest.substring(indexOfObjectName + 1);
transientVersion.setObjectManifest(objectManifest);
}
}
}
})
.flatMap(new PersistOrUpdateVersion(vertxContext))
.flatMap(transientVersion -> {
long length = transientVersion.getContentLength().get();
if (length > 0 && !transientVersion.getSegments().isEmpty()) {
TransientSegment latestSegment = transientVersion.getNewestSegment().get();
return just(latestSegment)
.flatMap(new AcknowledgeSegment(httpServerRequest.vertxContext()))
.map(modified -> latestSegment.getParent());
} else {
return just(transientVersion);
}
})
.flatMap(transientVersion -> {
final long versionId = transientVersion.getId();
XObject xObject = transientVersion.getParent();
return just((PersistentObject) xObject)
.map(persistentObject -> persistentObject.setUpdateTs(getInstance()))
.flatMap(new UpdateObject(httpServerRequest.vertxContext()))
.map(new ValidateOptimisticObjectLock())
.map(persistentObject -> persistentObject.getVersion(versionId).get());
})
.map(version -> TRUE);
} catch (InvalidProtocolBufferException e) {
throw new RuntimeException(e);
}
})
.onErrorResumeNext(throwable -> error(new IgnorePositionRuntimeException(throwable, entry.getHeaderPosition())));
});
})
.doOnNext(aVoid -> LOGGER.info("Done importing into container " + targetPersistentContainer.getId() + " from " + importDirectory))
.map(new ToVoid<>())
.map(aVoid -> {
JsonObject jsonResponse = new JsonObject();
jsonResponse.put("code", HTTP_OK);
return jsonResponse;
})
.onErrorResumeNext(throwable -> {
LOGGER.info("Failed importing into container " + targetPersistentContainer.getId() + " from " + importDirectory, throwable);
Optional<IgnorePositionRuntimeException> oIgnorePosition = unwrapCause(IgnorePositionRuntimeException.class, throwable);
if (oIgnorePosition.isPresent()) {
IgnorePositionRuntimeException ignorePositionRuntimeException = oIgnorePosition.get();
LOGGER.error("Handling Exception", ignorePositionRuntimeException);
long positionToIgnore = ignorePositionRuntimeException.getPosition();
JsonObject jsonResponse = new JsonObject();
skipPositions.add(positionToIgnore);
String joined = Joiner.on(',').join(skipPositions);
jsonResponse.put("code", HTTP_INTERNAL_ERROR);
jsonResponse.put("message", format("If you would like to ignore this position set the %s header with the value %s", X_SFS_IMPORT_SKIP_POSITIONS, joined));
jsonResponse.put(X_SFS_IMPORT_SKIP_POSITIONS, joined);
return just(jsonResponse);
} else {
return error(throwable);
}
});
})
.single()
.subscribe(new ConnectionCloseTerminus<JsonObject>(httpServerRequest) {
@Override
public void onNext(JsonObject jsonResponse) {
HttpServerResponse httpResponse = httpServerRequest.response();
httpResponse.write(jsonResponse.encode(), UTF_8.toString())
.write(DELIMITER_BUFFER);
}
});
}
private static class IgnorePositionRuntimeException extends RuntimeException {
private final long position;
public IgnorePositionRuntimeException(Throwable cause, long position) {
super(cause);
this.position = position;
}
public long getPosition() {
return position;
}
}
private static class ImportStartState {
private final JournalFile journalFile;
private final long startPosition;
private final AlgorithmDef algorithmDef;
private final byte[] secret;
public ImportStartState(JournalFile journalFile, long startPosition, AlgorithmDef algorithmDef, byte[] secret) {
this.journalFile = journalFile;
this.startPosition = startPosition;
this.algorithmDef = algorithmDef;
this.secret = secret;
}
public JournalFile getJournalFile() {
return journalFile;
}
public long getStartPosition() {
return startPosition;
}
public AlgorithmDef getAlgorithmDef() {
return algorithmDef;
}
public byte[] getSecret() {
return secret;
}
}
}