package com.emc.ecs.sync.config.storage;
import com.emc.ecs.sync.config.AbstractConfig;
import com.emc.ecs.sync.config.ConfigUtil;
import com.emc.ecs.sync.config.ConfigurationException;
import com.emc.ecs.sync.config.Protocol;
import com.emc.ecs.sync.config.annotation.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.xml.bind.annotation.XmlRootElement;
import javax.xml.bind.annotation.XmlTransient;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import static com.emc.ecs.sync.config.storage.EcsS3Config.PATTERN_DESC;
import static com.emc.ecs.sync.config.storage.EcsS3Config.URI_PREFIX;
@XmlRootElement
@StorageConfig(uriPrefix = URI_PREFIX)
@Label("ECS S3")
@Documentation("Reads and writes content from/to an ECS S3 bucket. This " +
"plugin is triggered by the pattern:\n" +
PATTERN_DESC + "\n" +
"Scheme, host and port are all required. " +
"key-prefix (optional) is the prefix under which to start " +
"enumerating or writing within the bucket, e.g. dir1/. If omitted the " +
"root of the bucket will be enumerated or written to.")
public class EcsS3Config extends AbstractConfig {
private static final Logger log = LoggerFactory.getLogger(EcsS3Config.class);
public static final String URI_PREFIX = "ecs-s3:";
public static final Pattern URI_PATTERN = Pattern.compile("^" + URI_PREFIX + "(?:(http|https)://)?([^:]+):([^@]+)@([^/:]+?)(:[0-9]+)?/([^/]+)(?:/(.*))?$");
public static final String PATTERN_DESC = URI_PREFIX + "http[s]://access_key:secret_key@hosts/bucket[/key-prefix] where hosts = host[,host][,..] or vdc-name(host,..)[,vdc-name(host,..)][,..] or load-balancer[:port]";
public static final Pattern VDC_PATTERN = Pattern.compile("(?:([^(,]+)?[(])?([^()]+)[)]?,?");
public static final int DEFAULT_MPU_THRESHOLD_MB = 512;
public static final int DEFAULT_MPU_PART_SIZE_MB = 128;
public static final int DEFAULT_MPU_THREAD_COUNT = 4;
public static final int DEFAULT_CONNECT_TIMEOUT = 15000; // 15 seconds
public static final int DEFAULT_READ_TIMEOUT = 60000; // 60 seconds
public static final int MIN_PART_SIZE_MB = 4;
private Protocol protocol;
private String[] vdcs;
private String host;
private int port;
private String accessKey;
private String secretKey;
private boolean enableVHosts;
private boolean smartClientEnabled = true;
private boolean geoPinningEnabled;
private String bucketName;
private boolean createBucket;
private String keyPrefix;
private boolean decodeKeys;
private boolean includeVersions;
private boolean apacheClientEnabled;
private int mpuThresholdMb = DEFAULT_MPU_THRESHOLD_MB;
private int mpuPartSizeMb = DEFAULT_MPU_PART_SIZE_MB;
private int mpuThreadCount = DEFAULT_MPU_THREAD_COUNT;
private boolean mpuEnabled;
private int socketConnectTimeoutMs = DEFAULT_CONNECT_TIMEOUT;
private int socketReadTimeoutMs = DEFAULT_READ_TIMEOUT;
private boolean preserveDirectories;
private boolean remoteCopy;
@XmlTransient
@UriGenerator
public String getUri() {
String portStr = port > 0 ? ":" + port : "";
String pathStr = keyPrefix == null ? "" : "/" + keyPrefix;
String hostStr = host == null ? ConfigUtil.join(vdcs) : host;
return String.format("%s%s://%s:%s@%s%s/%s%s",
URI_PREFIX, protocol, bin(accessKey), bin(secretKey), bin(hostStr), portStr, bin(bucketName), pathStr);
}
@UriParser
public void setUri(String uri) {
Matcher m = URI_PATTERN.matcher(uri);
if (!m.matches()) {
throw new ConfigurationException(String.format("URI does not match %s pattern (%s)", URI_PREFIX, PATTERN_DESC));
}
if (m.group(1) != null) protocol = Protocol.valueOf(m.group(1).toLowerCase());
String hostString = m.group(4);
if (hostString.contains(",") || hostString.contains("(")) {
// we can't simply split on comma, because each VDC could have commas separating the hosts
List<String> vdcList = new ArrayList<>();
Matcher matcher = VDC_PATTERN.matcher(hostString);
while (matcher.find()) {
String vdc = matcher.group();
if (vdc.charAt(vdc.length() - 1) == ',') vdc = vdc.substring(0, vdc.length() - 1);
log.info("parsed VDC: " + vdc);
vdcList.add(vdc);
}
if (!matcher.hitEnd())
throw new ConfigurationException("invalid VDC format: " + matcher.appendTail(new StringBuffer()).toString());
vdcs = vdcList.toArray(new String[vdcList.size()]);
} else {
host = hostString;
}
port = -1;
if (m.group(5) != null) port = Integer.parseInt(m.group(5).substring(1));
accessKey = m.group(2);
secretKey = m.group(3);
bucketName = m.group(6);
keyPrefix = m.group(7);
if (protocol == null || accessKey == null || secretKey == null || (host == null && (vdcs == null || vdcs.length == 0)) || bucketName == null)
throw new ConfigurationException("protocol, accessKey, secretKey, host[s] and bucket are required");
}
@Option(orderIndex = 10, locations = Option.Location.Form, required = true, description = "The protocol to use when connecting to ECS (http or https)")
public Protocol getProtocol() {
return protocol;
}
public void setProtocol(Protocol protocol) {
this.protocol = protocol;
}
@Option(orderIndex = 20, locations = Option.Location.Form, advanced = true, description = "The VDCs to use when connecting to ECS. The format for each VDC is vdc-name(node1,node2,..). If the smart-client is enabled (default) all of the nodes in each VDC will be discovered automatically. Specify multiple entries one-per-line in the UI form")
public String[] getVdcs() {
return vdcs;
}
public void setVdcs(String[] vdcs) {
this.vdcs = vdcs;
}
@Option(orderIndex = 30, locations = Option.Location.Form, description = "The load balancer or DNS name to use when connecting to ECS. Be sure to turn off the smart-client when using a load balancer")
public String getHost() {
return host;
}
public void setHost(String host) {
this.host = host;
}
@Option(orderIndex = 40, locations = Option.Location.Form, advanced = true, description = "Used to specify a non-standard port for a load balancer")
public int getPort() {
return port;
}
public void setPort(int port) {
this.port = port;
}
@Option(orderIndex = 50, locations = Option.Location.Form, required = true, description = "The ECS object user")
public String getAccessKey() {
return accessKey;
}
public void setAccessKey(String accessKey) {
this.accessKey = accessKey;
}
@Option(orderIndex = 60, locations = Option.Location.Form, required = true, description = "The secret key for the specified user")
public String getSecretKey() {
return secretKey;
}
public void setSecretKey(String secretKey) {
this.secretKey = secretKey;
}
@Option(orderIndex = 70, advanced = true, description = "Specifies whether virtual hosted buckets will be used (default is path-style buckets)")
public boolean isEnableVHosts() {
return enableVHosts;
}
public void setEnableVHosts(boolean enableVHosts) {
this.enableVHosts = enableVHosts;
}
@Option(orderIndex = 80, cliName = "no-smart-client", cliInverted = true, advanced = true, description = "The smart-client is enabled by default. Use this option to turn it off when using a load balancer or fixed set of nodes")
public boolean isSmartClientEnabled() {
return smartClientEnabled;
}
public void setSmartClientEnabled(boolean smartClientEnabled) {
this.smartClientEnabled = smartClientEnabled;
}
@Option(orderIndex = 90, advanced = true, description = "Enables geo-pinning. This will use a standard algorithm to select a consistent VDC for each object key or bucket name")
public boolean isGeoPinningEnabled() {
return geoPinningEnabled;
}
public void setGeoPinningEnabled(boolean geoPinningEnabled) {
this.geoPinningEnabled = geoPinningEnabled;
}
@Option(orderIndex = 100, locations = Option.Location.Form, required = true, description = "Specifies the bucket to use")
public String getBucketName() {
return bucketName;
}
public void setBucketName(String bucketName) {
this.bucketName = bucketName;
}
@Option(orderIndex = 110, description = "By default, the target bucket must exist. This option will create it if it does not")
public boolean isCreateBucket() {
return createBucket;
}
public void setCreateBucket(boolean createBucket) {
this.createBucket = createBucket;
}
@Option(orderIndex = 120, locations = Option.Location.Form, advanced = true, description = "The prefix to use when enumerating or writing to the bucket. Note that relative paths to objects will be relative to this prefix (when syncing to/from a different bucket or a filesystem)")
public String getKeyPrefix() {
return keyPrefix;
}
public void setKeyPrefix(String keyPrefix) {
this.keyPrefix = keyPrefix;
}
@Option(orderIndex = 130, advanced = true, description = "Specifies if keys will be URL-decoded after listing them. This can fix problems if you see file or directory names with characters like %2f in them")
public boolean isDecodeKeys() {
return decodeKeys;
}
public void setDecodeKeys(boolean decodeKeys) {
this.decodeKeys = decodeKeys;
}
@Option(orderIndex = 140, advanced = true, description = "Enable to transfer all versions of every object. NOTE: this will overwrite all versions of each source key in the target system if any exist!")
public boolean isIncludeVersions() {
return includeVersions;
}
public void setIncludeVersions(boolean includeVersions) {
this.includeVersions = includeVersions;
}
@Option(orderIndex = 150, advanced = true, description = "Enable this if you have disabled MPU and have objects larger than 2GB (the limit for the native Java HTTP client)")
public boolean isApacheClientEnabled() {
return apacheClientEnabled;
}
public void setApacheClientEnabled(boolean apacheClientEnabled) {
this.apacheClientEnabled = apacheClientEnabled;
}
@Option(orderIndex = 160, valueHint = "size-in-MB", advanced = true, description = "Sets the size threshold (in MB) when an upload shall become a multipart upload")
public int getMpuThresholdMb() {
return mpuThresholdMb;
}
public void setMpuThresholdMb(int mpuThresholdMb) {
this.mpuThresholdMb = mpuThresholdMb;
}
@Option(orderIndex = 170, valueHint = "size-in-MB", advanced = true, description = "Sets the part size to use when multipart upload is required (objects over 5GB). Default is " + DEFAULT_MPU_PART_SIZE_MB + "MB, minimum is " + MIN_PART_SIZE_MB + "MB")
public int getMpuPartSizeMb() {
return mpuPartSizeMb;
}
public void setMpuPartSizeMb(int mpuPartSizeMb) {
this.mpuPartSizeMb = mpuPartSizeMb;
}
@Option(orderIndex = 180, advanced = true, description = "The number of threads to use for multipart upload (only applicable for file sources)")
public int getMpuThreadCount() {
return mpuThreadCount;
}
public void setMpuThreadCount(int mpuThreadCount) {
this.mpuThreadCount = mpuThreadCount;
}
@Option(orderIndex = 190, advanced = true, description = "Enables multi-part upload (MPU). Large files will be split into multiple streams and (if possible) sent in parallel")
public boolean isMpuEnabled() {
return mpuEnabled;
}
public void setMpuEnabled(boolean mpuEnabled) {
this.mpuEnabled = mpuEnabled;
}
@Option(orderIndex = 200, valueHint = "timeout-ms", advanced = true, description = "Sets the connection timeout in milliseconds (default is " + DEFAULT_CONNECT_TIMEOUT + "ms)")
public int getSocketConnectTimeoutMs() {
return socketConnectTimeoutMs;
}
public void setSocketConnectTimeoutMs(int socketConnectTimeoutMs) {
this.socketConnectTimeoutMs = socketConnectTimeoutMs;
}
@Option(orderIndex = 210, valueHint = "timeout-ms", advanced = true, description = "Sets the read timeout in milliseconds (default is " + DEFAULT_READ_TIMEOUT + "ms)")
public int getSocketReadTimeoutMs() {
return socketReadTimeoutMs;
}
public void setSocketReadTimeoutMs(int socketReadTimeoutMs) {
this.socketReadTimeoutMs = socketReadTimeoutMs;
}
@Option(orderIndex = 220, advanced = true, description = "If enabled, directories are stored in S3 as empty objects to preserve empty dirs and metadata from the source")
public boolean isPreserveDirectories() {
return preserveDirectories;
}
public void setPreserveDirectories(boolean preserveDirectories) {
this.preserveDirectories = preserveDirectories;
}
@Option(orderIndex = 230, advanced = true, description = "If enabled, a remote-copy command is issued instead of streaming the data. Can only be used when the source and target is the same system")
public boolean isRemoteCopy() {
return remoteCopy;
}
public void setRemoteCopy(boolean remoteCopy) {
this.remoteCopy = remoteCopy;
}
}