/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.repositories.gcs;
import com.google.api.services.storage.Storage;
import org.elasticsearch.cluster.metadata.RepositoryMetaData;
import org.elasticsearch.common.Strings;
import org.elasticsearch.common.blobstore.BlobPath;
import org.elasticsearch.common.blobstore.BlobStore;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.unit.ByteSizeUnit;
import org.elasticsearch.common.unit.ByteSizeValue;
import org.elasticsearch.common.unit.TimeValue;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.env.Environment;
import org.elasticsearch.repositories.RepositoryException;
import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
import java.util.function.Function;
import static org.elasticsearch.common.settings.Setting.Property;
import static org.elasticsearch.common.settings.Setting.boolSetting;
import static org.elasticsearch.common.settings.Setting.byteSizeSetting;
import static org.elasticsearch.common.settings.Setting.simpleString;
import static org.elasticsearch.common.settings.Setting.timeSetting;
import static org.elasticsearch.common.unit.TimeValue.timeValueMillis;
class GoogleCloudStorageRepository extends BlobStoreRepository {
// package private for testing
static final ByteSizeValue MIN_CHUNK_SIZE = new ByteSizeValue(1, ByteSizeUnit.BYTES);
static final ByteSizeValue MAX_CHUNK_SIZE = new ByteSizeValue(100, ByteSizeUnit.MB);
public static final String TYPE = "gcs";
public static final TimeValue NO_TIMEOUT = timeValueMillis(-1);
public static final Setting<String> BUCKET =
simpleString("bucket", Property.NodeScope, Property.Dynamic);
public static final Setting<String> BASE_PATH =
simpleString("base_path", Property.NodeScope, Property.Dynamic);
public static final Setting<Boolean> COMPRESS =
boolSetting("compress", false, Property.NodeScope, Property.Dynamic);
public static final Setting<ByteSizeValue> CHUNK_SIZE =
byteSizeSetting("chunk_size", MAX_CHUNK_SIZE, MIN_CHUNK_SIZE, MAX_CHUNK_SIZE, Property.NodeScope, Property.Dynamic);
public static final Setting<String> APPLICATION_NAME =
new Setting<>("application_name", GoogleCloudStoragePlugin.NAME, Function.identity(), Property.NodeScope, Property.Dynamic);
public static final Setting<String> SERVICE_ACCOUNT =
simpleString("service_account", Property.NodeScope, Property.Dynamic);
public static final Setting<TimeValue> HTTP_READ_TIMEOUT =
timeSetting("http.read_timeout", NO_TIMEOUT, Property.NodeScope, Property.Dynamic);
public static final Setting<TimeValue> HTTP_CONNECT_TIMEOUT =
timeSetting("http.connect_timeout", NO_TIMEOUT, Property.NodeScope, Property.Dynamic);
private final ByteSizeValue chunkSize;
private final boolean compress;
private final BlobPath basePath;
private final GoogleCloudStorageBlobStore blobStore;
GoogleCloudStorageRepository(RepositoryMetaData metadata, Environment environment,
NamedXContentRegistry namedXContentRegistry,
GoogleCloudStorageService storageService) throws Exception {
super(metadata, environment.settings(), namedXContentRegistry);
String bucket = getSetting(BUCKET, metadata);
String application = getSetting(APPLICATION_NAME, metadata);
String serviceAccount = getSetting(SERVICE_ACCOUNT, metadata);
String basePath = BASE_PATH.get(metadata.settings());
if (Strings.hasLength(basePath)) {
BlobPath path = new BlobPath();
for (String elem : basePath.split("/")) {
path = path.add(elem);
}
this.basePath = path;
} else {
this.basePath = BlobPath.cleanPath();
}
TimeValue connectTimeout = null;
TimeValue readTimeout = null;
TimeValue timeout = HTTP_CONNECT_TIMEOUT.get(metadata.settings());
if ((timeout != null) && (timeout.millis() != NO_TIMEOUT.millis())) {
connectTimeout = timeout;
}
timeout = HTTP_READ_TIMEOUT.get(metadata.settings());
if ((timeout != null) && (timeout.millis() != NO_TIMEOUT.millis())) {
readTimeout = timeout;
}
this.compress = getSetting(COMPRESS, metadata);
this.chunkSize = getSetting(CHUNK_SIZE, metadata);
logger.debug("using bucket [{}], base_path [{}], chunk_size [{}], compress [{}], application [{}]",
bucket, basePath, chunkSize, compress, application);
Storage client = storageService.createClient(serviceAccount, application, connectTimeout, readTimeout);
this.blobStore = new GoogleCloudStorageBlobStore(settings, bucket, client);
}
@Override
protected BlobStore blobStore() {
return blobStore;
}
@Override
protected BlobPath basePath() {
return basePath;
}
@Override
protected boolean isCompress() {
return compress;
}
@Override
protected ByteSizeValue chunkSize() {
return chunkSize;
}
/**
* Get a given setting from the repository settings, throwing a {@link RepositoryException} if the setting does not exist or is empty.
*/
static <T> T getSetting(Setting<T> setting, RepositoryMetaData metadata) {
T value = setting.get(metadata.settings());
if (value == null) {
throw new RepositoryException(metadata.name(), "Setting [" + setting.getKey() + "] is not defined for repository");
}
if ((value instanceof String) && (Strings.hasText((String) value)) == false) {
throw new RepositoryException(metadata.name(), "Setting [" + setting.getKey() + "] is empty for repository");
}
return value;
}
}