/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.repositories.url;
import org.elasticsearch.cluster.metadata.RepositoryMetaData;
import org.elasticsearch.common.blobstore.BlobPath;
import org.elasticsearch.common.blobstore.BlobStore;
import org.elasticsearch.common.blobstore.url.URLBlobStore;
import org.elasticsearch.common.settings.Setting;
import org.elasticsearch.common.settings.Setting.Property;
import org.elasticsearch.common.util.URIPattern;
import org.elasticsearch.common.xcontent.NamedXContentRegistry;
import org.elasticsearch.env.Environment;
import org.elasticsearch.repositories.RepositoryException;
import org.elasticsearch.repositories.blobstore.BlobStoreRepository;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URISyntaxException;
import java.net.URL;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.function.Function;
/**
* Read-only URL-based implementation of the BlobStoreRepository
* <p>
* This repository supports the following settings
* <dl>
* <dt>{@code url}</dt><dd>URL to the root of repository. This is mandatory parameter.</dd>
* <dt>{@code concurrent_streams}</dt><dd>Number of concurrent read/write stream (per repository on each node). Defaults to 5.</dd>
* </dl>
*/
public class URLRepository extends BlobStoreRepository {
public static final String TYPE = "url";
public static final Setting<List<String>> SUPPORTED_PROTOCOLS_SETTING =
Setting.listSetting("repositories.url.supported_protocols", Arrays.asList("http", "https", "ftp", "file", "jar"),
Function.identity(), Property.NodeScope);
public static final Setting<List<URIPattern>> ALLOWED_URLS_SETTING =
Setting.listSetting("repositories.url.allowed_urls", Collections.emptyList(), URIPattern::new, Property.NodeScope);
public static final Setting<URL> URL_SETTING = new Setting<>("url", "http:", URLRepository::parseURL, Property.NodeScope);
public static final Setting<URL> REPOSITORIES_URL_SETTING =
new Setting<>("repositories.url.url", (s) -> s.get("repositories.uri.url", "http:"), URLRepository::parseURL,
Property.NodeScope);
private final List<String> supportedProtocols;
private final URIPattern[] urlWhiteList;
private final Environment environment;
private final URLBlobStore blobStore;
private final BlobPath basePath;
/**
* Constructs a read-only URL-based repository
*/
public URLRepository(RepositoryMetaData metadata, Environment environment,
NamedXContentRegistry namedXContentRegistry) throws IOException {
super(metadata, environment.settings(), namedXContentRegistry);
if (URL_SETTING.exists(metadata.settings()) == false && REPOSITORIES_URL_SETTING.exists(settings) == false) {
throw new RepositoryException(metadata.name(), "missing url");
}
supportedProtocols = SUPPORTED_PROTOCOLS_SETTING.get(settings);
urlWhiteList = ALLOWED_URLS_SETTING.get(settings).toArray(new URIPattern[]{});
this.environment = environment;
URL url = URL_SETTING.exists(metadata.settings()) ? URL_SETTING.get(metadata.settings()) : REPOSITORIES_URL_SETTING.get(settings);
URL normalizedURL = checkURL(url);
blobStore = new URLBlobStore(settings, normalizedURL);
basePath = BlobPath.cleanPath();
}
@Override
protected BlobStore blobStore() {
return blobStore;
}
@Override
protected BlobPath basePath() {
return basePath;
}
/**
* Makes sure that the url is white listed or if it points to the local file system it matches one on of the root path in path.repo
*/
private URL checkURL(URL url) {
String protocol = url.getProtocol();
if (protocol == null) {
throw new RepositoryException(getMetadata().name(), "unknown url protocol from URL [" + url + "]");
}
for (String supportedProtocol : supportedProtocols) {
if (supportedProtocol.equals(protocol)) {
try {
if (URIPattern.match(urlWhiteList, url.toURI())) {
// URL matches white list - no additional processing is needed
return url;
}
} catch (URISyntaxException ex) {
logger.warn("cannot parse the specified url [{}]", url);
throw new RepositoryException(getMetadata().name(), "cannot parse the specified url [" + url + "]");
}
// We didn't match white list - try to resolve against path.repo
URL normalizedUrl = environment.resolveRepoURL(url);
if (normalizedUrl == null) {
String logMessage = "The specified url [{}] doesn't start with any repository paths specified by the " +
"path.repo setting or by {} setting: [{}] ";
logger.warn(logMessage, url, ALLOWED_URLS_SETTING.getKey(), environment.repoFiles());
String exceptionMessage = "file url [" + url + "] doesn't match any of the locations specified by path.repo or "
+ ALLOWED_URLS_SETTING.getKey();
throw new RepositoryException(getMetadata().name(), exceptionMessage);
}
return normalizedUrl;
}
}
throw new RepositoryException(getMetadata().name(), "unsupported url protocol [" + protocol + "] from URL [" + url + "]");
}
@Override
public boolean isReadOnly() {
return true;
}
private static URL parseURL(String s) {
try {
return new URL(s);
} catch (MalformedURLException e) {
throw new IllegalArgumentException("Unable to parse URL repository setting", e);
}
}
}