/***************************************************************************
* Copyright (c) 2012-2015 VMware, Inc. All Rights Reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
***************************************************************************/
package com.vmware.bdd.plugin.ironfan.impl;
import java.io.BufferedReader;
import java.io.File;
import java.io.InputStreamReader;
import java.lang.reflect.Type;
import java.net.URI;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import com.vmware.bdd.security.tls.TlsClientConfiguration;
import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.HttpStatus;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.conn.scheme.Scheme;
import org.apache.http.conn.ssl.SSLContexts;
import org.apache.http.conn.ssl.SSLSocketFactory;
import org.apache.http.impl.client.DefaultHttpClient;
import org.apache.http.util.EntityUtils;
import org.apache.log4j.Logger;
import com.google.gson.Gson;
import com.google.gson.JsonSyntaxException;
import com.google.gson.annotations.Expose;
import com.google.gson.annotations.SerializedName;
import com.google.gson.reflect.TypeToken;
import com.vmware.aurora.global.Configuration;
import com.vmware.bdd.apitypes.DistroRead;
import com.vmware.bdd.exception.BddException;
import com.vmware.bdd.spectypes.HadoopRole;
import com.vmware.bdd.utils.CommonUtil;
import com.vmware.bdd.utils.Constants;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager;
class RolePackageMapping {
private List<String> roles;
private String tarball;
//yum or ubuntu apt repos
@Expose
@SerializedName("package_repos")
private List<String> packageRepos;
public List<String> getRoles() {
return roles;
}
public void setRoles(List<String> roles) {
this.roles = roles;
}
public String getTarball() {
return tarball;
}
public void setTarball(String tarball) {
this.tarball = tarball;
}
public List<String> getPackageRepos() {
return packageRepos;
}
public void setPackageRepos(List<String> packageRepos) {
this.packageRepos = packageRepos;
}
}
class Distro {
private static final Logger logger = Logger.getLogger(Distro.class);
private String name;
private Boolean hveSupported;
private String vendor = Constants.DEFAULT_VENDOR;
private String version;
private List<RolePackageMapping> packages;
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public Boolean getHveSupported() {
return hveSupported;
}
public void setHveSupported(Boolean hveSupported) {
this.hveSupported = hveSupported;
}
public String getVendor() {
return vendor;
}
public void setVendor(String vendor) {
this.vendor = vendor;
}
public String getVersion() {
return version;
}
public void setVersion(String version) {
this.version = version;
}
public List<RolePackageMapping> getPackages() {
return packages;
}
public void setPackages(List<RolePackageMapping> packages) {
this.packages = packages;
}
public DistroRead convert() {
DistroRead dr = new DistroRead();
Set<String> roles = new TreeSet<String>();
if (this.getPackages() == null) {
return null;
}
for (RolePackageMapping pkg : this.getPackages()) {
if (pkg.getRoles() == null) {
return null;
}
for (String r : pkg.getRoles()) {
if (r == null || r.isEmpty()) {
return null;
}
if (!roles.contains(r)) {
roles.add(r);
} else {
logger.error("duplicated roles: " + r);
}
}
}
dr.setName(this.getName());
dr.setVendor(this.getVendor());
dr.setVersion(this.getVersion());
dr.setHveSupported(this.getHveSupported() == null ? false : this.getHveSupported());
dr.setRoles(new ArrayList<String>(roles));
return dr;
}
@Override
public String toString() {
return "Distro [name=" + name + ", hveSupported=" + hveSupported + ", version="
+ version + ", packages=" + packages + "]";
}
}
public class DistroManager {
private static String DISTRO_MANIFEST_FILE_PATH = "/opt/serengeti/www/distros/manifest";
private static String VENDOR = "serengeti.distro_vendor";
private static String distroRootUrl = "http://localhost/distros/";
private static String distrosManifestUrl;
private static final Logger logger = Logger.getLogger(DistroManager.class);
static {
distroRootUrl = Configuration.getString("serengeti.distro_root", distroRootUrl);
distrosManifestUrl = distroRootUrl + "/manifest";
}
public enum PackagesExistStatus {
NONE, TARBALL, REPO, BOTH
}
private String eTag = null;
private long lastModified = 0L;
private Map<String, Distro> distros = null;
public DistroManager() {
try {
loadManifest();
} catch (BddException ex) {
logger.error("failed to load distro manifest");
}
}
/*
public List<DistroRead> getPluginSupportDistro(String appManager) {
SoftwareManager softwareManager =
softwareManagerCollector.getSoftwareManager(appManager);
if (softwareManager == null) {
logger.error("Failed to get softwareManger.");
throw ClusterConfigException.FAILED_TO_GET_SOFTWARE_MANAGER(appManager);
}
List<HadoopStack> hadoopStacks = softwareManager.getSupportedStacks();
if (hadoopStacks != null && hadoopStacks.size() > 0) {
List<DistroRead> distros = new ArrayList<DistroRead>();
for (HadoopStack hadoopStack : hadoopStacks) {
DistroRead distro = new DistroRead();
distro.setName(hadoopStack.getDistro());
distro.setVendor(hadoopStack.getVendor());
distro.setVersion(hadoopStack.getFullVersion());
distros.add(distro);
}
return distros;
}
return null;
}*/
/*
* Return the content of distro manifest file.
* Return null if the content is not changed since last visit.
*/
@SuppressWarnings("deprecation")
private String readDistroManifest() throws Exception {
File manifestFile = new File(DISTRO_MANIFEST_FILE_PATH);
if (manifestFile.exists()) {
// The manifest file is on the local server.
// No need to reload the file if it's not modified.
if (lastModified != manifestFile.lastModified()) {
lastModified = manifestFile.lastModified();
logger.info("last modified date of manifest file changed. Reloading manifest.");
} else {
return null;
}
}
BufferedReader in = null;
DefaultHttpClient httpclient = new DefaultHttpClient();
try {
SSLContext sslContext = SSLContexts.custom().useTLS().build();
sslContext.init(null, new TrustManager[]{new X509TrustManager() {
@Override
public void checkClientTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException {
return;
}
@Override
public void checkServerTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException {
return;
}
@Override
public X509Certificate[] getAcceptedIssuers() {
return new X509Certificate[0];
}
}}, null);
TlsClientConfiguration tlsConfiguration = new TlsClientConfiguration();
SSLSocketFactory socketFactory = new SSLSocketFactory(sslContext,
tlsConfiguration.getSslProtocols(), tlsConfiguration.getCipherSuites(),
SSLSocketFactory.ALLOW_ALL_HOSTNAME_VERIFIER);
Scheme sch = new Scheme("https", 443, socketFactory);
httpclient.getConnectionManager().getSchemeRegistry().register(sch);
HttpGet httpget = new HttpGet(new URI(distrosManifestUrl));
if (eTag != null) {
httpget.addHeader("If-None-Match", eTag);
}
logger.info("executing request: " + httpget.getRequestLine());
HttpResponse response = httpclient.execute(httpget);
if (!manifestFile.exists()) {
if (response.getStatusLine().getStatusCode() == HttpStatus.SC_NOT_MODIFIED) {
return null;
} else {
logger.debug("ETag of manifest file changed. Reloading manifest.");
eTag = response.getFirstHeader("ETag").getValue();;
}
}
HttpEntity entity = response.getEntity();
in = new BufferedReader(new InputStreamReader(entity.getContent()));
StringBuffer sb = new StringBuffer();
String line;
while ((line = in.readLine()) != null) {
sb.append(line);
}
EntityUtils.consume(entity);
return sb.toString();
} finally {
httpclient.getConnectionManager().shutdown();
if (in != null) {
in.close();
}
}
}
private void loadManifest() {
Map<String, Distro> distrosLoading = new HashMap<String, Distro>();
Gson gson = new Gson();
try {
Type type = new TypeToken<ArrayList<Distro>>() {}.getType();
String content = readDistroManifest();
if (content == null) {
// the manifest file is not modified since last load
return;
}
List<Distro> distrosList = gson.fromJson(content, type);
if (distrosList != null) {
for (Distro d : distrosList) {
distrosLoading.put(d.getName(), d);
}
}
} catch (JsonSyntaxException e) {
logger.error("failed to parse manifest: " + distrosManifestUrl, e);
throw BddException.INTERNAL(e, "Cannot parse manifest "
+ distrosManifestUrl + ".");
} catch (Exception e) {
logger.error("failed to read manifest: " + distrosManifestUrl, e);
throw BddException.INTERNAL(e, "Cannot read manifest "
+ distrosManifestUrl + ".");
}
distros = distrosLoading;
}
public String getPackageUrlByDistroRole(String distroName, String role) {
loadManifest();
Distro distro = distros.get(distroName);
if (distro != null) {
for (RolePackageMapping pkg : distro.getPackages()) {
for (String r : pkg.getRoles()) {
if (r.equals(role) || HadoopRole.isCustomizedRole(role)) {
return distroRootUrl + "/" + pkg.getTarball();
}
}
}
}
return null;
}
public PackagesExistStatus checkPackagesExistStatus(final String distroName) {
loadManifest();
Distro distro = distros.get(distroName);
boolean hasPackageRepo = false, hasTarball = false;
for (RolePackageMapping pkg : distro.getPackages()) {
List<String> packageRepos = pkg.getPackageRepos();
if (packageRepos != null && !packageRepos.isEmpty()) {
hasPackageRepo = true;
}
if (!CommonUtil.isBlank(pkg.getTarball())) {
hasTarball = true;
if (hasPackageRepo) {
return PackagesExistStatus.BOTH;
}
}
}
if (!hasPackageRepo && !hasTarball) {
return PackagesExistStatus.NONE;
} else if (hasPackageRepo) {
return PackagesExistStatus.REPO;
} else {
return PackagesExistStatus.TARBALL;
}
}
public List<String> getSupportedVendors() {
String vendorStr = Configuration.getStrings(VENDOR, "");
String vendorStrTmp = vendorStr.toLowerCase();
List<String> vendors =
Arrays.asList(vendorStrTmp.indexOf(",") != -1 ? vendorStrTmp.split(",")
: new String[] { vendorStrTmp });
return vendors;
}
public List<DistroRead> getDistros() {
loadManifest();
List<DistroRead> drs = new ArrayList<DistroRead>();
List<String> vendors = getSupportedVendors();
List<String> errorVendors = new ArrayList<String> ();
for (Distro distro : distros.values()) {
DistroRead dr = distro.convert();
//check vendor name is whether configured in serengeti.properties
if (! vendors.contains(dr.getVendor().toLowerCase())) {
if (!errorVendors.contains(dr.getVendor())) {
errorVendors.add(dr.getVendor());
}
}
drs.add(dr);
}
StringBuffer errorMsg = new StringBuffer();
if (!errorVendors.isEmpty()) {
String errorVendorsStr = errorVendors.toString();
errorMsg
.append(errorVendorsStr)
.append(" not configured. To configure ")
.append(errorVendorsStr)
.append(", add ")
.append(errorVendorsStr)
.append(
" to the serengeti.distro_vendor property in the serengeti.properties file.");
}
if(errorMsg.length() > 0) {
throw BddException.INTERNAL(null, errorMsg.toString());
}
return drs;
}
public DistroRead getDistroByName(String name) {
loadManifest();
DistroRead dr = null;
Distro distro = distros.get(name);
if (distro != null) {
dr = distro.convert();
if (dr == null) {
logger.error("discard invalid distro: " + distro);
}
}
return dr;
}
/* public DistroRead getDistroByName(final String appManager, final String name) {
List<DistroRead> distros = this.getPluginSupportDistro(appManager);
if (distros != null && distros.size() > 0) {
for (DistroRead distro : distros) {
if (distro.getName().equalsIgnoreCase(name)) {
return distro;
}
}
}
return null;
}*/
public List<String> getPackageRepos(String distroName) {
loadManifest();
Distro distro = distros.get(distroName);
List<String> packageRepos = new ArrayList<String> ();
for (RolePackageMapping pkg : distro.getPackages()) {
List<String> subPackageRepos = pkg.getPackageRepos();
if (subPackageRepos != null && !subPackageRepos.isEmpty()) {
packageRepos.addAll(subPackageRepos);
}
}
return packageRepos.isEmpty() ? null : packageRepos;
}
}