/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.ambari.msi;
import org.apache.ambari.scom.ClusterDefinitionProvider;
import org.apache.ambari.scom.HostInfoProvider;
import org.apache.ambari.server.controller.internal.ResourceImpl;
import org.apache.ambari.server.controller.spi.Resource;
import org.apache.commons.lang.StringUtils;
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
/**
* Defines the cluster created by the MSI.
*/
public class ClusterDefinition {
private static final String COMMENT_TAG = "#";
private static final String HA_PROPERTY_INDICATOR = "HA";
private static Boolean HA_ENABLE = Boolean.FALSE;
private final Set<String> services = new HashSet<String>();
private final Set<String> hosts = new HashSet<String>();
private final Map<String, Set<String>> components = new HashMap<String, Set<String>>();
private final Map<String, Map<String, Set<String>>> hostComponents = new HashMap<String, Map<String, Set<String>>>();
private final Map<Integer, StateProvider.Process> processes = new HashMap<Integer, StateProvider.Process>();
private final Set<Resource> requestResources = new HashSet<Resource>();
private final Set<Resource> taskResources = new HashSet<Resource>();
private final StateProvider stateProvider;
private final ClusterDefinitionProvider definitionProvider;
private final HostInfoProvider hostInfoProvider;
private String clusterName;
private String versionId;
private int nextRequestId = 1;
private int nextTaskId = 1;
/**
* Client only components
*/
private final Set<String> clientOnlyComponents = new HashSet<String>(){{
add("PIG");
add("SQOOP");
add("YARN_CLIENT");
add("MAPREDUCE2_CLIENT");
}};
private Boolean isClientOnlyComponent(String componentName) {
return clientOnlyComponents.contains(componentName);
}
/**
* Client only services
*/
private final Set<String> clientOnlyServices = new HashSet<String>(){{
add("PIG");
add("SQOOP");
}};
private Boolean isClientOnlyService(String serviceName) {
return clientOnlyServices.contains(serviceName);
}
/**
* Component name mapping to account for differences in what is provided by the MSI
* and what is expected by the Ambari providers.
*/
private final Map<String, Set<String>> componentNameMap = new HashMap<String, Set<String>>();
private void initComponentNameMap() {
componentNameMap.put("NAMENODE_HOST", Collections.singleton("NAMENODE"));
componentNameMap.put("SECONDARY_NAMENODE_HOST", Collections.singleton("SECONDARY_NAMENODE"));
componentNameMap.put("OOZIE_SERVER_HOST", Collections.singleton("OOZIE_SERVER"));
componentNameMap.put("WEBHCAT_HOST", Collections.singleton("WEBHCAT_SERVER"));
componentNameMap.put("FLUME_HOSTS", Collections.singleton("FLUME_SERVER"));
componentNameMap.put("HBASE_MASTER", Collections.singleton("HBASE_MASTER"));
componentNameMap.put("HBASE_REGIONSERVERS", Collections.singleton("HBASE_REGIONSERVER"));
componentNameMap.put("ZOOKEEPER_HOSTS", Collections.singleton("ZOOKEEPER_SERVER"));
Set<String> slaveComponents = new HashSet<String>();
slaveComponents.add("DATANODE");
componentNameMap.put("SLAVE_HOSTS", slaveComponents);
Set<String> hiveComponents = new HashSet<String>();
hiveComponents.add("HIVE_SERVER");
hiveComponents.add("HIVE_METASTORE");
hiveComponents.add("HIVE_CLIENT");
componentNameMap.put("HIVE_SERVER_HOST", hiveComponents);
Integer majorStackVersion = getMajorStackVersion();
Integer minorStackVersion = getMinorStackVersion();
if(majorStackVersion != null) {
if(majorStackVersion == 1) {
Set<String> mapReduceComponents = new HashSet<String>();
mapReduceComponents.add("JOBTRACKER");
mapReduceComponents.add("HISTORYSERVER");
componentNameMap.put("JOBTRACKER_HOST", mapReduceComponents);
slaveComponents.add("TASKTRACKER");
}
if(majorStackVersion == 2) {
componentNameMap.put("JOURNALNODE_HOST", Collections.singleton("JOURNALNODE"));
componentNameMap.put(minorStackVersion > 0 ? "NN_HA_JOURNALNODE_HOSTS" : "HA_JOURNALNODE_HOSTS", Collections.singleton("JOURNALNODE"));
Set<String> haNamenodeComponents = new HashSet<String>();
haNamenodeComponents.add("NAMENODE");
haNamenodeComponents.add("ZKFC");
componentNameMap.put(minorStackVersion > 0 ? "NN_HA_STANDBY_NAMENODE_HOST" : "HA_NAMENODE_HOST", haNamenodeComponents);
Set<String> mapReduce2Components = new HashSet<String>();
mapReduce2Components.add("HISTORYSERVER");
mapReduce2Components.add("RESOURCEMANAGER");
componentNameMap.put("RESOURCEMANAGER_HOST", mapReduce2Components);
componentNameMap.put("RM_HA_STANDBY_RESOURCEMANAGER_HOST", Collections.singleton("RESOURCEMANAGER"));
slaveComponents.add("NODEMANAGER");
//hiveComponents.add("MYSQL_SERVER");
Set<String> clientHosts = new HashSet<String>();
clientHosts.add("PIG");
clientHosts.add("SQOOP");
clientHosts.add("YARN_CLIENT");
clientHosts.add("MAPREDUCE2_CLIENT");
componentNameMap.put("CLIENT_HOSTS", clientHosts);
}
}
}
/**
* Component service mapping .
*/
private final Map<String, String> componentServiceMap = new HashMap<String, String>();
private void initComponentServiceMap() {
componentServiceMap.put("NAMENODE", "HDFS");
componentServiceMap.put("DATANODE", "HDFS");
componentServiceMap.put("SECONDARY_NAMENODE", "HDFS");
componentServiceMap.put("HIVE_SERVER", "HIVE");
componentServiceMap.put("HIVE_METASTORE", "HIVE");
componentServiceMap.put("HIVE_CLIENT", "HIVE");
componentServiceMap.put("OOZIE_SERVER", "OOZIE");
componentServiceMap.put("WEBHCAT_SERVER", "HIVE");
componentServiceMap.put("FLUME_SERVER", "FLUME");
componentServiceMap.put("HBASE_MASTER", "HBASE");
componentServiceMap.put("HBASE_REGIONSERVER", "HBASE");
componentServiceMap.put("ZOOKEEPER_SERVER", "ZOOKEEPER");
Integer majorStackVersion = getMajorStackVersion();
if(majorStackVersion != null) {
if(majorStackVersion == 1) {
componentServiceMap.put("JOBTRACKER", "MAPREDUCE");
componentServiceMap.put("HISTORYSERVER", "MAPREDUCE");
componentServiceMap.put("TASKTRACKER", "MAPREDUCE");
}
if(majorStackVersion == 2) {
componentServiceMap.put("PIG", "PIG");
componentServiceMap.put("SQOOP", "SQOOP");
componentServiceMap.put("HISTORYSERVER", "MAPREDUCE2");
componentServiceMap.put("MAPREDUCE2_CLIENT", "MAPREDUCE2");
componentServiceMap.put("JOURNALNODE", "HDFS");
componentServiceMap.put("NODEMANAGER", "YARN");
componentServiceMap.put("RESOURCEMANAGER", "YARN");
componentServiceMap.put("YARN_CLIENT", "YARN");
componentServiceMap.put("ZKFC", "HDFS");
//componentServiceMap.put("MYSQL_SERVER", "HIVE");
}
}
}
// ----- Constructors ------------------------------------------------------
/**
* Create a cluster definition.
*
* @param stateProvider the state provider
*/
public ClusterDefinition(StateProvider stateProvider,
ClusterDefinitionProvider definitionProvider,
HostInfoProvider hostInfoProvider) {
this.stateProvider = stateProvider;
this.definitionProvider = definitionProvider;
this.hostInfoProvider = hostInfoProvider;
this.clusterName = definitionProvider.getClusterName();
this.versionId = definitionProvider.getVersionId();
init();
try {
readClusterDefinition();
haEnableSetup();
} catch (IOException e) {
String msg = "Caught exception reading cluster definition file.";
throw new IllegalStateException(msg, e);
}
}
private void haEnableSetup() {
if(HA_ENABLE) {
Map<String, Set<String>> serviceHostComponents = hostComponents.get(componentServiceMap.get("ZKFC"));
if (serviceHostComponents != null) {
for(String host : serviceHostComponents.keySet()) {
Set<String> hostHostComponents = serviceHostComponents.get(host);
if(hostHostComponents != null && hostHostComponents.contains("NAMENODE")) {
hostHostComponents.add("ZKFC");
}
}
}
}
}
// ----- ClusterDefinition -------------------------------------------------
/**
* Get the major stack version for this cluster.
*
* @return the major stack version
*/
public Integer getMajorStackVersion() {
if(StringUtils.isNotEmpty(versionId)) {
String majorVersion = StringUtils.substring(versionId, 4, 5);
if(StringUtils.isNotEmpty(majorVersion)) {
return Integer.parseInt(majorVersion);
}
}
return null;
}
/**
* Get the minor stack version for this cluster.
*
* @return the minor stack version
*/
public Integer getMinorStackVersion() {
if(StringUtils.isNotEmpty(versionId)) {
String majorVersion = StringUtils.substring(versionId, 6, 7);
if(StringUtils.isNotEmpty(majorVersion)) {
return Integer.parseInt(majorVersion);
}
}
return null;
}
/**
* Get the name of the cluster.
*
* @return the cluster name
*/
public String getClusterName() {
return clusterName;
}
/**
* Get the name of the cluster.
*
* @return the cluster name
*/
public String getVersionId() {
return versionId;
}
/**
* Get the services for the cluster.
*
* @return the set of service names
*/
public Set<String> getServices() {
return services;
}
/**
* Get the hosts for the cluster.
*
* @return the set of hosts names
*/
public Set<String> getHosts() {
return hosts;
}
/**
* Get the host info provider associated with this cluster definition.
*
* @return the host info provider
*/
public HostInfoProvider getHostInfoProvider() {
return hostInfoProvider;
}
/**
* Get the components for the given service.
*
* @param service the service name
*
* @return the set of component names for the given service name
*/
public Set<String> getComponents(String service) {
Set<String> componentSet = components.get(service);
return componentSet == null ? Collections.<String>emptySet() : componentSet;
}
/**
* Get the host components for the given service and host.
*
* @param service the service name
* @param host the host name
*
* @return the set of host component names for the given service and host names
*/
public Set<String> getHostComponents(String service, String host) {
Set<String> resultSet = null;
Map<String, Set<String>> serviceHostComponents = hostComponents.get(service);
if (serviceHostComponents != null) {
resultSet = serviceHostComponents.get(host);
}
return resultSet == null ? Collections.<String>emptySet() : resultSet;
}
/**
* Get the host state from the given host name.
*
* @param hostName the host name
*
* @return the host state
*/
public String getHostState(String hostName) {
for (Map.Entry<String, Map<String, Set<String>>> entry : hostComponents.entrySet()) {
Map<String, Set<String>> serviceHostComponents = entry.getValue();
for (Map.Entry<String, Set<String>> hostEntry : serviceHostComponents.entrySet()) {
if (hostEntry.getKey().equals(hostName)) {
Set<String> componentNames = hostEntry.getValue();
for (String componentName : componentNames) {
if (isClientOnlyComponent(componentName)) continue;
if (stateProvider.getRunningState(hostName, componentName) != StateProvider.State.Running) {
return "UNHEALTHY";
}
}
}
}
}
return "HEALTHY";
}
/**
* Get the service state from the given service name.
*
* @param serviceName the service name
*
* @return the service state
*/
public String getServiceState(String serviceName) {
if (isClientOnlyService(serviceName)) return "INSTALLED";
Map<String, Set<String>> serviceHostComponents = hostComponents.get(serviceName);
if (serviceHostComponents != null) {
for (Map.Entry<String, Set<String>> entry : serviceHostComponents.entrySet()) {
String hostName = entry.getKey();
Set<String> componentNames = entry.getValue();
for (String componentName : componentNames) {
if (isClientOnlyComponent(componentName)) continue;
if (stateProvider.getRunningState(hostName, componentName) != StateProvider.State.Running) {
return "INSTALLED";
}
}
}
}
return "STARTED";
}
/**
* Set the service state for the given service name.
*
* @param serviceName the service name
*
* @return the request id
*/
public int setServiceState(String serviceName, String state) {
StateProvider.State s = state.equals("STARTED") ? StateProvider.State.Running :
state.equals("INSTALLED") ? StateProvider.State.Stopped : StateProvider.State.Unknown;
int requestId = -1;
if (!isClientOnlyService(serviceName)) {
// if the state is already set to the desired state or state is unknown then skip it
if (s != StateProvider.State.Unknown && !state.equals(getServiceState(serviceName))) {
Map<String, Set<String>> serviceHostComponents = hostComponents.get(serviceName);
if (serviceHostComponents != null) {
for (Map.Entry<String, Set<String>> entry : serviceHostComponents.entrySet()) {
String hostName = entry.getKey();
Set<String> componentNames = entry.getValue();
for (String componentName : componentNames) {
if(isClientOnlyComponent(componentName) || state.equals(getHostComponentState(hostName, componentName))) continue;
requestId = recordProcess(stateProvider.setRunningState(hostName, componentName, s), requestId,
"Set service " + serviceName + " state to " + s);
}
}
}
}
}
return requestId;
}
/**
* Get the component state from the give service name and component name.
*
* @param serviceName the service name
* @param componentName the component name
*
* @return the component state
*/
public String getComponentState(String serviceName, String componentName) {
Map<String, Set<String>> serviceHostComponents = hostComponents.get(serviceName);
if (serviceHostComponents != null) {
for (Map.Entry<String, Set<String>> entry : serviceHostComponents.entrySet()) {
String hostName = entry.getKey();
Set<String> componentNames = entry.getValue();
for (String name : componentNames) {
if (name.equals(componentName)) {
if (isClientOnlyComponent(componentName)) return "INSTALLED";
if (stateProvider.getRunningState(hostName, componentName) != StateProvider.State.Running) {
return "INSTALLED";
}
}
}
}
}
return "STARTED";
}
/**
* Set the component state for the given service name.
*
* @param serviceName the service name
* @param componentName the component name
* @param state the state
*
* @return the request id
*/
public int setComponentState(String serviceName, String componentName, String state) {
StateProvider.State s = state.equals("STARTED") ? StateProvider.State.Running :
state.equals("INSTALLED") ? StateProvider.State.Stopped : StateProvider.State.Unknown;
int requestId = -1;
if (!isClientOnlyComponent(componentName)) {
if (s != StateProvider.State.Unknown) {
Map<String, Set<String>> serviceHostComponents = hostComponents.get(serviceName);
if (serviceHostComponents != null) {
for (Map.Entry<String, Set<String>> entry : serviceHostComponents.entrySet()) {
String hostName = entry.getKey();
Set<String> componentNames = entry.getValue();
for (String name : componentNames) {
if (name.equals(componentName)) {
if(state.equals(getHostComponentState(hostName, componentName))) continue;
requestId = recordProcess(stateProvider.setRunningState(hostName, componentName, s), requestId,
"Set component " + componentName + " state to " + s);
}
}
}
}
}
}
return requestId;
}
/**
* Get the host component state from the given host name and component name.
*
* @param hostName the host name
* @param componentName the component name
*
* @return the host component state
*/
public String getHostComponentState(String hostName, String componentName) {
Boolean healthy = Boolean.FALSE;
if (!isClientOnlyComponent(componentName))
healthy = stateProvider.getRunningState(hostName, componentName) == StateProvider.State.Running;
return healthy ? "STARTED" : "INSTALLED";
}
/**
* Set the host component state for the given host name and component name.
*
* @param hostName the host name
* @param componentName the component name
*
* @return the request id
*/
public int setHostComponentState(String hostName, String componentName, String state) {
StateProvider.State s = state.equals("STARTED") ? StateProvider.State.Running :
state.equals("INSTALLED") ? StateProvider.State.Stopped : StateProvider.State.Unknown;
int requestId = -1;
if (!isClientOnlyComponent(componentName)) {
if (s != StateProvider.State.Unknown && !state.equals(getHostComponentState(hostName, componentName))) {
requestId = recordProcess(stateProvider.setRunningState(hostName, componentName, s), -1,
"Set host component " + componentName + " state to " + s);
}
}
return requestId;
}
/**
* Return the process that is associated with the given id.
*
* @param id the id
*
* @return the process
*/
public StateProvider.Process getProcess(Integer id) {
return processes.get(id);
}
/**
* Get the set of request resources.
*
* @return the set of request resources
*/
public Set<Resource> getRequestResources() {
return requestResources;
}
/**
* Get the set of task resources
*
* @return the set of task resources
*/
public Set<Resource> getTaskResources() {
return taskResources;
}
// ----- helper methods ----------------------------------------------------
// initialize
private void init() {
initComponentNameMap();
initComponentServiceMap();
}
// record a process and create the corresponding request and task resource
private synchronized int recordProcess(StateProvider.Process process, int requestId, String context) {
if (requestId == -1) {
requestId = nextRequestId++;
Resource request = new ResourceImpl(Resource.Type.Request);
request.setProperty(RequestProvider.REQUEST_ID_PROPERTY_ID, requestId);
request.setProperty(RequestProvider.REQUEST_CLUSTER_NAME_PROPERTY_ID, clusterName);
request.setProperty(RequestProvider.REQUEST_CONTEXT_ID, context);
requestResources.add(request);
}
Resource task = new ResourceImpl(Resource.Type.Task);
int taskId = nextTaskId++;
taskResources.add(task);
task.setProperty(TaskProvider.TASK_ID_PROPERTY_ID, taskId);
task.setProperty(TaskProvider.TASK_REQUEST_ID_PROPERTY_ID, requestId);
task.setProperty(TaskProvider.TASK_CLUSTER_NAME_PROPERTY_ID, clusterName);
processes.put(taskId, process);
return requestId;
}
/**
* Read the MSI cluster definition file.
*/
private void readClusterDefinition() throws IOException {
InputStream is = definitionProvider.getInputStream();
try {
BufferedReader br = new BufferedReader(new InputStreamReader(is));
String line;
while ((line = br.readLine()) != null) {
line = line.trim();
if (line.startsWith(COMMENT_TAG)) continue;
int i = line.indexOf('=');
if (i == -1) continue;
String propertyName = line.substring(0, i);
String propertyValue = line.substring(i + 1);
if(propertyName.equalsIgnoreCase(HA_PROPERTY_INDICATOR)) {
HA_ENABLE = propertyValue.equalsIgnoreCase("YES") ? Boolean.TRUE : Boolean.FALSE;
}
Set<String> componentNames = componentNameMap.get(propertyName);
if (componentNames != null) {
for (String componentName : componentNames) {
String serviceName = componentServiceMap.get(componentName);
services.add(serviceName);
Set<String> serviceComponents = components.get(serviceName);
if (serviceComponents == null) {
serviceComponents = new HashSet<String>();
components.put(serviceName, serviceComponents);
}
serviceComponents.add(componentName);
Map<String, Set<String>> serviceHostComponents = hostComponents.get(serviceName);
if (serviceHostComponents == null) {
serviceHostComponents = new HashMap<String, Set<String>>();
hostComponents.put(serviceName, serviceHostComponents);
}
String[] hostNames = propertyValue.split(",");
for (String hostName : hostNames) {
hostName = hostName.trim();
Set<String> hostHostComponents = serviceHostComponents.get(hostName);
if (hostHostComponents == null) {
hostHostComponents = new HashSet<String>();
serviceHostComponents.put(hostName, hostHostComponents);
}
hostHostComponents.add(componentName);
hosts.add(hostName);
}
}
}
}
} finally {
is.close();
}
}
}