/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.ambari.server.upgrade;
import java.io.File;
import java.io.FileReader;
import java.lang.reflect.Type;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.api.services.AmbariMetaInfo;
import org.apache.ambari.server.controller.AmbariManagementController;
import org.apache.ambari.server.orm.DBAccessor;
import org.apache.ambari.server.orm.dao.AlertDefinitionDAO;
import org.apache.ambari.server.orm.dao.DaoUtils;
import org.apache.ambari.server.orm.dao.WidgetDAO;
import org.apache.ambari.server.orm.entities.AlertDefinitionEntity;
import org.apache.ambari.server.orm.entities.WidgetEntity;
import org.apache.ambari.server.state.Cluster;
import org.apache.ambari.server.state.Clusters;
import org.apache.ambari.server.state.Config;
import org.apache.ambari.server.state.Service;
import org.apache.ambari.server.state.ServiceComponentHost;
import org.apache.ambari.server.state.ServiceInfo;
import org.apache.ambari.server.state.StackId;
import org.apache.ambari.server.state.StackInfo;
import org.apache.ambari.server.state.stack.WidgetLayout;
import org.apache.ambari.server.state.stack.WidgetLayoutInfo;
import org.apache.ambari.server.utils.VersionUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.reflect.TypeToken;
import com.google.gson.Gson;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import com.google.inject.Inject;
import com.google.inject.Injector;
/**
* Upgrade catalog for version 2.2.2.
*/
public class UpgradeCatalog222 extends AbstractUpgradeCatalog {
@Inject
DaoUtils daoUtils;
/**
* Logger.
*/
private static final Logger LOG = LoggerFactory.getLogger(UpgradeCatalog222.class);
private static final String AMS_SITE = "ams-site";
private static final String AMS_HBASE_SITE = "ams-hbase-site";
private static final String HIVE_SITE_CONFIG = "hive-site";
private static final String ATLAS_APPLICATION_PROPERTIES_CONFIG = "application-properties";
private static final String ATLAS_HOOK_HIVE_MINTHREADS_PROPERTY = "atlas.hook.hive.minThreads";
private static final String ATLAS_HOOK_HIVE_MAXTHREADS_PROPERTY = "atlas.hook.hive.maxThreads";
private static final String ATLAS_CLUSTER_NAME_PROPERTY = "atlas.cluster.name";
private static final String ATLAS_ENABLETLS_PROPERTY = "atlas.enableTLS";
private static final String ATLAS_SERVER_HTTP_PORT_PROPERTY = "atlas.server.http.port";
private static final String ATLAS_SERVER_HTTPS_PORT_PROPERTY = "atlas.server.https.port";
private static final String ATLAS_REST_ADDRESS_PROPERTY = "atlas.rest.address";
private static final String HBASE_ENV_CONFIG = "hbase-env";
private static final String CONTENT_PROPERTY = "content";
private static final String UPGRADE_TABLE = "upgrade";
private static final String UPGRADE_SUSPENDED_COLUMN = "suspended";
private static final String HOST_AGGREGATOR_DAILY_CHECKPOINTCUTOFFMULTIPIER_PROPERTY =
"timeline.metrics.host.aggregator.daily.checkpointCutOffMultiplier";
private static final String CLUSTER_AGGREGATOR_DAILY_CHECKPOINTCUTOFFMULTIPIER_PROPERTY =
"timeline.metrics.cluster.aggregator.daily.checkpointCutOffMultiplier";
private static final String TIMELINE_METRICS_SERVICE_WATCHER_DISBALED_PROPERTY = "timeline.metrics.service.watcher.disabled";
private static final String AMS_MODE_PROPERTY = "timeline.metrics.service.operation.mode";
public static final String PRECISION_TABLE_TTL_PROPERTY = "timeline.metrics.host.aggregator.ttl";
public static final String CLUSTER_SECOND_TABLE_TTL_PROPERTY = "timeline.metrics.cluster.aggregator.second.ttl";
public static final String CLUSTER_MINUTE_TABLE_TTL_PROPERTY = "timeline.metrics.cluster.aggregator.minute.ttl";
public static final String AMS_WEBAPP_ADDRESS_PROPERTY = "timeline.metrics.service.webapp.address";
public static final String HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD_PROPERTY = "hbase.client.scanner.timeout.period";
public static final String HBASE_RPC_TIMEOUT_PROPERTY = "hbase.rpc.timeout";
public static final String PHOENIX_QUERY_TIMEOUT_PROPERTY = "phoenix.query.timeoutMs";
public static final String PHOENIX_QUERY_KEEPALIVE_PROPERTY = "phoenix.query.keepAliveMs";
public static final String TIMELINE_METRICS_CLUSTER_AGGREGATOR_INTERPOLATION_ENABLED
= "timeline.metrics.cluster.aggregator.interpolation.enabled";
public static final String TIMELINE_METRICS_SINK_COLLECTION_PERIOD = "timeline.metrics.sink.collection.period";
public static final String AMS_SERVICE_NAME = "AMBARI_METRICS";
public static final String AMS_COLLECTOR_COMPONENT_NAME = "METRICS_COLLECTOR";
protected static final String WIDGET_TABLE = "widget";
protected static final String WIDGET_DESCRIPTION = "description";
protected static final String WIDGET_NAME = "widget_name";
protected static final String WIDGET_CORRUPT_BLOCKS = "Corrupted Blocks";
protected static final String WIDGET_CORRUPT_REPLICAS = "Blocks With Corrupted Replicas";
protected static final String WIDGET_CORRUPT_REPLICAS_DESCRIPTION = "Number represents data blocks with at least one " +
"corrupted replica (but not all of them). Its indicative of HDFS bad health.";
protected static final String WIDGET_VALUES = "widget_values";
protected static final String WIDGET_VALUES_VALUE =
"${Hadoop:service\\" +
"\\u003dNameNode,name\\" +
"\\u003dFSNamesystem.CorruptBlocks}";
public final static String HBASE_SITE_HBASE_COPROCESSOR_MASTER_CLASSES = "hbase.coprocessor.master.classes";
public final static String HBASE_SITE_HBASE_COPROCESSOR_REGION_CLASSES = "hbase.coprocessor.region.classes";
public final static String HBASE_SITE_HBASE_COPROCESSOR_REGIONSERVER_CLASSES = "hbase.coprocessor.regionserver.classes";
// ----- Constructors ------------------------------------------------------
/**
* Don't forget to register new UpgradeCatalogs in {@link org.apache.ambari.server.upgrade.SchemaUpgradeHelper.UpgradeHelperModule#configure()}
*
* @param injector Guice injector to track dependencies and uses bindings to inject them.
*/
@Inject
public UpgradeCatalog222(Injector injector) {
super(injector);
this.injector = injector;
}
// ----- UpgradeCatalog ----------------------------------------------------
/**
* {@inheritDoc}
*/
@Override
public String getTargetVersion() {
return "2.2.2";
}
// ----- AbstractUpgradeCatalog --------------------------------------------
/**
* {@inheritDoc}
*/
@Override
public String getSourceVersion() {
return "2.2.1";
}
@Override
protected void executeDDLUpdates() throws AmbariException, SQLException {
DBAccessor.DBColumnInfo columnInfo = new DBAccessor.DBColumnInfo("host_id", Long.class);
dbAccessor.addColumn("topology_host_info", columnInfo);
dbAccessor.addFKConstraint("topology_host_info", "FK_hostinfo_host_id", "host_id", "hosts", "host_id", true);
dbAccessor.executeUpdate("update topology_host_info set host_id = (select hosts.host_id from hosts where hosts.host_name = topology_host_info.fqdn)");
updateUpgradeTable();
}
@Override
protected void executePreDMLUpdates() throws AmbariException, SQLException {
//To change body of implemented methods use File | Settings | File Templates.
}
@Override
protected void executeDMLUpdates() throws AmbariException, SQLException {
addNewConfigurationsFromXml();
updateAlerts();
updateStormConfigs();
updateAMSConfigs();
updateHiveConfig();
updateHostRoleCommands();
updateHDFSWidgetDefinition();
updateYARNWidgetDefinition();
updateHBASEWidgetDefinition();
updateHbaseEnvConfig();
updateCorruptedReplicaWidget();
updateZookeeperConfigs();
updateHBASEConfigs();
createNewSliderConfigVersion();
initializeStromAndKafkaWidgets();
}
protected void createNewSliderConfigVersion() {
// Here we are creating new service config version for SLIDER, to link slider-client
// config to SLIDER service, in serviceconfigmapping table. It could be not mapped because
// of bug which we had a long time ago.
AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
Map<String, Cluster> clusterMap = getCheckedClusterMap(ambariManagementController.getClusters());
for (final Cluster cluster : clusterMap.values()) {
Service sliderService = null;
try {
sliderService = cluster.getService("SLIDER");
} catch(AmbariException ambariException) {
LOG.info("SLIDER service not found in cluster while creating new serviceconfig version for SLIDER service.");
}
if (sliderService != null) {
cluster.createServiceConfigVersion("SLIDER", AUTHENTICATED_USER_NAME, "Creating new service config version for SLIDER service.", null);
}
}
}
protected void updateZookeeperConfigs() throws AmbariException {
AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
Map<String, Cluster> clusterMap = getCheckedClusterMap(ambariManagementController.getClusters());
for (final Cluster cluster : clusterMap.values()) {
Config zooEnv = cluster.getDesiredConfigByType("zookeeper-env");
if (zooEnv != null && zooEnv.getProperties().containsKey("zk_server_heapsize")) {
String heapSizeValue = zooEnv.getProperties().get("zk_server_heapsize");
if(!heapSizeValue.endsWith("m")) {
Map<String, String> updates = new HashMap<>();
updates.put("zk_server_heapsize", heapSizeValue+"m");
updateConfigurationPropertiesForCluster(cluster, "zookeeper-env", updates, true, false);
}
}
}
}
protected void updateHBASEConfigs() throws AmbariException {
AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
Map<String, Cluster> clusterMap = getCheckedClusterMap(ambariManagementController.getClusters());
for (final Cluster cluster : clusterMap.values()) {
Config hbaseSite = cluster.getDesiredConfigByType("hbase-site");
boolean rangerHbasePluginEnabled = isConfigEnabled(cluster,
AbstractUpgradeCatalog.CONFIGURATION_TYPE_RANGER_HBASE_PLUGIN_PROPERTIES,
AbstractUpgradeCatalog.PROPERTY_RANGER_HBASE_PLUGIN_ENABLED);
if (hbaseSite != null && rangerHbasePluginEnabled) {
Map<String, String> updates = new HashMap<>();
String stackVersion = cluster.getCurrentStackVersion().getStackVersion();
if (VersionUtils.compareVersions(stackVersion, "2.2") == 0) {
if (hbaseSite.getProperties().containsKey(HBASE_SITE_HBASE_COPROCESSOR_MASTER_CLASSES)) {
updates.put(HBASE_SITE_HBASE_COPROCESSOR_MASTER_CLASSES,
"com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor");
}
if (hbaseSite.getProperties().containsKey(HBASE_SITE_HBASE_COPROCESSOR_REGIONSERVER_CLASSES)) {
updates.put(HBASE_SITE_HBASE_COPROCESSOR_REGIONSERVER_CLASSES,
"com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor");
}
if (hbaseSite.getProperties().containsKey(HBASE_SITE_HBASE_COPROCESSOR_REGION_CLASSES)) {
updates.put(HBASE_SITE_HBASE_COPROCESSOR_REGION_CLASSES,
"org.apache.hadoop.hbase.security.token.TokenProvider,org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint," +
"com.xasecure.authorization.hbase.XaSecureAuthorizationCoprocessor");
}
} else if (VersionUtils.compareVersions(stackVersion, "2.3") == 0) {
if (hbaseSite.getProperties().containsKey(HBASE_SITE_HBASE_COPROCESSOR_MASTER_CLASSES)) {
updates.put(HBASE_SITE_HBASE_COPROCESSOR_MASTER_CLASSES,
"org.apache.ranger.authorization.hbase.RangerAuthorizationCoprocessor ");
}
if (hbaseSite.getProperties().containsKey(HBASE_SITE_HBASE_COPROCESSOR_REGIONSERVER_CLASSES)) {
updates.put(HBASE_SITE_HBASE_COPROCESSOR_REGIONSERVER_CLASSES,
"org.apache.ranger.authorization.hbase.RangerAuthorizationCoprocessor");
}
if (hbaseSite.getProperties().containsKey(HBASE_SITE_HBASE_COPROCESSOR_REGION_CLASSES)) {
updates.put(HBASE_SITE_HBASE_COPROCESSOR_REGION_CLASSES,
"org.apache.hadoop.hbase.security.token.TokenProvider,org.apache.hadoop.hbase.security.access.SecureBulkLoadEndpoint," +
"org.apache.ranger.authorization.hbase.RangerAuthorizationCoprocessor");
}
}
if (! updates.isEmpty()) {
updateConfigurationPropertiesForCluster(cluster, "hbase-site", updates, true, false);
}
}
}
}
protected void updateStormConfigs() throws AmbariException {
AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
Map<String, Cluster> clusterMap = getCheckedClusterMap(ambariManagementController.getClusters());
for (final Cluster cluster : clusterMap.values()) {
if (cluster.getDesiredConfigByType("storm-site") != null && cluster.getDesiredConfigByType("storm-site").getProperties().containsKey("storm.zookeeper.superACL")
&& cluster.getDesiredConfigByType("storm-site").getProperties().get("storm.zookeeper.superACL").equals("sasl:{{storm_base_jaas_principal}}")) {
Map<String, String> newStormProps = new HashMap<>();
newStormProps.put("storm.zookeeper.superACL", "sasl:{{storm_bare_jaas_principal}}");
updateConfigurationPropertiesForCluster(cluster, "storm-site", newStormProps, true, false);
}
}
}
protected void updateAlerts() {
LOG.info("Updating alert definitions.");
AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
AlertDefinitionDAO alertDefinitionDAO = injector.getInstance(AlertDefinitionDAO.class);
Clusters clusters = ambariManagementController.getClusters();
Map<String, Cluster> clusterMap = getCheckedClusterMap(clusters);
for (final Cluster cluster : clusterMap.values()) {
long clusterID = cluster.getClusterId();
final AlertDefinitionEntity regionserverHealthSummaryDefinitionEntity = alertDefinitionDAO.findByName(
clusterID, "regionservers_health_summary");
final AlertDefinitionEntity atsWebAlert = alertDefinitionDAO.findByName(
clusterID, "yarn_app_timeline_server_webui");
if (regionserverHealthSummaryDefinitionEntity != null) {
alertDefinitionDAO.remove(regionserverHealthSummaryDefinitionEntity);
}
if (atsWebAlert != null) {
String source = atsWebAlert.getSource();
JsonObject sourceJson = new JsonParser().parse(source).getAsJsonObject();
JsonObject uriJson = sourceJson.get("uri").getAsJsonObject();
uriJson.remove("http");
uriJson.remove("https");
uriJson.addProperty("http", "{{yarn-site/yarn.timeline-service.webapp.address}}/ws/v1/timeline");
uriJson.addProperty("https", "{{yarn-site/yarn.timeline-service.webapp.https.address}}/ws/v1/timeline");
atsWebAlert.setSource(sourceJson.toString());
alertDefinitionDAO.merge(atsWebAlert);
}
//update Atlas alert
final AlertDefinitionEntity atlasMetadataServerWebUI = alertDefinitionDAO.findByName(
clusterID, "metadata_server_webui");
if (atlasMetadataServerWebUI != null) {
String source = atlasMetadataServerWebUI.getSource();
JsonObject sourceJson = new JsonParser().parse(source).getAsJsonObject();
JsonObject uriJson = sourceJson.get("uri").getAsJsonObject();
uriJson.remove("http");
uriJson.remove("https");
uriJson.addProperty("http", "{{application-properties/atlas.server.http.port}}");
uriJson.addProperty("https", "{{application-properties/atlas.server.https.port}}");
atlasMetadataServerWebUI.setSource(sourceJson.toString());
alertDefinitionDAO.merge(atlasMetadataServerWebUI);
}
}
}
protected void updateHostRoleCommands() throws SQLException {
dbAccessor.createIndex("idx_hrc_status_role", "host_role_command", "status", "role");
}
protected void updateAMSConfigs() throws AmbariException {
AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
Clusters clusters = ambariManagementController.getClusters();
if (clusters != null) {
Map<String, Cluster> clusterMap = clusters.getClusters();
if (clusterMap != null && !clusterMap.isEmpty()) {
for (final Cluster cluster : clusterMap.values()) {
Config amsSite = cluster.getDesiredConfigByType(AMS_SITE);
if (amsSite != null) {
Map<String, String> amsSiteProperties = amsSite.getProperties();
Map<String, String> newProperties = new HashMap<>();
if (amsSiteProperties.containsKey(AMS_WEBAPP_ADDRESS_PROPERTY)) {
Set<String> collectorHostNames = cluster.getHosts(AMS_SERVICE_NAME, AMS_COLLECTOR_COMPONENT_NAME);
for (String collector: collectorHostNames) {
String currentValue = amsSiteProperties.get(AMS_WEBAPP_ADDRESS_PROPERTY);
if (currentValue.startsWith("0.0.0.0")) {
newProperties.put(AMS_WEBAPP_ADDRESS_PROPERTY, currentValue.replace("0.0.0.0", collector));
} else if (currentValue.startsWith("localhost")) {
newProperties.put(AMS_WEBAPP_ADDRESS_PROPERTY, currentValue.replace("localhost", collector));
}
}
}
if (amsSiteProperties.containsKey(HOST_AGGREGATOR_DAILY_CHECKPOINTCUTOFFMULTIPIER_PROPERTY) &&
amsSiteProperties.get(HOST_AGGREGATOR_DAILY_CHECKPOINTCUTOFFMULTIPIER_PROPERTY).equals("1")) {
LOG.info("Setting value of " + HOST_AGGREGATOR_DAILY_CHECKPOINTCUTOFFMULTIPIER_PROPERTY + " : 2");
newProperties.put(HOST_AGGREGATOR_DAILY_CHECKPOINTCUTOFFMULTIPIER_PROPERTY, String.valueOf(2));
}
if (amsSiteProperties.containsKey(CLUSTER_AGGREGATOR_DAILY_CHECKPOINTCUTOFFMULTIPIER_PROPERTY) &&
amsSiteProperties.get(CLUSTER_AGGREGATOR_DAILY_CHECKPOINTCUTOFFMULTIPIER_PROPERTY).equals("1")) {
LOG.info("Setting value of " + CLUSTER_AGGREGATOR_DAILY_CHECKPOINTCUTOFFMULTIPIER_PROPERTY + " : 2");
newProperties.put(CLUSTER_AGGREGATOR_DAILY_CHECKPOINTCUTOFFMULTIPIER_PROPERTY, String.valueOf(2));
}
if (!amsSiteProperties.containsKey(TIMELINE_METRICS_SERVICE_WATCHER_DISBALED_PROPERTY)) {
LOG.info("Add config " + TIMELINE_METRICS_SERVICE_WATCHER_DISBALED_PROPERTY + " = false");
newProperties.put(TIMELINE_METRICS_SERVICE_WATCHER_DISBALED_PROPERTY, String.valueOf(false));
}
boolean isDistributed = false;
if ("distributed".equals(amsSite.getProperties().get(AMS_MODE_PROPERTY))) {
isDistributed = true;
}
if (amsSiteProperties.containsKey(PRECISION_TABLE_TTL_PROPERTY)) {
String oldTtl = amsSiteProperties.get(PRECISION_TABLE_TTL_PROPERTY);
String newTtl = oldTtl;
if (isDistributed) {
if ("86400".equals(oldTtl)) {
newTtl = String.valueOf(3 * 86400); // 3 days
}
}
newProperties.put(PRECISION_TABLE_TTL_PROPERTY, newTtl);
LOG.info("Setting value of " + PRECISION_TABLE_TTL_PROPERTY + " : " + newTtl);
}
if (amsSiteProperties.containsKey(CLUSTER_SECOND_TABLE_TTL_PROPERTY)) {
String oldTtl = amsSiteProperties.get(CLUSTER_SECOND_TABLE_TTL_PROPERTY);
String newTtl = oldTtl;
if ("2592000".equals(oldTtl)) {
newTtl = String.valueOf(7 * 86400); // 7 days
}
newProperties.put(CLUSTER_SECOND_TABLE_TTL_PROPERTY, newTtl);
LOG.info("Setting value of " + CLUSTER_SECOND_TABLE_TTL_PROPERTY + " : " + newTtl);
}
if (amsSiteProperties.containsKey(CLUSTER_MINUTE_TABLE_TTL_PROPERTY)) {
String oldTtl = amsSiteProperties.get(CLUSTER_MINUTE_TABLE_TTL_PROPERTY);
String newTtl = oldTtl;
if ("7776000".equals(oldTtl)) {
newTtl = String.valueOf(30 * 86400); // 30 days
}
newProperties.put(CLUSTER_MINUTE_TABLE_TTL_PROPERTY, newTtl);
LOG.info("Setting value of " + CLUSTER_MINUTE_TABLE_TTL_PROPERTY + " : " + newTtl);
}
if (!amsSiteProperties.containsKey(TIMELINE_METRICS_CLUSTER_AGGREGATOR_INTERPOLATION_ENABLED)) {
LOG.info("Add config " + TIMELINE_METRICS_CLUSTER_AGGREGATOR_INTERPOLATION_ENABLED + " = true");
newProperties.put(TIMELINE_METRICS_CLUSTER_AGGREGATOR_INTERPOLATION_ENABLED, String.valueOf(true));
}
if (!amsSiteProperties.containsKey(TIMELINE_METRICS_SINK_COLLECTION_PERIOD) ||
"60".equals(amsSiteProperties.get(TIMELINE_METRICS_SINK_COLLECTION_PERIOD))) {
newProperties.put(TIMELINE_METRICS_SINK_COLLECTION_PERIOD, "10");
LOG.info("Setting value of " + TIMELINE_METRICS_SINK_COLLECTION_PERIOD + " : 10");
}
updateConfigurationPropertiesForCluster(cluster, AMS_SITE, newProperties, true, true);
}
Config amsHbaseSite = cluster.getDesiredConfigByType(AMS_HBASE_SITE);
if (amsHbaseSite != null) {
Map<String, String> amsHbaseSiteProperties = amsHbaseSite.getProperties();
Map<String, String> newProperties = new HashMap<>();
if (!amsHbaseSiteProperties.containsKey(HBASE_RPC_TIMEOUT_PROPERTY)) {
newProperties.put(HBASE_RPC_TIMEOUT_PROPERTY, String.valueOf(300000));
}
if (!amsHbaseSiteProperties.containsKey(PHOENIX_QUERY_KEEPALIVE_PROPERTY)) {
newProperties.put(PHOENIX_QUERY_KEEPALIVE_PROPERTY, String.valueOf(300000));
}
if (!amsHbaseSiteProperties.containsKey(HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD_PROPERTY) ||
amsHbaseSiteProperties.get(HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD_PROPERTY).equals("900000")) {
amsHbaseSiteProperties.put(HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD_PROPERTY, String.valueOf(300000));
}
if (!amsHbaseSiteProperties.containsKey(PHOENIX_QUERY_TIMEOUT_PROPERTY) ||
amsHbaseSiteProperties.get(PHOENIX_QUERY_TIMEOUT_PROPERTY).equals("1200000")) {
amsHbaseSiteProperties.put(PHOENIX_QUERY_TIMEOUT_PROPERTY, String.valueOf(300000));
}
updateConfigurationPropertiesForCluster(cluster, AMS_HBASE_SITE, newProperties, true, true);
}
}
}
}
}
protected void updateHDFSWidgetDefinition() throws AmbariException {
LOG.info("Updating HDFS widget definition.");
Map<String, List<String>> widgetMap = new HashMap<>();
Map<String, String> sectionLayoutMap = new HashMap<>();
List<String> hdfsSummaryWidgets = new ArrayList<>(Arrays.asList("NameNode RPC", "NN Connection Load",
"NameNode GC count", "NameNode GC time", "NameNode Host Load"));
widgetMap.put("HDFS_SUMMARY", hdfsSummaryWidgets);
sectionLayoutMap.put("HDFS_SUMMARY", "default_hdfs_dashboard");
List<String> hdfsHeatmapWidgets = new ArrayList<>(Arrays.asList("HDFS Bytes Read", "HDFS Bytes Written",
"DataNode Process Disk I/O Utilization", "DataNode Process Network I/O Utilization"));
widgetMap.put("HDFS_HEATMAPS", hdfsHeatmapWidgets);
sectionLayoutMap.put("HDFS_HEATMAPS", "default_hdfs_heatmap");
updateWidgetDefinitionsForService("HDFS", widgetMap, sectionLayoutMap);
}
protected void updateYARNWidgetDefinition() throws AmbariException {
LOG.info("Updating YARN widget definition.");
Map<String, List<String>> widgetMap = new HashMap<>();
Map<String, String> sectionLayoutMap = new HashMap<>();
List<String> yarnSummaryWidgets = new ArrayList<>(Arrays.asList("Container Failures", "App Failures", "Cluster Memory"));
widgetMap.put("YARN_SUMMARY", yarnSummaryWidgets);
sectionLayoutMap.put("YARN_SUMMARY", "default_yarn_dashboard");
List<String> yarnHeatmapWidgets = new ArrayList<>(Arrays.asList("Container Failures"));
widgetMap.put("YARN_HEATMAPS", yarnHeatmapWidgets);
sectionLayoutMap.put("YARN_HEATMAPS", "default_yarn_heatmap");
updateWidgetDefinitionsForService("YARN", widgetMap, sectionLayoutMap);
}
protected void updateHBASEWidgetDefinition() throws AmbariException {
LOG.info("Updating HBASE widget definition.");
Map<String, List<String>> widgetMap = new HashMap<>();
Map<String, String> sectionLayoutMap = new HashMap<>();
List<String> hbaseSummaryWidgets = new ArrayList<>(Arrays.asList("Reads and Writes", "Blocked Updates"));
widgetMap.put("HBASE_SUMMARY", hbaseSummaryWidgets);
sectionLayoutMap.put("HBASE_SUMMARY", "default_hbase_dashboard");
updateWidgetDefinitionsForService("HBASE", widgetMap, sectionLayoutMap);
}
protected void updateHbaseEnvConfig() throws AmbariException {
AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
for (final Cluster cluster : getCheckedClusterMap(ambariManagementController.getClusters()).values()) {
Config hbaseEnvConfig = cluster.getDesiredConfigByType(HBASE_ENV_CONFIG);
if (hbaseEnvConfig != null) {
Map<String, String> updates = getUpdatedHbaseEnvProperties(hbaseEnvConfig.getProperties().get(CONTENT_PROPERTY));
if (!updates.isEmpty()) {
updateConfigurationPropertiesForCluster(cluster, HBASE_ENV_CONFIG, updates, true, false);
}
}
}
}
protected Map<String, String> getUpdatedHbaseEnvProperties(String content) {
if (content != null) {
//Fix bad config added in Upgrade 2.2.0.
String badConfig = "export HBASE_OPTS=\"-Djava.io.tmpdir={{java_io_tmpdir}}\"";
String correctConfig = "export HBASE_OPTS=\"${HBASE_OPTS} -Djava.io.tmpdir={{java_io_tmpdir}}\"";
if (content.contains(badConfig)) {
content = content.replace(badConfig, correctConfig);
return Collections.singletonMap(CONTENT_PROPERTY, content);
}
}
return Collections.emptyMap();
}
protected void updateWidgetDefinitionsForService(String serviceName, Map<String, List<String>> widgetMap,
Map<String, String> sectionLayoutMap) throws AmbariException {
AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
AmbariMetaInfo ambariMetaInfo = injector.getInstance(AmbariMetaInfo.class);
Type widgetLayoutType = new TypeToken<Map<String, List<WidgetLayout>>>(){}.getType();
Gson gson = injector.getInstance(Gson.class);
WidgetDAO widgetDAO = injector.getInstance(WidgetDAO.class);
Clusters clusters = ambariManagementController.getClusters();
Map<String, Cluster> clusterMap = getCheckedClusterMap(clusters);
for (final Cluster cluster : clusterMap.values()) {
long clusterID = cluster.getClusterId();
StackId stackId = cluster.getDesiredStackVersion();
Map<String, Object> widgetDescriptor = null;
StackInfo stackInfo = ambariMetaInfo.getStack(stackId.getStackName(), stackId.getStackVersion());
ServiceInfo serviceInfo = stackInfo.getService(serviceName);
if (serviceInfo == null) {
LOG.info("Skipping updating widget definition, because " + serviceName + " service is not present in cluster " +
"cluster_name= " + cluster.getClusterName());
continue;
}
for (String section : widgetMap.keySet()) {
List<String> widgets = widgetMap.get(section);
for (String widgetName : widgets) {
List<WidgetEntity> widgetEntities = widgetDAO.findByName(clusterID,
widgetName, "ambari", section);
if (widgetEntities != null && widgetEntities.size() > 0) {
WidgetEntity entityToUpdate = null;
if (widgetEntities.size() > 1) {
LOG.info("Found more that 1 entity with name = "+ widgetName +
" for cluster = " + cluster.getClusterName() + ", skipping update.");
} else {
entityToUpdate = widgetEntities.iterator().next();
}
if (entityToUpdate != null) {
LOG.info("Updating widget: " + entityToUpdate.getWidgetName());
// Get the definition from widgets.json file
WidgetLayoutInfo targetWidgetLayoutInfo = null;
File widgetDescriptorFile = serviceInfo.getWidgetsDescriptorFile();
if (widgetDescriptorFile != null && widgetDescriptorFile.exists()) {
try {
widgetDescriptor = gson.fromJson(new FileReader(widgetDescriptorFile), widgetLayoutType);
} catch (Exception ex) {
String msg = "Error loading widgets from file: " + widgetDescriptorFile;
LOG.error(msg, ex);
widgetDescriptor = null;
}
}
if (widgetDescriptor != null) {
LOG.debug("Loaded widget descriptor: " + widgetDescriptor);
for (Object artifact : widgetDescriptor.values()) {
List<WidgetLayout> widgetLayouts = (List<WidgetLayout>) artifact;
for (WidgetLayout widgetLayout : widgetLayouts) {
if (widgetLayout.getLayoutName().equals(sectionLayoutMap.get(section))) {
for (WidgetLayoutInfo layoutInfo : widgetLayout.getWidgetLayoutInfoList()) {
if (layoutInfo.getWidgetName().equals(widgetName)) {
targetWidgetLayoutInfo = layoutInfo;
}
}
}
}
}
}
if (targetWidgetLayoutInfo != null) {
entityToUpdate.setMetrics(gson.toJson(targetWidgetLayoutInfo.getMetricsInfo()));
entityToUpdate.setWidgetValues(gson.toJson(targetWidgetLayoutInfo.getValues()));
if ("HBASE".equals(serviceName) && "Reads and Writes".equals(widgetName)) {
entityToUpdate.setDescription(targetWidgetLayoutInfo.getDescription());
LOG.info("Update description for HBase Reads and Writes widget");
}
widgetDAO.merge(entityToUpdate);
} else {
LOG.warn("Unable to find widget layout info for " + widgetName +
" in the stack: " + stackId);
}
}
}
}
}
}
}
protected void updateHiveConfig() throws AmbariException {
AmbariManagementController ambariManagementController = injector.getInstance(AmbariManagementController.class);
for (final Cluster cluster : getCheckedClusterMap(ambariManagementController.getClusters()).values()) {
Config hiveSiteConfig = cluster.getDesiredConfigByType(HIVE_SITE_CONFIG);
Config atlasConfig = cluster.getDesiredConfigByType(ATLAS_APPLICATION_PROPERTIES_CONFIG);
StackId stackId = cluster.getCurrentStackVersion();
boolean isStackNotLess23 = (stackId != null && stackId.getStackName().equals("HDP") &&
VersionUtils.compareVersions(stackId.getStackVersion(), "2.3") >= 0);
List<ServiceComponentHost> atlasHost = cluster.getServiceComponentHosts("ATLAS", "ATLAS_SERVER");
Map<String, String> updates = new HashMap<>();
if (isStackNotLess23 && atlasHost.size() != 0 && hiveSiteConfig != null) {
updates.put(ATLAS_HOOK_HIVE_MINTHREADS_PROPERTY, "1");
updates.put(ATLAS_HOOK_HIVE_MAXTHREADS_PROPERTY, "1");
updates.put(ATLAS_CLUSTER_NAME_PROPERTY, "primary");
if (atlasConfig != null && atlasConfig.getProperties().containsKey(ATLAS_ENABLETLS_PROPERTY)) {
String atlasEnableTLSProperty = atlasConfig.getProperties().get(ATLAS_ENABLETLS_PROPERTY);
String atlasScheme = "http";
String atlasServerHttpPortProperty = atlasConfig.getProperties().get(ATLAS_SERVER_HTTP_PORT_PROPERTY);
if (atlasEnableTLSProperty.toLowerCase().equals("true")) {
atlasServerHttpPortProperty = atlasConfig.getProperties().get(ATLAS_SERVER_HTTPS_PORT_PROPERTY);
atlasScheme = "https";
}
updates.put(ATLAS_REST_ADDRESS_PROPERTY, String.format("%s://%s:%s", atlasScheme, atlasHost.get(0).getHostName(), atlasServerHttpPortProperty));
}
updateConfigurationPropertiesForCluster(cluster, HIVE_SITE_CONFIG, updates, false, false);
}
}
}
protected void updateCorruptedReplicaWidget() throws SQLException {
String widgetValues = String.format("[{\"name\": \"%s\", \"value\": \"%s\"}]",
WIDGET_CORRUPT_REPLICAS, WIDGET_VALUES_VALUE);
String updateStatement = "UPDATE %s SET %s='%s', %s='%s', %s='%s' WHERE %s='%s'";
LOG.info("Update widget definition for HDFS corrupted blocks metric");
dbAccessor.executeUpdate(String.format(updateStatement,
WIDGET_TABLE,
WIDGET_NAME, WIDGET_CORRUPT_REPLICAS,
WIDGET_DESCRIPTION, WIDGET_CORRUPT_REPLICAS_DESCRIPTION,
WIDGET_VALUES, widgetValues,
WIDGET_NAME, WIDGET_CORRUPT_BLOCKS
));
}
/**
* Updates the {@value #UPGRADE_TABLE} in the following ways:
* <ul>
* <li>{value {@link #UPGRADE_SUSPENDED_COLUMN} is added</li>
* </ul>
*
* @throws AmbariException
* @throws SQLException
*/
protected void updateUpgradeTable() throws AmbariException, SQLException {
dbAccessor.addColumn(UPGRADE_TABLE,
new DBAccessor.DBColumnInfo(UPGRADE_SUSPENDED_COLUMN, Short.class, 1, 0, false));
}
/**
* Copy cluster & service widgets for Storm and Kafka from stack to DB.
*/
protected void initializeStromAndKafkaWidgets() throws AmbariException {
AmbariManagementController controller = injector.getInstance(AmbariManagementController.class);
Clusters clusters = controller.getClusters();
if (clusters == null) {
return;
}
Map<String, Cluster> clusterMap = clusters.getClusters();
if (clusterMap != null && !clusterMap.isEmpty()) {
for (Cluster cluster : clusterMap.values()) {
Map<String, Service> serviceMap = cluster.getServices();
if (serviceMap != null && !serviceMap.isEmpty()) {
for (Service service : serviceMap.values()) {
if ("STORM".equals(service.getName()) || "KAFKA".equals(service.getName())) {
controller.initializeWidgetsAndLayouts(cluster, service);
}
}
}
}
}
}
}