/** * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.ambari.server.state.stack.upgrade; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; import javax.xml.bind.annotation.XmlType; import org.apache.ambari.server.AmbariException; import org.apache.ambari.server.Role; import org.apache.ambari.server.RoleCommand; import org.apache.ambari.server.actionmanager.HostRoleCommand; import org.apache.ambari.server.actionmanager.HostRoleCommandFactory; import org.apache.ambari.server.api.services.AmbariMetaInfo; import org.apache.ambari.server.metadata.RoleCommandOrder; import org.apache.ambari.server.stack.HostsType; import org.apache.ambari.server.stageplanner.RoleGraph; import org.apache.ambari.server.stageplanner.RoleGraphFactory; import org.apache.ambari.server.state.Cluster; import org.apache.ambari.server.state.ComponentInfo; import org.apache.ambari.server.state.ServiceComponentHost; import org.apache.ambari.server.state.StackId; import org.apache.ambari.server.state.UpgradeContext; import org.apache.ambari.server.state.stack.UpgradePack.ProcessingComponent; import org.apache.commons.collections.CollectionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.gson.JsonObject; /** * Marker group for Host-Ordered upgrades. */ @XmlType(name="host-order") public class HostOrderGrouping extends Grouping { private static final String TYPE = "type"; private static final String HOST = "host"; private static Logger LOG = LoggerFactory.getLogger(HostOrderGrouping.class); /** * Contains the ordered actions to schedule for this grouping. */ private List<HostOrderItem> m_hostOrderItems; /** * Constructor */ public HostOrderGrouping() { } /** * Sets the {@link HostOrderItem}s on this grouping. * * @param hostOrderItems */ public void setHostOrderItems(List<HostOrderItem> hostOrderItems) { m_hostOrderItems = hostOrderItems; } @Override public StageWrapperBuilder getBuilder() { return new HostBuilder(this); } /** * Builder for host upgrades. */ private static class HostBuilder extends StageWrapperBuilder { private final List<HostOrderItem> hostOrderItems; /** * @param grouping the grouping */ protected HostBuilder(HostOrderGrouping grouping) { super(grouping); hostOrderItems = grouping.m_hostOrderItems; } @Override public void add(UpgradeContext upgradeContext, HostsType hostsType, String service, boolean clientOnly, ProcessingComponent pc, Map<String, String> params) { // !!! NOOP, this called when there are services in the group, and there // are none for host-ordered. } @Override public List<StageWrapper> build(UpgradeContext upgradeContext, List<StageWrapper> stageWrappers) { List<StageWrapper> wrappers = new ArrayList<>(stageWrappers); for (HostOrderItem orderItem : hostOrderItems) { switch (orderItem.getType()) { case HOST_UPGRADE: wrappers.addAll(buildHosts(upgradeContext, orderItem.getActionItems())); break; case SERVICE_CHECK: wrappers.addAll(buildServiceChecks(upgradeContext, orderItem.getActionItems())); break; } } return wrappers; } /** * Builds the stages for each host which typically consist of a STOP, a * manual wait, and a START. The starting of components can be a single * stage or may consist of several stages if the host components have * dependencies on each other. * * @param upgradeContext * the context * @param hosts * the list of hostnames * @return the wrappers for a host */ private List<StageWrapper> buildHosts(UpgradeContext upgradeContext, List<String> hosts) { if (CollectionUtils.isEmpty(hosts)) { return Collections.emptyList(); } Cluster cluster = upgradeContext.getCluster(); List<StageWrapper> wrappers = new ArrayList<>(); HostRoleCommandFactory hrcFactory = upgradeContext.getHostRoleCommandFactory(); // get a role command order instance that we can adjust for HOU since HOU // may use a different ordering than normal start operations RoleCommandOrder roleCommandOrder = getRoleCommandOrderForUpgrade(cluster); for (String hostName : hosts) { // initialize the collection for all stop tasks for every component on // the host List<TaskWrapper> stopTasks = new ArrayList<>(); // initialize the collection which will be passed into the RoleGraph for // ordering Map<String, Map<String, HostRoleCommand>> restartCommandsForHost = new HashMap<>(); Map<String, HostRoleCommand> restartCommandsByRole = new HashMap<>(); restartCommandsForHost.put(hostName, restartCommandsByRole); // iterating over every host component, build the commands for (ServiceComponentHost sch : cluster.getServiceComponentHosts(hostName)) { if (!isVersionAdvertised(upgradeContext, sch)) { continue; } HostsType hostsType = upgradeContext.getResolver().getMasterAndHosts( sch.getServiceName(), sch.getServiceComponentName()); // !!! if the hosts do not contain the current one, that means the component // either doesn't exist or the downgrade is to the current target version. // hostsType better not be null either, but check anyway if (null != hostsType && !hostsType.hosts.contains(hostName)) { LOG.warn("Host {} could not be orchestrated. Either there are no components for {}/{} " + "or the target version {} is already current.", hostName, sch.getServiceName(), sch.getServiceComponentName(), upgradeContext.getVersion()); continue; } // create a STOP task for this host component if (!sch.isClientComponent()) { stopTasks.add(new TaskWrapper(sch.getServiceName(), sch.getServiceComponentName(), Collections.singleton(hostName), new StopTask())); } // generate a placeholder HRC that can be used to generate the // dependency graph - we must use START here since that's what the // role command order is defined with - each of these will turn into a // RESTART when we create the wrappers later on Role role = Role.valueOf(sch.getServiceComponentName()); HostRoleCommand hostRoleCommand = hrcFactory.create(hostName, role, null, RoleCommand.START); // add the newly created HRC RESTART restartCommandsByRole.put(role.name(), hostRoleCommand); } // short circuit and move to the next host if there are no commands if (stopTasks.isEmpty() && restartCommandsByRole.isEmpty()) { LOG.info("There were no {} commands generated for {}", upgradeContext.getDirection().getText(false), hostName); continue; } // now process the HRCs created so that we can create the appropriate // stage/task wrappers for the RESTARTs RoleGraphFactory roleGraphFactory = upgradeContext.getRoleGraphFactory(); RoleGraph roleGraph = roleGraphFactory.createNew(roleCommandOrder); List<Map<String, List<HostRoleCommand>>> stages = roleGraph.getOrderedHostRoleCommands( restartCommandsForHost); // initialize the list of stage wrappers List<StageWrapper> stageWrappers = new ArrayList<>(); // for every stage, create a stage wrapper around the tasks int phaseCounter = 1; for (Map<String, List<HostRoleCommand>> stage : stages) { List<HostRoleCommand> stageCommandsForHost = stage.get(hostName); String stageTitle = String.format("Starting components on %s (phase %d)", hostName, phaseCounter++); // create task wrappers List<TaskWrapper> taskWrappers = new ArrayList<>(); for (HostRoleCommand command : stageCommandsForHost) { StackId stackId = upgradeContext.getEffectiveStackId(); String componentName = command.getRole().name(); String serviceName = null; try { AmbariMetaInfo ambariMetaInfo = upgradeContext.getAmbariMetaInfo(); serviceName = ambariMetaInfo.getComponentToService(stackId.getStackName(), stackId.getStackVersion(), componentName); } catch (AmbariException ambariException) { LOG.error("Unable to lookup service by component {} for stack {}-{}", componentName, stackId.getStackName(), stackId.getStackVersion()); } TaskWrapper taskWrapper = new TaskWrapper(serviceName, componentName, Collections.singleton(hostName), new RestartTask()); taskWrappers.add(taskWrapper); } if (!taskWrappers.isEmpty()) { StageWrapper startWrapper = new StageWrapper(StageWrapper.Type.RESTART, stageTitle, taskWrappers.toArray(new TaskWrapper[taskWrappers.size()])); stageWrappers.add(startWrapper); } } // create the manual task between the STOP and START stages ManualTask mt = new ManualTask(); String message = String.format("Please acknowledge that host %s has been prepared.", hostName); mt.messages.add(message); JsonObject structuredOut = new JsonObject(); structuredOut.addProperty(TYPE, HostOrderItem.HostOrderActionType.HOST_UPGRADE.toString()); structuredOut.addProperty(HOST, hostName); mt.structuredOut = structuredOut.toString(); // build the single STOP stage, but only if there are components to // stop; client-only hosts have no components which need stopping if (!stopTasks.isEmpty()) { StageWrapper stopWrapper = new StageWrapper(StageWrapper.Type.STOP, String.format("Stop on %s", hostName), stopTasks.toArray(new TaskWrapper[stopTasks.size()])); wrappers.add(stopWrapper); } StageWrapper manualWrapper = new StageWrapper(StageWrapper.Type.SERVER_SIDE_ACTION, "Manual Confirmation", new TaskWrapper(null, null, Collections.<String>emptySet(), mt)); wrappers.add(manualWrapper); // !!! TODO install_packages for hdp and conf-select changes. Hopefully these will no-op. wrappers.addAll(stageWrappers); } return wrappers; } /** * @param upgradeContext the context * @return the wrappers for a host */ private List<StageWrapper> buildServiceChecks(UpgradeContext upgradeContext, List<String> serviceChecks) { if (CollectionUtils.isEmpty(serviceChecks)) { return Collections.emptyList(); } List<StageWrapper> wrappers = new ArrayList<>(); Cluster cluster = upgradeContext.getCluster(); for (String serviceName : serviceChecks) { boolean hasService = false; try { cluster.getService(serviceName); hasService = true; } catch (Exception e) { LOG.warn("Service {} not found to orchestrate", serviceName); } if (!hasService) { continue; } StageWrapper wrapper = new StageWrapper(StageWrapper.Type.SERVICE_CHECK, String.format("Service Check %s", upgradeContext.getServiceDisplay(serviceName)), new TaskWrapper(serviceName, "", Collections.<String>emptySet(), new ServiceCheckTask())); wrappers.add(wrapper); } return wrappers; } /** * @param upgradeContext the context * @param sch the host component * @return {@code true} if the host component advertises its version */ private boolean isVersionAdvertised(UpgradeContext upgradeContext, ServiceComponentHost sch) { StackId targetStack = upgradeContext.getTargetStackId(); try { ComponentInfo component = upgradeContext.getAmbariMetaInfo().getComponent( targetStack.getStackName(), targetStack.getStackVersion(), sch.getServiceName(), sch.getServiceComponentName()); return component.isVersionAdvertised(); } catch (AmbariException e) { LOG.warn("Could not determine if {}/{}/{} could be upgraded; returning false", targetStack, sch.getServiceName(), sch.getServiceComponentName(), e); return false; } } /** * Gets a {@link RoleCommandOrder} instance initialized with * {@code host_ordered_upgrade} overrides. * * @param cluster * the cluster to get the {@link RoleCommandOrder} instance for. * @return the order of commands for the cluster */ private RoleCommandOrder getRoleCommandOrderForUpgrade(Cluster cluster) { RoleCommandOrder roleCommandOrder = cluster.getRoleCommandOrder(); try { roleCommandOrder = (RoleCommandOrder) roleCommandOrder.clone(); } catch (CloneNotSupportedException cloneNotSupportedException) { LOG.warn("Unable to clone role command order and apply overrides for this upgrade", cloneNotSupportedException); } LinkedHashSet<String> sectionKeys = roleCommandOrder.getSectionKeys(); sectionKeys.add("host_ordered_upgrade"); roleCommandOrder.initialize(cluster, sectionKeys); return roleCommandOrder; } } }