/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.ambari.server.state.stack.upgrade;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import javax.xml.bind.annotation.XmlAttribute;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlSeeAlso;
import org.apache.ambari.server.AmbariException;
import org.apache.ambari.server.stack.HostsType;
import org.apache.ambari.server.state.UpgradeContext;
import org.apache.ambari.server.state.stack.UpgradePack;
import org.apache.ambari.server.state.stack.UpgradePack.OrderService;
import org.apache.ambari.server.state.stack.UpgradePack.ProcessingComponent;
import org.apache.ambari.server.utils.SetUtils;
import org.apache.commons.lang.StringUtils;
import com.google.common.base.Objects;
/**
*
*/
@XmlSeeAlso(value = { ColocatedGrouping.class, ClusterGrouping.class,
UpdateStackGrouping.class, ServiceCheckGrouping.class, RestartGrouping.class,
StartGrouping.class, StopGrouping.class, HostOrderGrouping.class })
public class Grouping {
@XmlAttribute(name="name")
public String name;
@XmlAttribute(name="title")
public String title;
@XmlElement(name="add-after-group")
public String addAfterGroup;
@XmlElement(name="add-after-group-entry")
public String addAfterGroupEntry;
@XmlElement(name="skippable", defaultValue="false")
public boolean skippable = false;
@XmlElement(name = "supports-auto-skip-failure", defaultValue = "true")
public boolean supportsAutoSkipOnFailure = true;
@XmlElement(name="allow-retry", defaultValue="true")
public boolean allowRetry = true;
@XmlElement(name="service")
public List<UpgradePack.OrderService> services = new ArrayList<>();
@XmlElement(name="service-check", defaultValue="true")
public boolean performServiceCheck = true;
@XmlElement(name="direction")
public Direction intendedDirection = null;
@XmlElement(name="parallel-scheduler")
public ParallelScheduler parallelScheduler;
@XmlElement(name="scope")
public UpgradeScope scope = UpgradeScope.ANY;
/**
* A condition element with can prevent this entire group from being scheduled
* in the upgrade.
*/
@XmlElement(name = "condition")
public Condition condition;
/**
* Gets the default builder.
*/
public StageWrapperBuilder getBuilder() {
return new DefaultBuilder(this, performServiceCheck);
}
private static class DefaultBuilder extends StageWrapperBuilder {
private List<StageWrapper> m_stages = new ArrayList<>();
private Set<String> m_servicesToCheck = new HashSet<>();
private boolean m_serviceCheck = true;
private DefaultBuilder(Grouping grouping, boolean serviceCheck) {
super(grouping);
m_serviceCheck = serviceCheck;
}
/**
* Add stages where the restart stages are ordered
* E.g., preupgrade, restart hosts(0), ..., restart hosts(n-1), postupgrade
* @param context the context
* @param hostsType the order collection of hosts, which may have a master and secondary
* @param service the service name
* @param pc the ProcessingComponent derived from the upgrade pack.
* @param params additional parameters
*/
@Override
public void add(UpgradeContext context, HostsType hostsType, String service,
boolean clientOnly, ProcessingComponent pc, Map<String, String> params) {
// Construct the pre tasks during Upgrade/Downgrade direction.
// Buckets are grouped by the type, e.g., bucket of all Execute tasks, or all Configure tasks.
List<TaskBucket> buckets = buckets(resolveTasks(context, true, pc));
for (TaskBucket bucket : buckets) {
// The TaskWrappers take into account if a task is meant to run on all, any, or master.
// A TaskWrapper may contain multiple tasks, but typically only one, and they all run on the same set of hosts.
// Generate a task wrapper for every task in the bucket
List<TaskWrapper> preTasks = TaskWrapperBuilder.getTaskList(service, pc.name, hostsType, bucket.tasks, params);
List<List<TaskWrapper>> organizedTasks = organizeTaskWrappersBySyncRules(preTasks);
for (List<TaskWrapper> tasks : organizedTasks) {
addTasksToStageInBatches(tasks, "Preparing", context, service, pc, params);
}
}
// Add the processing component
Task t = resolveTask(context, pc);
if (null != t) {
TaskWrapper tw = new TaskWrapper(service, pc.name, hostsType.hosts, params, Collections.singletonList(t));
addTasksToStageInBatches(Collections.singletonList(tw), t.getActionVerb(), context, service, pc, params);
}
// Construct the post tasks during Upgrade/Downgrade direction.
buckets = buckets(resolveTasks(context, false, pc));
for (TaskBucket bucket : buckets) {
List<TaskWrapper> postTasks = TaskWrapperBuilder.getTaskList(service, pc.name, hostsType, bucket.tasks, params);
List<List<TaskWrapper>> organizedTasks = organizeTaskWrappersBySyncRules(postTasks);
for (List<TaskWrapper> tasks : organizedTasks) {
addTasksToStageInBatches(tasks, "Completing", context, service, pc, params);
}
}
// Potentially add a service check
if (m_serviceCheck && !clientOnly) {
m_servicesToCheck.add(service);
}
}
/**
* Split a list of TaskWrappers into a list of lists where a TaskWrapper that has any task with isSequential == true
* must be a singleton in its own list.
* @param tasks List of TaskWrappers to analyze
* @return List of list of TaskWrappers, where each outer list is a separate stage.
*/
private List<List<TaskWrapper>> organizeTaskWrappersBySyncRules(List<TaskWrapper> tasks) {
List<List<TaskWrapper>> groupedTasks = new ArrayList<>();
List<TaskWrapper> subTasks = new ArrayList<>();
for (TaskWrapper tw : tasks) {
// If an of this TaskWrapper's tasks must be on its own stage, write out the previous subtasks if possible into one complete stage.
if (tw.isAnyTaskSequential()) {
if (!subTasks.isEmpty()) {
groupedTasks.add(subTasks);
subTasks = new ArrayList<>();
}
groupedTasks.add(Collections.singletonList(tw));
} else {
subTasks.add(tw);
}
}
if (!subTasks.isEmpty()) {
groupedTasks.add(subTasks);
}
return groupedTasks;
}
/**
* Helper function to analyze a ProcessingComponent and add its task to stages, depending on the batch size.
* @param tasks Collection of tasks for this stage
* @param verb Verb string to use in the title of the task
* @param ctx Upgrade Context
* @param service Service
* @param pc Processing Component
* @param params Params to add to the stage.
*/
private void addTasksToStageInBatches(List<TaskWrapper> tasks, String verb, UpgradeContext ctx, String service, ProcessingComponent pc, Map<String, String> params) {
if (tasks == null || tasks.isEmpty() || tasks.get(0).getTasks() == null || tasks.get(0).getTasks().isEmpty()) {
return;
}
// Our assumption is that all of the tasks in the StageWrapper are of the same type.
StageWrapper.Type type = tasks.get(0).getTasks().get(0).getStageWrapperType();
// Expand some of the TaskWrappers into multiple based on the batch size.
for (TaskWrapper tw : tasks) {
List<Set<String>> hostSets = null;
if (m_grouping.parallelScheduler != null && m_grouping.parallelScheduler.maxDegreeOfParallelism > 0) {
hostSets = SetUtils.split(tw.getHosts(), m_grouping.parallelScheduler.maxDegreeOfParallelism);
} else {
hostSets = SetUtils.split(tw.getHosts(), 1);
}
int numBatchesNeeded = hostSets.size();
int batchNum = 0;
for (Set<String> hostSubset : hostSets) {
batchNum++;
String stageText = getStageText(verb, ctx.getComponentDisplay(service, pc.name), hostSubset, batchNum, numBatchesNeeded);
StageWrapper stage = new StageWrapper(
type,
stageText,
params,
new TaskWrapper(service, pc.name, hostSubset, params, tw.getTasks()));
m_stages.add(stage);
}
}
}
/**
* Determine if service checks need to be ran after the stages.
* @param upgradeContext the upgrade context
* @return Return the stages, which may potentially be followed by service checks.
*/
@Override
public List<StageWrapper> build(UpgradeContext upgradeContext,
List<StageWrapper> stageWrappers) {
// insert all pre-processed stage wrappers first
if (!stageWrappers.isEmpty()) {
m_stages.addAll(0, stageWrappers);
}
List<TaskWrapper> tasks = new ArrayList<>();
List<String> displays = new ArrayList<>();
for (String service : m_servicesToCheck) {
tasks.add(new TaskWrapper(
service, "", Collections.<String>emptySet(), new ServiceCheckTask()));
displays.add(upgradeContext.getServiceDisplay(service));
}
if (upgradeContext.getDirection().isUpgrade() && m_serviceCheck
&& m_servicesToCheck.size() > 0) {
StageWrapper wrapper = new StageWrapper(StageWrapper.Type.SERVICE_CHECK,
"Service Check " + StringUtils.join(displays, ", "), tasks.toArray(new TaskWrapper[0]));
m_stages.add(wrapper);
}
return m_stages;
}
}
/**
* Group all like-typed tasks together. When they change, create a new type.
*/
private static List<TaskBucket> buckets(List<Task> tasks) {
if (null == tasks || tasks.isEmpty()) {
return Collections.emptyList();
}
List<TaskBucket> holders = new ArrayList<>();
TaskBucket current = null;
int i = 0;
for (Task t : tasks) {
if (i == 0) {
current = new TaskBucket(t);
holders.add(current);
} else if (i > 0 && t.getType() != tasks.get(i-1).getType()) {
current = new TaskBucket(t);
holders.add(current);
} else {
current.tasks.add(t);
}
i++;
}
return holders;
}
private static class TaskBucket {
private StageWrapper.Type type;
private List<Task> tasks = new ArrayList<>();
private TaskBucket(Task initial) {
switch (initial.getType()) {
case CONFIGURE:
case SERVER_ACTION:
case MANUAL:
type = StageWrapper.Type.SERVER_SIDE_ACTION;
break;
case EXECUTE:
type = StageWrapper.Type.RU_TASKS;
break;
case CONFIGURE_FUNCTION:
type = StageWrapper.Type.CONFIGURE;
break;
case RESTART:
type = StageWrapper.Type.RESTART;
break;
case START:
type = StageWrapper.Type.START;
break;
case STOP:
type = StageWrapper.Type.STOP;
break;
case SERVICE_CHECK:
type = StageWrapper.Type.SERVICE_CHECK;
break;
}
tasks.add(initial);
}
}
/**
* Merge the services of all the child groups, with the current services.
* Keeping the order specified by the group.
*/
public void merge(Iterator<Grouping> iterator) throws AmbariException {
Map<String, List<OrderService>> skippedServices = new HashMap<>();
while (iterator.hasNext()) {
Grouping group = iterator.next();
boolean added = addGroupingServices(group.services, group.addAfterGroupEntry);
if (added) {
addSkippedServices(skippedServices, group.services);
} else {
// store these services until later
if (skippedServices.containsKey(group.addAfterGroupEntry)) {
List<OrderService> tmp = skippedServices.get(group.addAfterGroupEntry);
tmp.addAll(group.services);
} else {
skippedServices.put(group.addAfterGroupEntry, group.services);
}
}
}
}
/**
* Merge the services to add after a particular service name
*/
private boolean addGroupingServices(List<OrderService> servicesToAdd, String after) {
if (after == null) {
services.addAll(servicesToAdd);
return true;
}
else {
// Check the current services, if the "after" service is there then add these
for (int index = services.size() - 1; index >= 0; index--) {
OrderService service = services.get(index);
if (service.serviceName.equals(after)) {
services.addAll(index + 1, servicesToAdd);
return true;
}
}
}
return false;
}
/**
* Adds services which were previously skipped, if the service they are supposed
* to come after has been added.
*/
private void addSkippedServices(Map<String, List<OrderService>> skippedServices, List<OrderService> servicesJustAdded) {
for (OrderService service : servicesJustAdded) {
if (skippedServices.containsKey(service.serviceName)) {
List<OrderService> servicesToAdd = skippedServices.remove(service.serviceName);
addGroupingServices(servicesToAdd, service.serviceName);
addSkippedServices(skippedServices, servicesToAdd);
}
}
}
/**
* {@inheritDoc}
*/
@Override
public String toString() {
return Objects.toStringHelper(this).add("name", name).toString();
}
}