/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.brooklyn.entity.group;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkNotNull;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.atomic.AtomicInteger;
import javax.annotation.Nullable;
import org.apache.brooklyn.api.entity.Entity;
import org.apache.brooklyn.api.entity.EntitySpec;
import org.apache.brooklyn.api.entity.Group;
import org.apache.brooklyn.api.location.Location;
import org.apache.brooklyn.api.location.MachineProvisioningLocation;
import org.apache.brooklyn.api.location.NoMachinesAvailableException;
import org.apache.brooklyn.api.mgmt.Task;
import org.apache.brooklyn.api.policy.Policy;
import org.apache.brooklyn.api.sensor.AttributeSensor;
import org.apache.brooklyn.core.config.Sanitizer;
import org.apache.brooklyn.core.config.render.RendererHints;
import org.apache.brooklyn.core.effector.Effectors;
import org.apache.brooklyn.core.entity.Entities;
import org.apache.brooklyn.core.entity.EntityPredicates;
import org.apache.brooklyn.core.entity.factory.EntityFactory;
import org.apache.brooklyn.core.entity.factory.EntityFactoryForLocation;
import org.apache.brooklyn.core.entity.lifecycle.Lifecycle;
import org.apache.brooklyn.core.entity.lifecycle.ServiceStateLogic;
import org.apache.brooklyn.core.entity.lifecycle.ServiceStateLogic.ServiceProblemsLogic;
import org.apache.brooklyn.core.entity.trait.Resizable;
import org.apache.brooklyn.core.entity.trait.Startable;
import org.apache.brooklyn.core.entity.trait.StartableMethods;
import org.apache.brooklyn.core.location.Locations;
import org.apache.brooklyn.core.location.cloud.AvailabilityZoneExtension;
import org.apache.brooklyn.core.sensor.Sensors;
import org.apache.brooklyn.entity.stock.DelegateEntity;
import org.apache.brooklyn.feed.function.FunctionFeed;
import org.apache.brooklyn.feed.function.FunctionPollConfig;
import org.apache.brooklyn.util.collections.MutableList;
import org.apache.brooklyn.util.collections.MutableMap;
import org.apache.brooklyn.util.collections.QuorumCheck.QuorumChecks;
import org.apache.brooklyn.util.core.flags.TypeCoercions;
import org.apache.brooklyn.util.core.task.DynamicTasks;
import org.apache.brooklyn.util.core.task.TaskTags;
import org.apache.brooklyn.util.core.task.Tasks;
import org.apache.brooklyn.util.exceptions.Exceptions;
import org.apache.brooklyn.util.exceptions.ReferenceWithError;
import org.apache.brooklyn.util.guava.Maybe;
import org.apache.brooklyn.util.javalang.JavaClassNames;
import org.apache.brooklyn.util.javalang.Reflections;
import org.apache.brooklyn.util.text.StringPredicates;
import org.apache.brooklyn.util.text.Strings;
import org.apache.brooklyn.util.time.Duration;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.base.Function;
import com.google.common.base.Functions;
import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
import com.google.common.base.Predicate;
import com.google.common.base.Predicates;
import com.google.common.base.Supplier;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Iterables;
import com.google.common.collect.LinkedHashMultimap;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Multimap;
import com.google.common.collect.Sets;
import com.google.common.reflect.TypeToken;
/**
* A cluster of entities that can dynamically increase or decrease the number of entities.
*/
public class DynamicClusterImpl extends AbstractGroupImpl implements DynamicCluster {
@SuppressWarnings("serial")
private static final AttributeSensor<Supplier<Integer>> NEXT_CLUSTER_MEMBER_ID = Sensors.newSensor(new TypeToken<Supplier<Integer>>() {},
"next.cluster.member.id", "Returns the ID number of the next member to be added");
private volatile FunctionFeed clusterOneAndAllMembersUp;
// TODO better mechanism for arbitrary class name to instance type coercion
static {
TypeCoercions.registerAdapter(String.class, NodePlacementStrategy.class, new Function<String, NodePlacementStrategy>() {
@Override
public NodePlacementStrategy apply(final String input) {
ClassLoader classLoader = NodePlacementStrategy.class.getClassLoader();
Optional<NodePlacementStrategy> strategy = Reflections.<NodePlacementStrategy>invokeConstructorWithArgs(classLoader, input);
if (strategy.isPresent()) {
return strategy.get();
} else {
throw new IllegalStateException("Failed to create NodePlacementStrategy "+input);
}
}
});
TypeCoercions.registerAdapter(String.class, ZoneFailureDetector.class, new Function<String, ZoneFailureDetector>() {
@Override
public ZoneFailureDetector apply(final String input) {
ClassLoader classLoader = ZoneFailureDetector.class.getClassLoader();
Optional<ZoneFailureDetector> detector = Reflections.<ZoneFailureDetector>invokeConstructorWithArgs(classLoader, input);
if (detector.isPresent()) {
return detector.get();
} else {
throw new IllegalStateException("Failed to create ZoneFailureDetector "+input);
}
}
});
}
static {
RendererHints.register(FIRST, RendererHints.namedActionWithUrl("Open", DelegateEntity.EntityUrl.entityUrl()));
RendererHints.register(CLUSTER, RendererHints.namedActionWithUrl("Open", DelegateEntity.EntityUrl.entityUrl()));
}
private static final Logger LOG = LoggerFactory.getLogger(DynamicClusterImpl.class);
/**
* Mutex for synchronizing during re-size operations.
* Sub-classes should use with great caution, to not introduce deadlocks!
*/
protected final Object mutex = new Object[0];
private static final Function<Collection<Entity>, Entity> defaultRemovalStrategy = new Function<Collection<Entity>, Entity>() {
@Override public Entity apply(Collection<Entity> contenders) {
/*
* Choose the newest entity (largest cluster member ID or latest timestamp) that is stoppable.
* If none are stoppable, take the newest non-stoppable.
*
* Both cluster member ID and timestamp must be taken into consideration to account for legacy
* clusters that were created before the addition of the cluster member ID config value.
*/
int largestClusterMemberId = -1;
long newestTime = 0L;
Entity newest = null;
for (Entity contender : contenders) {
Integer contenderClusterMemberId = contender.config().get(CLUSTER_MEMBER_ID);
long contenderCreationTime = contender.getCreationTime();
boolean newer = (contenderClusterMemberId != null && contenderClusterMemberId > largestClusterMemberId) ||
contenderCreationTime > newestTime;
if ((contender instanceof Startable && newer) ||
(!(newest instanceof Startable) && ((contender instanceof Startable) || newer))) {
newest = contender;
if (contenderClusterMemberId != null) largestClusterMemberId = contenderClusterMemberId;
newestTime = contenderCreationTime;
}
}
return newest;
}
};
private static class NextClusterMemberIdSupplier implements Supplier<Integer> {
private AtomicInteger nextId = new AtomicInteger(0);
@Override
public Integer get() {
return nextId.getAndIncrement();
}
}
public DynamicClusterImpl() {
}
@Override
public void init() {
super.init();
initialiseMemberId();
connectAllMembersUp();
}
private void initialiseMemberId() {
synchronized (mutex) {
if (sensors().get(NEXT_CLUSTER_MEMBER_ID) == null) {
sensors().set(NEXT_CLUSTER_MEMBER_ID, new NextClusterMemberIdSupplier());
}
}
}
private void connectAllMembersUp() {
clusterOneAndAllMembersUp = FunctionFeed.builder()
.entity(this)
.period(Duration.FIVE_SECONDS)
.poll(new FunctionPollConfig<Boolean, Boolean>(CLUSTER_ONE_AND_ALL_MEMBERS_UP)
.onException(Functions.constant(Boolean.FALSE))
.callable(new ClusterOneAndAllMembersUpCallable(this)))
.build();
}
private static class ClusterOneAndAllMembersUpCallable implements Callable<Boolean> {
private final Group cluster;
public ClusterOneAndAllMembersUpCallable(Group cluster) {
this.cluster = cluster;
}
@Override
public Boolean call() throws Exception {
if (cluster.getMembers().isEmpty())
return false;
if (Lifecycle.RUNNING != cluster.sensors().get(SERVICE_STATE_ACTUAL))
return false;
for (Entity member : cluster.getMembers())
if (!Boolean.TRUE.equals(member.sensors().get(SERVICE_UP)))
return false;
return true;
}
}
@Override
protected void initEnrichers() {
if (config().getRaw(UP_QUORUM_CHECK).isAbsent() && getConfig(INITIAL_SIZE)==0) {
// if initial size is 0 then override up check to allow zero if empty
config().set(UP_QUORUM_CHECK, QuorumChecks.atLeastOneUnlessEmpty());
sensors().set(SERVICE_UP, true);
} else {
sensors().set(SERVICE_UP, false);
}
super.initEnrichers();
// override previous enricher so that only members are checked
ServiceStateLogic.newEnricherFromChildrenUp().checkMembersOnly().requireUpChildren(getConfig(UP_QUORUM_CHECK)).addTo(this);
}
@Override
public void setRemovalStrategy(Function<Collection<Entity>, Entity> val) {
config().set(REMOVAL_STRATEGY, checkNotNull(val, "removalStrategy"));
}
protected Function<Collection<Entity>, Entity> getRemovalStrategy() {
Function<Collection<Entity>, Entity> result = getConfig(REMOVAL_STRATEGY);
return (result != null) ? result : defaultRemovalStrategy;
}
@Override
public void setZonePlacementStrategy(NodePlacementStrategy val) {
config().set(ZONE_PLACEMENT_STRATEGY, checkNotNull(val, "zonePlacementStrategy"));
}
protected NodePlacementStrategy getZonePlacementStrategy() {
return checkNotNull(getConfig(ZONE_PLACEMENT_STRATEGY), "zonePlacementStrategy config");
}
@Override
public void setZoneFailureDetector(ZoneFailureDetector val) {
config().set(ZONE_FAILURE_DETECTOR, checkNotNull(val, "zoneFailureDetector"));
}
protected ZoneFailureDetector getZoneFailureDetector() {
return checkNotNull(getConfig(ZONE_FAILURE_DETECTOR), "zoneFailureDetector config");
}
protected EntitySpec<?> getFirstMemberSpec() {
return getConfig(FIRST_MEMBER_SPEC);
}
protected EntitySpec<?> getMemberSpec() {
return getConfig(MEMBER_SPEC);
}
/** @deprecated since 0.7.0; use {@link #getMemberSpec()} */
@Deprecated
protected EntityFactory<?> getFactory() {
return getConfig(FACTORY);
}
@Override
public void setMemberSpec(EntitySpec<?> memberSpec) {
setConfigEvenIfOwned(MEMBER_SPEC, memberSpec);
}
/** @deprecated since 0.7.0; use {@link #setMemberSpec(EntitySpec)} */
@Deprecated
@Override
public void setFactory(EntityFactory<?> factory) {
setConfigEvenIfOwned(FACTORY, factory);
}
private Location getLocation(boolean required) {
Collection<? extends Location> ll = Locations.getLocationsCheckingAncestors(getLocations(), this);
if (ll.isEmpty()) {
if (!required) return null;
throw new IllegalStateException("No location available for "+this);
}
if (ll.size()>1) {
throw new IllegalStateException("Ambiguous location for "+this+"; expected one but had "+ll);
}
return Iterables.getOnlyElement(ll);
}
protected boolean isAvailabilityZoneEnabled() {
return getConfig(ENABLE_AVAILABILITY_ZONES);
}
protected boolean isQuarantineEnabled() {
return getConfig(QUARANTINE_FAILED_ENTITIES);
}
protected QuarantineGroup getQuarantineGroup() {
return getAttribute(QUARANTINE_GROUP);
}
protected Predicate<? super Throwable> getQuarantineFilter() {
Predicate<? super Throwable> result = getConfig(QUARANTINE_FILTER);
if (result != null) {
return result;
} else {
return new Predicate<Throwable>() {
@Override public boolean apply(Throwable input) {
return Exceptions.getFirstThrowableOfType(input, NoMachinesAvailableException.class) == null;
}
};
}
}
protected int getInitialQuorumSize() {
int initialSize = getConfig(INITIAL_SIZE).intValue();
int initialQuorumSize = getConfig(INITIAL_QUORUM_SIZE).intValue();
if (initialQuorumSize < 0) initialQuorumSize = initialSize;
if (initialQuorumSize > initialSize) {
LOG.warn("On start of cluster {}, misconfigured initial quorum size {} greater than initial size{}; using {}", new Object[] {initialQuorumSize, initialSize, initialSize});
initialQuorumSize = initialSize;
}
return initialQuorumSize;
}
@Override
public void start(Collection<? extends Location> locsO) {
addLocations(locsO);
Location loc = getLocation(false);
EntitySpec<?> spec = getConfig(MEMBER_SPEC);
if (spec!=null) {
setDefaultDisplayName("Cluster of "+JavaClassNames.simpleClassName(spec.getType()) +
(loc!=null ? " ("+loc+")" : ""));
}
if (isAvailabilityZoneEnabled()) {
if (loc==null) throw new IllegalStateException("When using availability zones, a location must be specified on the cluster");
sensors().set(SUB_LOCATIONS, findSubLocations(loc));
}
ServiceStateLogic.setExpectedState(this, Lifecycle.STARTING);
ServiceProblemsLogic.clearProblemsIndicator(this, START);
try {
doStart();
DynamicTasks.waitForLast();
} catch (Exception e) {
ServiceProblemsLogic.updateProblemsIndicator(this, START, "start failed with error: "+e);
throw Exceptions.propagate(e);
} finally {
ServiceStateLogic.setExpectedState(this, Lifecycle.RUNNING);
}
}
protected void doStart() {
if (isQuarantineEnabled()) {
QuarantineGroup quarantineGroup = getAttribute(QUARANTINE_GROUP);
if (quarantineGroup==null || !Entities.isManaged(quarantineGroup)) {
quarantineGroup = addChild(EntitySpec.create(QuarantineGroup.class).displayName("quarantine"));
sensors().set(QUARANTINE_GROUP, quarantineGroup);
}
}
int initialSize = getConfig(INITIAL_SIZE).intValue();
int initialQuorumSize = getInitialQuorumSize();
Exception internalError = null;
try {
resize(initialSize);
} catch (Exception e) {
Exceptions.propagateIfFatal(e);
// Apart from logging, ignore problems here; we extract them below.
// But if it was this thread that threw the exception (rather than a sub-task), then need
// to record that failure here.
LOG.debug("Error resizing "+this+" to size "+initialSize+" (collecting and handling): "+e, e);
internalError = e;
}
Iterable<Task<?>> failed = Tasks.failed(Tasks.children(Tasks.current()));
boolean noFailed = Iterables.isEmpty(failed);
boolean severalFailed = Iterables.size(failed) > 1;
int currentSize = getCurrentSize().intValue();
if (currentSize < initialQuorumSize) {
String message;
if (currentSize == 0 && !noFailed) {
if (severalFailed)
message = "All nodes in cluster "+this+" failed";
else
message = "Node in cluster "+this+" failed";
} else {
message = "On start of cluster " + this + ", failed to get to initial size of " + initialSize
+ "; size is " + getCurrentSize()
+ (initialQuorumSize != initialSize ? " (initial quorum size is " + initialQuorumSize + ")" : "");
}
Throwable firstError = Tasks.getError(Maybe.next(failed.iterator()).orNull());
if (firstError==null && internalError!=null) {
// only use the internal error if there were no nested task failures
// (otherwise the internal error should be a wrapper around the nested failures)
firstError = internalError;
}
if (firstError!=null) {
if (severalFailed) {
message += "; first failure is: "+Exceptions.collapseText(firstError);
} else {
message += ": "+Exceptions.collapseText(firstError);
}
}
throw new IllegalStateException(message, firstError);
} else if (currentSize < initialSize) {
LOG.warn(
"On start of cluster {}, size {} reached initial minimum quorum size of {} but did not reach desired size {}; continuing",
new Object[] { this, currentSize, initialQuorumSize, initialSize });
}
for (Policy it : policies()) {
it.resume();
}
}
protected List<Location> findSubLocations(Location loc) {
if (!loc.hasExtension(AvailabilityZoneExtension.class)) {
throw new IllegalStateException("Availability zone extension not supported for location " + loc);
}
AvailabilityZoneExtension zoneExtension = loc.getExtension(AvailabilityZoneExtension.class);
Collection<String> zoneNames = getConfig(AVAILABILITY_ZONE_NAMES);
Integer numZones = getConfig(NUM_AVAILABILITY_ZONES);
List<Location> subLocations;
if (zoneNames == null || zoneNames.isEmpty()) {
if (numZones != null) {
subLocations = zoneExtension.getSubLocations(numZones);
checkArgument(numZones > 0, "numZones must be greater than zero: %s", numZones);
if (numZones > subLocations.size()) {
throw new IllegalStateException("Number of required zones (" + numZones + ") not satisfied in " + loc
+ "; only " + subLocations.size() + " available: " + subLocations);
}
} else {
subLocations = zoneExtension.getAllSubLocations();
}
} else {
// TODO check that these are valid region / availabilityZones?
subLocations = zoneExtension.getSubLocationsByName(StringPredicates.equalToAny(zoneNames), zoneNames.size());
if (zoneNames.size() > subLocations.size()) {
throw new IllegalStateException("Number of required zones (" + zoneNames.size() + " - " + zoneNames
+ ") not satisfied in " + loc + "; only " + subLocations.size() + " available: " + subLocations);
}
}
LOG.info("Returning {} sub-locations: {}", subLocations.size(), Iterables.toString(subLocations));
return subLocations;
}
@Override
public void stop() {
ServiceStateLogic.setExpectedState(this, Lifecycle.STOPPING);
try {
for (Policy it : policies()) { it.suspend(); }
// run shrink without mutex to make things stop even if starting,
int size = getCurrentSize();
if (size > 0) { shrink(-size); }
// run resize with mutex to prevent others from starting things
resize(0);
// also stop any remaining stoppable children -- eg those on fire
// (this ignores the quarantine node which is not stoppable)
StartableMethods.stop(this);
ServiceStateLogic.setExpectedState(this, Lifecycle.STOPPED);
} catch (Exception e) {
ServiceStateLogic.setExpectedState(this, Lifecycle.ON_FIRE);
throw Exceptions.propagate(e);
} finally {
if (clusterOneAndAllMembersUp != null) clusterOneAndAllMembersUp.stop();
}
}
@Override
public void restart() {
String mode = getConfig(RESTART_MODE);
if (mode==null) {
throw new UnsupportedOperationException("Restart not supported for this cluster: "+RESTART_MODE.getName()+" is not configured.");
}
if ("off".equalsIgnoreCase(mode)) {
throw new UnsupportedOperationException("Restart not supported for this cluster.");
}
if ("sequential".equalsIgnoreCase(mode)) {
ServiceStateLogic.setExpectedState(this, Lifecycle.STARTING);
DynamicTasks.queue(Effectors.invocationSequential(Startable.RESTART, null,
Iterables.filter(getChildren(), Predicates.and(Predicates.instanceOf(Startable.class), EntityPredicates.isManaged()))));
} else if ("parallel".equalsIgnoreCase(mode)) {
ServiceStateLogic.setExpectedState(this, Lifecycle.STARTING);
DynamicTasks.queue(Effectors.invocationParallel(Startable.RESTART, null,
Iterables.filter(getChildren(), Predicates.and(Predicates.instanceOf(Startable.class), EntityPredicates.isManaged()))));
} else {
throw new IllegalArgumentException("Unknown "+RESTART_MODE.getName()+" '"+mode+"'");
}
DynamicTasks.waitForLast();
ServiceStateLogic.setExpectedState(this, Lifecycle.RUNNING);
}
@Override
public Integer resize(Integer desiredSize) {
synchronized (mutex) {
int originalSize = getCurrentSize();
int delta = desiredSize - originalSize;
if (delta != 0) {
LOG.info("Resize {} from {} to {}", new Object[] {this, originalSize, desiredSize});
} else {
if (LOG.isDebugEnabled()) LOG.debug("Resize no-op {} from {} to {}", new Object[] {this, originalSize, desiredSize});
}
// If we managed to grow at all, then expect no exception.
// Otherwise, if failed because NoMachinesAvailable, then propagate as InsufficientCapacityException.
// This tells things like the AutoScalerPolicy to not keep retrying.
try {
resizeByDelta(delta);
} catch (Exception e) {
Exceptions.propagateIfFatal(e);
NoMachinesAvailableException nmae = Exceptions.getFirstThrowableOfType(e, NoMachinesAvailableException.class);
if (nmae != null) {
throw new Resizable.InsufficientCapacityException("Failed to resize", e);
} else {
throw Exceptions.propagate(e);
}
}
}
return getCurrentSize();
}
/**
* {@inheritDoc}
*
* <strong>Note</strong> for sub-classes; this method can be called while synchronized on {@link #mutex}.
*/
@Override
public String replaceMember(String memberId) {
Entity member = getEntityManager().getEntity(memberId);
LOG.info("In {}, replacing member {} ({})", new Object[] {this, memberId, member});
if (member == null) {
throw new NoSuchElementException("In "+this+", entity "+memberId+" cannot be resolved, so not replacing");
}
synchronized (mutex) {
if (!getMembers().contains(member)) {
throw new NoSuchElementException("In "+this+", entity "+member+" is not a member so not replacing");
}
Location memberLoc = null;
if (isAvailabilityZoneEnabled()) {
// this member's location could be a machine provisioned by a sub-location, or the actual sub-location
List<Location> subLocations = findSubLocations(getLocation(true));
Collection<Location> actualMemberLocs = member.getLocations();
boolean foundMatch = false;
for (Iterator<Location> iter = actualMemberLocs.iterator(); !foundMatch && iter.hasNext();) {
Location actualMemberLoc = iter.next();
Location contenderMemberLoc = actualMemberLoc;
do {
if (subLocations.contains(contenderMemberLoc)) {
memberLoc = contenderMemberLoc;
foundMatch = true;
LOG.debug("In {} replacing member {} ({}), inferred its sub-location is {}", new Object[] {this, memberId, member, memberLoc});
}
contenderMemberLoc = contenderMemberLoc.getParent();
} while (!foundMatch && contenderMemberLoc != null);
}
if (!foundMatch) {
if (actualMemberLocs.isEmpty()) {
memberLoc = subLocations.get(0);
LOG.warn("In {} replacing member {} ({}), has no locations; falling back to first availability zone: {}", new Object[] {this, memberId, member, memberLoc});
} else {
memberLoc = Iterables.tryFind(actualMemberLocs, Predicates.instanceOf(MachineProvisioningLocation.class)).or(Iterables.getFirst(actualMemberLocs, null));
LOG.warn("In {} replacing member {} ({}), could not find matching sub-location; falling back to its actual location: {}", new Object[] {this, memberId, member, memberLoc});
}
} else if (memberLoc == null) {
// impossible to get here, based on logic above!
throw new IllegalStateException("Unexpected condition! cluster="+this+"; member="+member+"; actualMemberLocs="+actualMemberLocs);
}
} else {
// Replacing member, so new member should be in the same location as that being replaced.
// Expect this to agree with `getMemberSpec().getLocations()` (if set). If not, then
// presumably there was a reason this specific member was started somewhere else!
memberLoc = getLocation(false);
}
Entity replacement = replaceMember(member, memberLoc, ImmutableMap.of());
return replacement.getId();
}
}
/**
* @throws StopFailedRuntimeException If stop failed, after successfully starting replacement
*/
protected Entity replaceMember(Entity member, @Nullable Location memberLoc, Map<?, ?> extraFlags) {
synchronized (mutex) {
ReferenceWithError<Optional<Entity>> added = addInSingleLocation(memberLoc, extraFlags);
if (!added.getWithoutError().isPresent()) {
String msg = String.format("In %s, failed to grow, to replace %s; not removing", this, member);
if (added.hasError())
throw new IllegalStateException(msg, added.getError());
throw new IllegalStateException(msg);
}
try {
stopAndRemoveNode(member);
} catch (Exception e) {
Exceptions.propagateIfFatal(e);
throw new StopFailedRuntimeException("replaceMember failed to stop and remove old member "+member.getId(), e);
}
return added.getWithError().get();
}
}
protected Multimap<Location, Entity> getMembersByLocation() {
Multimap<Location, Entity> result = LinkedHashMultimap.create();
for (Entity member : getMembers()) {
Collection<Location> memberLocs = member.getLocations();
Location memberLoc = Iterables.getFirst(memberLocs, null);
if (memberLoc != null) {
result.put(memberLoc, member);
}
}
return result;
}
protected List<Location> getNonFailedSubLocations() {
List<Location> result = Lists.newArrayList();
Set<Location> failed = Sets.newLinkedHashSet();
List<Location> subLocations = findSubLocations(getLocation(true));
Set<Location> oldFailedSubLocations = getAttribute(FAILED_SUB_LOCATIONS);
if (oldFailedSubLocations == null)
oldFailedSubLocations = ImmutableSet.<Location> of();
for (Location subLocation : subLocations) {
if (getZoneFailureDetector().hasFailed(subLocation)) {
failed.add(subLocation);
} else {
result.add(subLocation);
}
}
Set<Location> newlyFailed = Sets.difference(failed, oldFailedSubLocations);
Set<Location> newlyRecovered = Sets.difference(oldFailedSubLocations, failed);
sensors().set(FAILED_SUB_LOCATIONS, failed);
sensors().set(SUB_LOCATIONS, result);
if (newlyFailed.size() > 0) {
LOG.warn("Detected probably zone failures for {}: {}", this, newlyFailed);
}
if (newlyRecovered.size() > 0) {
LOG.warn("Detected probably zone recoveries for {}: {}", this, newlyRecovered);
}
return result;
}
/**
* {@inheritDoc}
*
* <strong>Note</strong> for sub-classes; this method can be called while synchronized on {@link #mutex}.
*/
@Override
public Collection<Entity> resizeByDelta(int delta) {
synchronized (mutex) {
if (delta > 0) {
return grow(delta);
} else if (delta < 0) {
return shrink(delta);
} else {
return ImmutableList.<Entity>of();
}
}
}
/** <strong>Note</strong> for sub-classes; this method can be called while synchronized on {@link #mutex}. */
protected Collection<Entity> grow(int delta) {
Preconditions.checkArgument(delta > 0, "Must call grow with positive delta.");
// choose locations to be deployed to
List<Location> chosenLocations;
List<Location> memberLocations = getMemberSpec() == null ? null : getMemberSpec().getLocations();
if (memberLocations != null && memberLocations.size() > 0) {
// The memberSpec overrides the location passed to cluster.start(); use
// the location defined on the member.
if (isAvailabilityZoneEnabled()) {
LOG.warn("Cluster {} has availability-zone enabled, but memberSpec overrides location with {}; using "
+ "memberSpec's location; availability-zone behaviour will not apply", this, memberLocations);
}
chosenLocations = Collections.nCopies(delta, memberLocations.get(0));
} else if (isAvailabilityZoneEnabled()) {
List<Location> subLocations = getNonFailedSubLocations();
Multimap<Location, Entity> membersByLocation = getMembersByLocation();
chosenLocations = getZonePlacementStrategy().locationsForAdditions(membersByLocation, subLocations, delta);
if (chosenLocations.size() != delta) {
throw new IllegalStateException("Node placement strategy chose " + Iterables.size(chosenLocations)
+ ", when expected delta " + delta + " in " + this);
}
} else {
chosenLocations = Collections.nCopies(delta, getLocation(false));
}
// create and start the entities.
// if any fail, then propagate the error.
ReferenceWithError<Collection<Entity>> result = addInEachLocation(chosenLocations, ImmutableMap.of());
return result.getWithError();
}
/** <strong>Note</strong> for sub-clases; this method can be called while synchronized on {@link #mutex}. */
@SuppressWarnings("unchecked")
protected Collection<Entity> shrink(int delta) {
Preconditions.checkArgument(delta < 0, "Must call shrink with negative delta.");
int size = getCurrentSize();
if (-delta > size) {
// some subclasses (esp in tests) use custom sizes without the members set always being accurate, so put a limit on the size
LOG.warn("Call to shrink "+this+" by "+delta+" when size is "+size+"; amending");
delta = -size;
}
if (delta==0) return ImmutableList.<Entity>of();
Collection<Entity> removedEntities = pickAndRemoveMembers(delta * -1);
// FIXME symmetry in order of added as child, managed, started, and added to group
Task<?> invoke = Entities.invokeEffector(this, (Iterable<Entity>)(Iterable<?>)Iterables.filter(removedEntities, Startable.class), Startable.STOP, Collections.<String,Object>emptyMap());
try {
invoke.get();
return removedEntities;
} catch (Exception e) {
throw Exceptions.propagate(e);
} finally {
for (Entity removedEntity : removedEntities) {
discardNode(removedEntity);
}
}
}
protected ReferenceWithError<Optional<Entity>> addInSingleLocation(@Nullable Location location, Map<?,?> flags) {
ReferenceWithError<Collection<Entity>> added = addInEachLocation(Arrays.asList(location), flags);
Optional<Entity> result = Iterables.isEmpty(added.getWithoutError()) ? Optional.<Entity>absent() : Optional.of(Iterables.getOnlyElement(added.get()));
if (!added.hasError()) {
return ReferenceWithError.newInstanceWithoutError( result );
} else {
if (added.masksErrorIfPresent()) {
return ReferenceWithError.newInstanceMaskingError( result, added.getError() );
} else {
return ReferenceWithError.newInstanceThrowingError( result, added.getError() );
}
}
}
protected ReferenceWithError<Collection<Entity>> addInEachLocation(Iterable<Location> locations, Map<?,?> flags) {
List<Entity> addedEntities = Lists.newArrayList();
Map<Entity, Location> addedEntityLocations = Maps.newLinkedHashMap();
Map<Entity, Task<?>> tasks = Maps.newLinkedHashMap();
for (Location loc : locations) {
Entity entity = addNode(loc, flags);
addedEntities.add(entity);
addedEntityLocations.put(entity, loc);
if (entity instanceof Startable) {
Map<String, ?> args = ImmutableMap.of("locations", ImmutableList.of(loc));
Task<Void> task = Effectors.invocation(entity, Startable.START, args).asTask();
tasks.put(entity, task);
}
}
Task<List<?>> parallel = Tasks.parallel("starting "+tasks.size()+" node"+Strings.s(tasks.size())+" (parallel)", tasks.values());
TaskTags.markInessential(parallel);
DynamicTasks.queueIfPossible(parallel).orSubmitAsync(this);
Map<Entity, Throwable> errors = waitForTasksOnEntityStart(tasks);
// if tracking, then report success/fail to the ZoneFailureDetector
if (isAvailabilityZoneEnabled()) {
for (Map.Entry<Entity, Location> entry : addedEntityLocations.entrySet()) {
Entity entity = entry.getKey();
Location loc = entry.getValue();
Throwable err = errors.get(entity);
if (err == null) {
getZoneFailureDetector().onStartupSuccess(loc, entity);
} else {
getZoneFailureDetector().onStartupFailure(loc, entity, err);
}
}
}
Collection<Entity> result = MutableList.<Entity> builder()
.addAll(addedEntities)
.removeAll(errors.keySet())
.build();
// quarantine/cleanup as necessary
if (!errors.isEmpty()) {
if (isQuarantineEnabled()) {
quarantineFailedNodes(errors);
} else {
cleanupFailedNodes(errors.keySet());
}
return ReferenceWithError.newInstanceMaskingError(result, Exceptions.create(errors.values()));
}
return ReferenceWithError.newInstanceWithoutError(result);
}
protected void quarantineFailedNodes(Map<Entity, Throwable> failedEntities) {
for (Map.Entry<Entity, Throwable> entry : failedEntities.entrySet()) {
Entity entity = entry.getKey();
Throwable cause = entry.getValue();
if (cause == null || getQuarantineFilter().apply(cause)) {
sensors().emit(ENTITY_QUARANTINED, entity);
getQuarantineGroup().addMember(entity);
removeMember(entity);
} else {
LOG.info("Cluster {} discarding failed node {}, rather than quarantining", this, entity);
discardNode(entity);
}
}
}
protected void cleanupFailedNodes(Collection<Entity> failedEntities) {
// TODO Could also call stop on them?
for (Entity entity : failedEntities) {
discardNode(entity);
}
}
protected Map<Entity, Throwable> waitForTasksOnEntityStart(Map<? extends Entity,? extends Task<?>> tasks) {
// TODO Could have CompoundException, rather than propagating first
Map<Entity, Throwable> errors = Maps.newLinkedHashMap();
for (Map.Entry<? extends Entity,? extends Task<?>> entry : tasks.entrySet()) {
Entity entity = entry.getKey();
Task<?> task = entry.getValue();
try {
task.get();
} catch (InterruptedException e) {
throw Exceptions.propagate(e);
} catch (Throwable t) {
Throwable interesting = Exceptions.getFirstInteresting(t);
LOG.error("Cluster "+this+" failed to start entity "+entity+" (removing): "+interesting, interesting);
LOG.debug("Trace for: Cluster "+this+" failed to start entity "+entity+" (removing): "+t, t);
// previously we unwrapped but now there is no need I think
errors.put(entity, t);
}
}
return errors;
}
@Override
public boolean removeChild(Entity child) {
boolean changed = super.removeChild(child);
if (changed) {
removeMember(child);
}
return changed;
}
protected Map<?,?> getCustomChildFlags() {
return getConfig(CUSTOM_CHILD_FLAGS);
}
@Override
public Entity addNode(@Nullable Location loc, Map<?, ?> extraFlags) {
// In case subclasses are foolish and do not call super.init() when overriding.
initialiseMemberId();
Map<?, ?> createFlags = MutableMap.builder()
.putAll(getCustomChildFlags())
.putAll(extraFlags)
.put(CLUSTER_MEMBER_ID, sensors().get(NEXT_CLUSTER_MEMBER_ID).get())
.build();
if (LOG.isDebugEnabled()) {
LOG.debug("Creating and adding a node to cluster {}({}) with properties {}", new Object[] { this, getId(), Sanitizer.sanitize(createFlags) });
}
// TODO should refactor to have a createNodeSpec; and spec should support initial sensor values
Entity entity = createNode(loc, createFlags);
entity.sensors().set(CLUSTER_MEMBER, true);
entity.sensors().set(CLUSTER, this);
// Continue to call manage(), because some uses of NodeFactory (in tests) still instantiate the
// entity via its constructor
Entities.manage(entity);
addMember(entity);
return entity;
}
protected Entity createNode(@Nullable Location loc, Map<?,?> flags) {
EntitySpec<?> memberSpec = null;
if (getMembers().isEmpty()) memberSpec = getFirstMemberSpec();
if (memberSpec == null) memberSpec = getMemberSpec();
if (memberSpec != null) {
EntitySpec<?> specConfigured = EntitySpec.create(memberSpec).configure(flags);
if (loc!=null) specConfigured.location(loc);
return addChild(specConfigured);
}
EntityFactory<?> factory = getFactory();
if (factory == null) {
throw new IllegalStateException("No member spec nor entity factory supplied for dynamic cluster "+this);
}
EntityFactory<?> factoryToUse = (factory instanceof EntityFactoryForLocation) ? ((EntityFactoryForLocation<?>) factory).newFactoryForLocation(loc) : factory;
Entity entity = factoryToUse.newEntity(flags, this);
if (entity==null) {
throw new IllegalStateException("EntityFactory factory routine returned null entity, in "+this);
}
if (entity.getParent()==null) entity.setParent(this);
return entity;
}
protected List<Entity> pickAndRemoveMembers(int delta) {
if (delta==0)
return Lists.newArrayList();
if (delta == 1 && !isAvailabilityZoneEnabled()) {
Maybe<Entity> member = tryPickAndRemoveMember();
return (member.isPresent()) ? ImmutableList.of(member.get()) : ImmutableList.<Entity>of();
}
// TODO inefficient impl
Preconditions.checkState(getMembers().size() > 0, "Attempt to remove a node (delta "+delta+") when members is empty, from cluster " + this);
if (LOG.isDebugEnabled()) LOG.debug("Removing a node from {}", this);
if (isAvailabilityZoneEnabled()) {
Multimap<Location, Entity> membersByLocation = getMembersByLocation();
List<Entity> entities = getZonePlacementStrategy().entitiesToRemove(membersByLocation, delta);
Preconditions.checkState(entities.size() == delta, "Incorrect num entity chosen for removal from %s (%s when expected %s)",
getId(), entities.size(), delta);
for (Entity entity : entities) {
removeMember(entity);
}
return entities;
} else {
List<Entity> entities = Lists.newArrayList();
for (int i = 0; i < delta; i++) {
// don't assume we have enough members; e.g. if shrinking to zero and someone else concurrently stops a member,
// then just return what we were able to remove.
Maybe<Entity> member = tryPickAndRemoveMember();
if (member.isPresent()) entities.add(member.get());
}
return entities;
}
}
private Maybe<Entity> tryPickAndRemoveMember() {
assert !isAvailabilityZoneEnabled() : "should instead call pickAndRemoveMembers(int) if using availability zones";
// TODO inefficient impl
Collection<Entity> members = getMembers();
if (members.isEmpty()) return Maybe.absent();
if (LOG.isDebugEnabled()) LOG.debug("Removing a node from {}", this);
Entity entity = getRemovalStrategy().apply(members);
Preconditions.checkNotNull(entity, "No entity chosen for removal from "+getId());
removeMember(entity);
return Maybe.of(entity);
}
protected void discardNode(Entity entity) {
removeMember(entity);
try {
Entities.unmanage(entity);
} catch (IllegalStateException e) {
//probably already unmanaged
LOG.debug("Exception during removing member of cluster " + this + ", unmanaging node " + entity + ". The node is probably already unmanaged.", e);
}
}
protected void stopAndRemoveNode(Entity member) {
removeMember(member);
try {
if (member instanceof Startable) {
Task<?> task = member.invoke(Startable.STOP, Collections.<String,Object>emptyMap());
task.getUnchecked();
}
} finally {
Entities.unmanage(member);
}
}
}