/*
* JBoss, Home of Professional Open Source
* Copyright 2011 Red Hat Inc. and/or its affiliates and other contributors
* as indicated by the @author tags. All rights reserved.
* See the copyright.txt in the distribution for a
* full listing of individual contributors.
*
* This copyrighted material is made available to anyone wishing to use,
* modify, copy, or redistribute it subject to the terms and conditions
* of the GNU Lesser General Public License, v. 2.1.
* This program is distributed in the hope that it will be useful, but WITHOUT A
* WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
* PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details.
* You should have received a copy of the GNU Lesser General Public License,
* v.2.1 along with this distribution; if not, write to the Free Software
* Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA.
*/
package org.infinispan.statetransfer;
import org.infinispan.cacheviews.CacheView;
import org.infinispan.cacheviews.CacheViewListener;
import org.infinispan.cacheviews.CacheViewsManager;
import org.infinispan.commands.CommandsFactory;
import org.infinispan.commands.control.StateTransferControlCommand;
import org.infinispan.commands.remote.ConfigurationStateCommand;
import org.infinispan.commands.write.PutKeyValueCommand;
import org.infinispan.config.Configuration;
import org.infinispan.container.DataContainer;
import org.infinispan.container.entries.CacheEntry;
import org.infinispan.container.entries.InternalCacheEntry;
import org.infinispan.container.versioning.VersionGenerator;
import org.infinispan.context.InvocationContext;
import org.infinispan.context.InvocationContextContainer;
import org.infinispan.distribution.DistributionManager;
import org.infinispan.distribution.ch.ConsistentHash;
import org.infinispan.factories.annotations.Inject;
import org.infinispan.factories.annotations.Start;
import org.infinispan.factories.annotations.Stop;
import org.infinispan.interceptors.InterceptorChain;
import org.infinispan.loaders.CacheLoaderManager;
import org.infinispan.loaders.CacheStore;
import org.infinispan.notifications.cachelistener.CacheNotifier;
import org.infinispan.reconfigurableprotocol.ProtocolTable;
import org.infinispan.reconfigurableprotocol.manager.ReconfigurableReplicationManager;
import org.infinispan.remoting.responses.Response;
import org.infinispan.remoting.responses.SuccessfulResponse;
import org.infinispan.remoting.rpc.ResponseMode;
import org.infinispan.remoting.rpc.RpcManager;
import org.infinispan.remoting.transport.Address;
import org.infinispan.remoting.transport.Transport;
import org.infinispan.transaction.LockingMode;
import org.infinispan.transaction.RemoteTransaction;
import org.infinispan.transaction.TransactionTable;
import org.infinispan.transaction.totalorder.TotalOrderManager;
import org.infinispan.util.concurrent.NotifyingNotifiableFuture;
import org.infinispan.util.concurrent.ReclosableLatch;
import org.infinispan.util.concurrent.locks.containers.LockContainer;
import org.infinispan.util.logging.Log;
import org.infinispan.util.logging.LogFactory;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
import static org.infinispan.context.Flag.*;
/**
* State transfer manager.
* Base class for the distributed and replicated implementations.
*/
public abstract class BaseStateTransferManagerImpl implements StateTransferManager, CacheViewListener {
private static final Log log = LogFactory.getLog(BaseStateTransferManagerImpl.class);
private static final boolean trace = log.isTraceEnabled();
// Injected components
protected CacheLoaderManager cacheLoaderManager;
protected Configuration configuration;
protected RpcManager rpcManager;
protected CommandsFactory cf;
protected DataContainer dataContainer;
protected InterceptorChain interceptorChain;
protected InvocationContextContainer icc;
protected CacheNotifier cacheNotifier;
private CacheViewsManager cacheViewsManager;
protected StateTransferLock stateTransferLock;
protected volatile ConsistentHash chOld;
private volatile CacheView oldView;
protected volatile ConsistentHash chNew;
private volatile CacheView newView;
// closed before the component has been started, open afterwards
private final CountDownLatch joinStartedLatch = new CountDownLatch(1);
// closed before the initial state transfer has completed, open afterwards
private final CountDownLatch joinCompletedLatch = new CountDownLatch(1);
// closed during state transfer, open the rest of the time
private final ReclosableLatch stateTransferInProgressLatch = new ReclosableLatch(false);
private volatile BaseStateTransferTask stateTransferTask;
private CommandBuilder commandBuilder;
protected TransactionTable transactionTable;
private LockContainer<?> lockContainer;
private VersionGenerator versionGenerator;
protected TotalOrderManager totalOrderManager;
private ProtocolTable protocolTable;
private ReconfigurableReplicationManager manager;
private DistributionManager distributionManager;
public BaseStateTransferManagerImpl() {
}
@Inject
public void init(Configuration configuration, RpcManager rpcManager, CommandsFactory cf,
DataContainer dataContainer, InterceptorChain interceptorChain, InvocationContextContainer icc,
CacheLoaderManager cacheLoaderManager, CacheNotifier cacheNotifier, StateTransferLock stateTransferLock,
CacheViewsManager cacheViewsManager, TransactionTable transactionTable, LockContainer<?> lockContainer,
VersionGenerator versionGenerator, TotalOrderManager totalOrderManager, ProtocolTable protocolTable,
ReconfigurableReplicationManager manager, DistributionManager distributionManager) {
this.cacheLoaderManager = cacheLoaderManager;
this.configuration = configuration;
this.rpcManager = rpcManager;
this.cf = cf;
this.stateTransferLock = stateTransferLock;
this.dataContainer = dataContainer;
this.interceptorChain = interceptorChain;
this.icc = icc;
this.cacheNotifier = cacheNotifier;
this.cacheViewsManager = cacheViewsManager;
this.transactionTable = transactionTable;
this.lockContainer = lockContainer;
this.versionGenerator = versionGenerator;
this.totalOrderManager = totalOrderManager;
this.protocolTable = protocolTable;
this.manager = manager;
this.distributionManager = distributionManager;
}
// needs to be AFTER the DistributionManager and *after* the cache loader manager (if any) inits and preloads
@Start(priority = 60)
private void start() throws Exception {
if (configuration.isTransactionalCache() &&
configuration.isEnableVersioning() &&
configuration.isWriteSkewCheck() &&
configuration.getTransactionLockingMode() == LockingMode.OPTIMISTIC &&
configuration.getCacheMode().isClustered()) {
// We need to use a special form of PutKeyValueCommand that can apply versions too.
commandBuilder = new CommandBuilder() {
@Override
public PutKeyValueCommand buildPut(InvocationContext ctx, CacheEntry e) {
return cf.buildVersionedPutKeyValueCommand(e.getKey(), e.getValue(), e.getLifespan(), e.getMaxIdle(), e.getVersion(), ctx.getFlags());
}
};
} else {
commandBuilder = new CommandBuilder() {
@Override
public PutKeyValueCommand buildPut(InvocationContext ctx, CacheEntry e) {
return cf.buildPutKeyValueCommand(e.getKey(), e.getValue(), e.getLifespan(), e.getMaxIdle(), ctx.getFlags());
}
};
}
if (trace) log.tracef("Starting state transfer manager on " + getAddress());
setConfigurationChanges();
// set up the old CH, but it shouldn't be used until we get the prepare call
cacheViewsManager.join(configuration.getName(), this);
}
private void setConfigurationChanges() {
if (log.isDebugEnabled()) {
log.debug("Setting configuration tunned parameters");
}
ConfigurationState configurationState = getClusterConfigurationState();
if (configurationState == null) {
if (log.isDebugEnabled()) {
log.debug("No State found. Using original configuration parameters");
}
} else {
if (log.isDebugEnabled()) {
log.debugf("Configuration State found: %s", configurationState);
}
manager.initialProtocol(configurationState.getProtocolName(), configurationState.getEpoch());
if (distributionManager != null) {
distributionManager.setReplicationDegree(configurationState.getNumberOfOwners());
}
}
}
private ConfigurationState getClusterConfigurationState() {
Transport transport = rpcManager.getTransport();
ConfigurationStateCommand configurationStateCommand = new ConfigurationStateCommand(configuration.getName());
Map<Address, Response> responseMap;
try {
responseMap = transport.invokeRemotely(null, configurationStateCommand,
ResponseMode.SYNCHRONOUS_IGNORE_LEAVERS,
configuration.getSyncReplTimeout(), true, null, false, false);
if (log.isDebugEnabled()) {
log.debugf("Configuration State responses are %s", responseMap);
}
for (Response response : responseMap.values()) {
if (response.isSuccessful() && response.isValid()) {
ConfigurationState cfgState = (ConfigurationState) ((SuccessfulResponse) response).getResponseValue();
if (cfgState == null) {
continue;
}
return cfgState;
}
}
} catch (Exception e) {
if (log.isDebugEnabled()) {
log.debug("Exception while trying to get the Configuration State", e);
}
}
if (log.isDebugEnabled()) {
log.debug("No Configuration State found in cluster");
}
return null;
}
protected abstract ConsistentHash createConsistentHash(List<Address> members, int replicationDegree);
// To avoid blocking other components' start process, wait last, if necessary, for join to complete.
@Override
@Start(priority = 1000)
public void waitForJoinToComplete() throws InterruptedException {
joinCompletedLatch.await(getTimeout(), TimeUnit.MILLISECONDS);
}
@Stop(priority = 20)
public void stop() {
chOld = null;
chNew = null;
// cancel any pending state transfer before leaving
BaseStateTransferTask tempTask = stateTransferTask;
if (tempTask != null) {
tempTask.cancelStateTransfer(true);
stateTransferTask = null;
}
cacheViewsManager.leave(configuration.getName());
joinStartedLatch.countDown();
joinCompletedLatch.countDown();
stateTransferInProgressLatch.open();
}
protected Address getAddress() {
return rpcManager.getAddress();
}
@Override
public boolean hasJoinStarted() {
return isLatchOpen(joinStartedLatch);
}
@Override
public void waitForJoinToStart() throws InterruptedException {
joinStartedLatch.await(getTimeout(), TimeUnit.MILLISECONDS);
}
@Override
public boolean isJoinComplete() {
return isLatchOpen(joinCompletedLatch);
}
@Override
public boolean isStateTransferInProgress() {
return !isLatchOpen(stateTransferInProgressLatch);
}
public void waitForStateTransferToStart(int viewId) throws InterruptedException {
// TODO Add another latch for this, or maybe use a lock with condition variables instead
while ((newView == null || newView.getViewId() < viewId)
&& (oldView == null || oldView.getViewId() < viewId)) {
Thread.sleep(10);
}
}
@Override
public void waitForStateTransferToComplete() throws InterruptedException {
stateTransferInProgressLatch.await(getTimeout(), TimeUnit.MILLISECONDS);
}
private boolean isLatchOpen(CountDownLatch latch) {
return latch.getCount() == 0;
}
private boolean isLatchOpen(ReclosableLatch latch) {
return latch.isOpened();
}
@Override
public void applyState(Collection<InternalCacheEntry> state,
Address sender, int viewId) throws InterruptedException {
waitForStateTransferToStart(viewId);
if (newView == oldView) {
log.remoteStateRejected(sender, viewId, oldView.getViewId());
return;
}
if (viewId != newView.getViewId()) {
log.debugf("Rejecting state pushed by node %s for rehash %d (last view id we know is %d)", sender, viewId, newView.getViewId());
return;
}
if (state != null) {
log.debugf("Applying new state from %s: received %d keys", sender, state.size());
if (trace) log.tracef("Received keys: %s", keys(state));
for (InternalCacheEntry e : state) {
InvocationContext ctx = icc.createInvocationContext(false, 1);
// locking not necessary as during rehashing we block all transactions
ctx.setFlags(CACHE_MODE_LOCAL, SKIP_CACHE_LOAD, SKIP_REMOTE_LOOKUP, SKIP_SHARED_CACHE_STORE, SKIP_LOCKING,
SKIP_OWNERSHIP_CHECK);
try {
PutKeyValueCommand put = commandBuilder.buildPut(ctx, e);
interceptorChain.invoke(ctx, put);
} catch (Exception ee) {
log.problemApplyingStateForKey(ee.getMessage(), e.getKey());
}
}
if(trace) log.tracef("After applying state data container has %d keys", dataContainer.size(null));
}
}
@Override
public void applyLocks(Collection<LockInfo> lockInfo, Address sender, int viewId) throws InterruptedException {
if (lockInfo != null) {
log.debugf("Integrating %d locks from %s", lockInfo.size(), sender);
for (LockInfo lock : lockInfo) {
RemoteTransaction remoteTx = transactionTable.getRemoteTransaction(lock.getGlobalTransaction());
if (remoteTx == null) {
remoteTx = transactionTable.createRemoteTransaction(lock.getGlobalTransaction(), null);
remoteTx.setMissingModifications(true);
}
//TODO ==> HACK! it is possible to deliver the commit/rollback before this is invoked. avoid lock acquisition
//TODO to avoid live locks.
//TODO ==> WARNING: this can leave the data inconsistent (by Pedro)
//Object result = lockContainer.acquireLock(remoteTx.getGlobalTransaction(), lock.getKey(), 0, TimeUnit.SECONDS);
//if (result == null) {
// throw new IllegalStateException("Could not acquire lock for key " + lock.getKey());
//}
}
}
}
private Collection<Object> keys(Collection<InternalCacheEntry> state) {
Collection<Object> result = new ArrayList<Object>(state.size());
for (InternalCacheEntry e : state) {
result.add(e.getKey());
}
return result;
}
// package-protected methods used by StateTransferTask
/**
* @return <code>true</code> if the state transfer started successfully, <code>false</code> otherwise
*/
public boolean startStateTransfer(int viewId, Collection<Address> members, boolean initialView) throws TimeoutException, InterruptedException, StateTransferCancelledException {
if (newView == null || viewId != newView.getViewId()) {
log.debugf("Cannot start state transfer for view %d, we should be starting state transfer for view %s", viewId, newView);
return false;
}
stateTransferInProgressLatch.close();
return true;
}
public abstract CacheStore getCacheStoreForStateTransfer();
public void pushStateToNode(NotifyingNotifiableFuture<Object> stateTransferFuture, int viewId, Collection<Address> targets,
Collection<InternalCacheEntry> state, Collection<LockInfo> lockInfo) throws StateTransferCancelledException {
StateTransferControlCommand.Type type;
if (state != null) {
log.debugf("Pushing to nodes %s %d keys", targets, state.size());
log.tracef("Pushing to nodes %s keys: %s", targets, keys(state));
type = StateTransferControlCommand.Type.APPLY_STATE;
} else if (lockInfo != null) {
log.debugf("Migrating %d locks to node(s) %s", lockInfo.size(), targets);
type = StateTransferControlCommand.Type.APPLY_LOCKS;
} else {
throw new IllegalStateException("Cannot have both locks and state set to null.");
}
final StateTransferControlCommand cmd = cf.buildStateTransferCommand(type, getAddress(), viewId, state, lockInfo);
rpcManager.invokeRemotelyInFuture(targets, cmd, usePriorityQueue(), stateTransferFuture, getTimeout());
}
public boolean isLastViewId(int viewId) {
return viewId == newView.getViewId();
}
@Override
public void prepareView(CacheView pendingView, CacheView committedView, List<CacheView> viewHistory, int replicationDegree) throws Exception {
if (versionGenerator != null) {
versionGenerator.updateViewHistory(viewHistory);
}
log.tracef("Received new cache view: %s %s", configuration.getName(), pendingView);
joinStartedLatch.countDown();
// if this is the first view we're seeing, initialize the oldView as well
if (oldView == null) {
oldView = committedView;
}
newView = pendingView;
chNew = createConsistentHash(pendingView.getMembers(), replicationDegree);
stateTransferTask = createStateTransferTask(pendingView.getViewId(), pendingView.getMembers(), chOld == null, replicationDegree);
stateTransferTask.performStateTransfer();
}
@Override
public void commitView(int viewId) {
BaseStateTransferTask tempTask = stateTransferTask;
if (tempTask == null) {
if (viewId == oldView.getViewId()) {
log.tracef("Ignoring commit for cache view %d as we have already committed it", viewId);
return;
} else {
throw new IllegalArgumentException(String.format("Cannot commit view %d, we are at view %d",
viewId, oldView.getViewId()));
}
}
tempTask.commitStateTransfer();
stateTransferTask = null;
if (versionGenerator != null) {
versionGenerator.addCacheView(newView);
}
// we can now use the new CH as the baseline for the next rehash
oldView = newView;
chOld = chNew;
}
@Override
public void rollbackView(int newViewId, int committedViewId) {
BaseStateTransferTask tempTask = stateTransferTask;
if (tempTask == null) {
if (committedViewId == oldView.getViewId()) {
log.tracef("Ignoring rollback for cache view %d as we don't have a state transfer in progress",
committedViewId);
return;
} else {
throw new IllegalArgumentException(String.format("Cannot rollback to view %d, we are at view %d",
committedViewId, oldView.getViewId()));
}
}
tempTask.cancelStateTransfer(true);
stateTransferTask = null;
newView = new CacheView(newViewId, oldView.getMembers());
oldView = newView;
chNew = chOld;
stateTransferInProgressLatch.open();
joinCompletedLatch.countDown();
}
@Override
public void preInstallView() {
stateTransferLock.blockNewTransactionsAsync();
}
@Override
public void postInstallView(int viewId) {
try {
stateTransferLock.unblockNewTransactions(viewId);
} catch (Exception e) {
log.errorUnblockingTransactions(e);
}
stateTransferInProgressLatch.open();
// getCache() will only return after joining has completed, so we need that to be last
joinCompletedLatch.countDown();
}
protected abstract BaseStateTransferTask createStateTransferTask(int viewId, List<Address> members, boolean initialView, int replicationDegree);
private static interface CommandBuilder {
PutKeyValueCommand buildPut(InvocationContext ctx, CacheEntry e);
}
protected abstract long getTimeout();
protected boolean usePriorityQueue() {
return isTotalOrder();
}
protected final boolean isTotalOrder() {
return manager.isTotalOrderBasedProtocol(protocolTable.getThreadProtocolId());
}
}