package io.eguan.dtx; /* * #%L * Project eguan * %% * Copyright (C) 2012 - 2017 Oodrive * %% * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. * #L% */ import static io.eguan.dtx.DtxDummyRmFactory.DEFAULT_PAYLOAD; import static io.eguan.dtx.DtxNodeState.INITIALIZED; import static io.eguan.dtx.DtxNodeState.STARTED; import static io.eguan.dtx.DtxResourceManagerState.POST_SYNC_PROCESSING; import static io.eguan.dtx.DtxResourceManagerState.SYNCHRONIZING; import static io.eguan.dtx.DtxResourceManagerState.UP_TO_DATE; import static io.eguan.dtx.DtxTaskStatus.COMMITTED; import static io.eguan.dtx.DtxTaskStatus.ROLLED_BACK; import static io.eguan.dtx.DtxTaskStatus.UNKNOWN; import static io.eguan.dtx.DtxTestHelper.checkJournalSync; import static io.eguan.dtx.DtxTestHelper.prepareExistingJournals; import static javax.transaction.xa.XAException.XAER_RMERR; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.mockito.Matchers.any; import static org.mockito.Mockito.atLeast; import static org.mockito.Mockito.verify; import io.eguan.dtx.DtxManager; import io.eguan.dtx.DtxResourceManager; import io.eguan.dtx.DtxResourceManagerContext; import io.eguan.dtx.DtxTaskStatus; import io.eguan.dtx.TransactionManager; import io.eguan.dtx.DtxDummyRmFactory.DtxResourceManagerBuilder; import io.eguan.dtx.DtxDummyRmFactory.ToggleAnswer; import io.eguan.dtx.DtxEventListeners.SeparateStateCountListener; import io.eguan.dtx.DtxEventListeners.StateCountListener; import java.io.IOException; import java.nio.file.Path; import java.util.ArrayList; import java.util.Collections; import java.util.Map; import java.util.Set; import java.util.UUID; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; import javax.transaction.xa.XAException; import org.junit.Test; import org.mockito.invocation.InvocationOnMock; import org.mockito.stubbing.Answer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.collect.HashBasedTable; import com.google.common.collect.HashMultimap; import com.google.common.collect.Table; import com.google.common.collect.TreeBasedTable; /** * Tests covering the discovery phase between nodes upon joining the cluster. * * @author oodrive * @author pwehrle * */ public final class TestDtxManagerSynchronizationL extends AbstractCommonClusterTest { private static final Logger LOGGER = LoggerFactory.getLogger(TestDtxManagerSynchronizationL.class); private static final int NB_OF_TEST_TX = 10; private static final int SYNC_WAIT_TIME_S = 20; private static final int TX_WAIT_TIME_MS = 100; private static final int PRE_VERIFICATION_LATENCY_MS = 500; private static final int POST_SYNC_DURATION_MS = 10000; private static final int NB_OF_SYNC_ATTEMPTS = 5; /** * Tests successful synchronization of nodes upon startup with one pre-registered {@link DtxResourceManager}s. * * @throws XAException * if mock setup fails, not part of this test * @throws IOException * if writing transactions to journals fails, not part of this test * @throws IllegalStateException * if setting up existing journals fails, not part of this test * @throws InterruptedException * if interrupted while waiting for completion, not part of this test */ @Test public final void testSyncOneResMgrOnInit() throws XAException, IllegalStateException, IOException, InterruptedException { LOGGER.info("Executing"); final Set<DtxManager> dtxMgrs = DTX_MGR_JOURNAL_MAP.keySet(); final UUID resUuid = UUID.randomUUID(); final TreeBasedTable<Long, UUID, DtxManager> testTable = TreeBasedTable.create(); long targetTxId = DtxTestHelper.nextTxId(); for (final DtxManager currDtxMgr : dtxMgrs) { currDtxMgr.stop(); targetTxId += NB_OF_TEST_TX; testTable.put(Long.valueOf(targetTxId), resUuid, currDtxMgr); } final Table<DtxResourceManager, Long, Path> resMgrMap = prepareExistingJournals(testTable, DTX_MGR_JOURNAL_MAP, SETUP_ROT_MGR); final HashBasedTable<UUID, DtxManager, CountDownLatch> upToDateTable = HashBasedTable.create(); for (final Long currIndex : testTable.rowKeySet()) { upToDateTable.put(resUuid, testTable.get(currIndex, resUuid), new CountDownLatch(1)); } final SeparateStateCountListener upToDateListener = new SeparateStateCountListener(upToDateTable, UP_TO_DATE); // registers resource managers for (final DtxResourceManager currResMgr : resMgrMap.rowKeySet()) { final Map<Long, Path> currRow = resMgrMap.row(currResMgr); for (final Long currIndex : currRow.keySet()) { final UUID currResId = currResMgr.getId(); final DtxManager currDtxMgr = testTable.get(currIndex, currResId); currDtxMgr.registerDtxEventListener(upToDateListener); currDtxMgr.registerResourceManager(currResMgr); } } // gets the set of DtxManagers ordered by increasing last tx ID final Set<DtxManager> ordDtxMgrSet = upToDateTable.row(resUuid).keySet(); // starts nodes for (final DtxManager currDtxMgr : ordDtxMgrSet) { if (STARTED == currDtxMgr.getStatus()) { continue; } LOGGER.debug("Starting; node=" + currDtxMgr.getNodeId()); currDtxMgr.start(); LOGGER.debug("Started; node=" + currDtxMgr.getNodeId() + ", last txId=" + currDtxMgr.getLastCompleteTxIdForResMgr(resUuid)); } // waits for all nodes to be up to date for (final DtxManager currDtxMgr : ordDtxMgrSet) { assertTrue(upToDateTable.get(resUuid, currDtxMgr).await(SYNC_WAIT_TIME_S, TimeUnit.SECONDS)); } for (final DtxManager currDtxMgr : dtxMgrs) { currDtxMgr.unregisterDtxEventListener(upToDateListener); currDtxMgr.unregisterResourceManager(resUuid); } upToDateListener.checkForAssertErrors(LOGGER); } /** * Tests successful synchronization of nodes upon registering on a started cluster. * * @throws XAException * if mock setup fails, not part of this test * @throws IOException * if writing transactions to journals fails, not part of this test * @throws IllegalStateException * if setting up existing journals fails, not part of this test * @throws InterruptedException * if interrupted while waiting for completion, not part of this test */ @Test public final void testSyncOneResMgrOnRegister() throws XAException, IllegalStateException, IOException, InterruptedException { LOGGER.info("Executing"); final UUID resUuid = UUID.randomUUID(); final TreeBasedTable<Long, UUID, DtxManager> testTable = TreeBasedTable.create(); long targetTxId = DtxTestHelper.nextTxId(); final long firstTxId = targetTxId; final Set<DtxManager> dtxMgrs = DTX_MGR_JOURNAL_MAP.keySet(); for (final DtxManager currDtxMgr : dtxMgrs) { targetTxId += NB_OF_TEST_TX; testTable.put(Long.valueOf(targetTxId), resUuid, currDtxMgr); } final Table<DtxResourceManager, Long, Path> resMgrMap = prepareExistingJournals(testTable, DTX_MGR_JOURNAL_MAP, SETUP_ROT_MGR); final HashBasedTable<UUID, DtxManager, CountDownLatch> upToDateTable = HashBasedTable.create(); for (final Long currIndex : testTable.rowKeySet()) { upToDateTable.put(resUuid, testTable.get(currIndex, resUuid), new CountDownLatch(1)); } final SeparateStateCountListener upToDateListener = new SeparateStateCountListener(upToDateTable, UP_TO_DATE); final ArrayList<TransactionManager> txMgrList = new ArrayList<TransactionManager>(); // gets the set of DtxManagers ordered by increasing last tx ID final Set<DtxManager> ordDtxMgrSet = upToDateTable.row(resUuid).keySet(); // initializes and starts all nodes for (final DtxManager currDtxMgr : ordDtxMgrSet) { currDtxMgr.init(); LOGGER.debug("Initialized; node=" + currDtxMgr.getNodeId()); currDtxMgr.start(); LOGGER.debug("Started; node=" + currDtxMgr.getNodeId() + ", last txId=" + currDtxMgr.getLastCompleteTxIdForResMgr(resUuid)); txMgrList.add(currDtxMgr.getTxManager()); } for (final DtxResourceManager currResMgr : resMgrMap.rowKeySet()) { final Map<Long, Path> currRow = resMgrMap.row(currResMgr); for (final Long currIndex : currRow.keySet()) { final UUID currResId = currResMgr.getId(); final DtxManager currDtxMgr = testTable.get(currIndex, currResId); currDtxMgr.registerDtxEventListener(upToDateListener); currDtxMgr.registerResourceManager(currResMgr); } } // waits for all nodes to be up to date for (final DtxManager currDtxMgr : ordDtxMgrSet) { assertTrue(upToDateTable.get(resUuid, currDtxMgr).await(SYNC_WAIT_TIME_S, TimeUnit.SECONDS)); } final int nbOfTx = Long.valueOf(targetTxId - firstTxId).intValue(); checkJournalSync(resUuid, txMgrList, nbOfTx); // verify a minimum of transactions actually got replayed for (final Long currIndex : testTable.rowKeySet()) { final DtxManager currDtxMgr = testTable.get(currIndex, resUuid); final int expectedDiff = Long.valueOf(targetTxId - currIndex.longValue()).intValue(); verify(currDtxMgr.getRegisteredResourceManager(resUuid), atLeast(expectedDiff)).prepare( any(DtxResourceManagerContext.class)); } for (final DtxManager currDtxMgr : dtxMgrs) { currDtxMgr.unregisterDtxEventListener(upToDateListener); currDtxMgr.unregisterResourceManager(resUuid); } upToDateListener.checkForAssertErrors(LOGGER); } /** * Tests successful synchronization of nodes upon registering on a started cluster. * * @throws XAException * if mock setup fails, not part of this test * @throws IOException * if writing transactions to journals fails, not part of this test * @throws IllegalStateException * if setting up existing journals fails, not part of this test * @throws InterruptedException * if interrupted while waiting for completion, not part of this test */ @Test public final void testSyncMultipleResMgrOnRegister() throws XAException, IllegalStateException, IOException, InterruptedException { LOGGER.info("Executing"); final UUID firstResUuid = UUID.randomUUID(); final UUID secondResUuid = UUID.randomUUID(); final TreeBasedTable<Long, UUID, DtxManager> testTable = TreeBasedTable.create(); long targetTxId = DtxTestHelper.nextTxId(); final Set<DtxManager> dtxMgrs = DTX_MGR_JOURNAL_MAP.keySet(); final HashBasedTable<UUID, DtxManager, CountDownLatch> upToDateTable = HashBasedTable.create(); for (final DtxManager currDtxMgr : dtxMgrs) { targetTxId += NB_OF_TEST_TX; testTable.put(Long.valueOf(targetTxId), firstResUuid, currDtxMgr); upToDateTable.put(firstResUuid, currDtxMgr, new CountDownLatch(1)); targetTxId += NB_OF_TEST_TX; testTable.put(Long.valueOf(targetTxId), secondResUuid, currDtxMgr); upToDateTable.put(secondResUuid, currDtxMgr, new CountDownLatch(1)); } final SeparateStateCountListener upToDateListener = new SeparateStateCountListener(upToDateTable, UP_TO_DATE); final Table<DtxResourceManager, Long, Path> resMgrMap = prepareExistingJournals(testTable, DTX_MGR_JOURNAL_MAP, SETUP_ROT_MGR); for (final DtxResourceManager currResMgr : resMgrMap.rowKeySet()) { final Map<Long, Path> currRow = resMgrMap.row(currResMgr); for (final Long currIndex : currRow.keySet()) { final UUID currResId = currResMgr.getId(); final DtxManager currDtxMgr = testTable.get(currIndex, currResId); currDtxMgr.registerDtxEventListener(upToDateListener); currDtxMgr.registerResourceManager(currResMgr); } } // waits for all nodes to be up to date for (final DtxManager currDtxMgr : upToDateTable.row(firstResUuid).keySet()) { assertTrue(upToDateTable.get(firstResUuid, currDtxMgr).await(SYNC_WAIT_TIME_S, TimeUnit.SECONDS)); } for (final DtxManager currDtxMgr : upToDateTable.row(secondResUuid).keySet()) { assertTrue(upToDateTable.get(secondResUuid, currDtxMgr).await(SYNC_WAIT_TIME_S, TimeUnit.SECONDS)); } for (final DtxManager currDtxMgr : dtxMgrs) { currDtxMgr.unregisterDtxEventListener(upToDateListener); currDtxMgr.unregisterResourceManager(firstResUuid); currDtxMgr.unregisterResourceManager(secondResUuid); } upToDateListener.checkForAssertErrors(LOGGER); } /** * Tests successful synchronization of nodes upon startup in increasing last transaction ID order to trigger * sync->post-sync->late cycles for starts beyond the quorum. * * @throws Exception * if setup fails, not part of this test */ @Test public final void testSyncWithPostSyncCycles() throws Exception { LOGGER.info("Executing"); final Set<DtxManager> dtxMgrs = DTX_MGR_JOURNAL_MAP.keySet(); final UUID resUuid = UUID.randomUUID(); final TreeBasedTable<Long, UUID, DtxManager> testTable = TreeBasedTable.create(); long targetTxId = DtxTestHelper.nextTxId(); final long firstTxId = targetTxId; for (final DtxManager currDtxMgr : dtxMgrs) { currDtxMgr.stop(); targetTxId += NB_OF_TEST_TX; testTable.put(Long.valueOf(targetTxId), resUuid, currDtxMgr); } final Table<DtxResourceManager, Long, Path> resMgrMap = prepareExistingJournals(testTable, DTX_MGR_JOURNAL_MAP, SETUP_ROT_MGR); final HashBasedTable<UUID, DtxManager, CountDownLatch> upToDateTable = HashBasedTable.create(); for (final Long currIndex : testTable.rowKeySet()) { upToDateTable.put(resUuid, testTable.get(currIndex, resUuid), new CountDownLatch(1)); } final SeparateStateCountListener upToDateListener = new SeparateStateCountListener(upToDateTable, UP_TO_DATE); // registers resource managers for (final DtxResourceManager currResMgr : resMgrMap.rowKeySet()) { final Map<Long, Path> currRow = resMgrMap.row(currResMgr); for (final Long currIndex : currRow.keySet()) { final UUID currResId = currResMgr.getId(); final DtxManager currDtxMgr = testTable.get(currIndex, currResId); currDtxMgr.registerDtxEventListener(upToDateListener); final DtxResourceManager longPostSyncResMgr = new DtxResourceManagerBuilder().setId(currResId) .setPostSync(null, new Answer<Void>() { @Override public final Void answer(final InvocationOnMock invocation) throws Throwable { Thread.sleep(POST_SYNC_DURATION_MS); return null; } }).build(); currDtxMgr.registerResourceManager(longPostSyncResMgr); } } // gets the set of DtxManagers ordered by increasing last tx ID final Set<DtxManager> ordDtxMgrSet = upToDateTable.row(resUuid).keySet(); final ArrayList<TransactionManager> txMgrList = new ArrayList<TransactionManager>(); // starts nodes in increasing last tx ID order to trigger post-sync->late cycles for (final DtxManager currDtxMgr : dtxMgrs) { if (STARTED == currDtxMgr.getStatus()) { continue; } LOGGER.debug("Starting; node=" + currDtxMgr.getNodeId()); currDtxMgr.start(); LOGGER.debug("Started; node=" + currDtxMgr.getNodeId() + ", last txId=" + currDtxMgr.getLastCompleteTxIdForResMgr(resUuid)); txMgrList.add(currDtxMgr.getTxManager()); } // waits for all nodes to be up to date for (final DtxManager currDtxMgr : ordDtxMgrSet) { assertTrue(upToDateTable.get(resUuid, currDtxMgr).await(2 * SYNC_WAIT_TIME_S, TimeUnit.SECONDS)); } // delay verification by a few milliseconds to let things settle down Thread.sleep(PRE_VERIFICATION_LATENCY_MS); final int nbOfTx = Long.valueOf(targetTxId - firstTxId).intValue(); checkJournalSync(resUuid, txMgrList, nbOfTx); for (final DtxManager currDtxMgr : dtxMgrs) { currDtxMgr.unregisterDtxEventListener(upToDateListener); currDtxMgr.unregisterResourceManager(resUuid); } upToDateListener.checkForAssertErrors(LOGGER); } /** * Tests successful synchronization of nodes after shutting down any one node. * * @throws XAException * if mock setup fails, not part of this test * @throws IOException * if writing transactions to journals fails, not part of this test * @throws IllegalStateException * if setting up existing journals fails, not part of this test * @throws InterruptedException * if interrupted while waiting for completion, not part of this test */ @Test public final void testSyncOneResMgrTempOffline() throws XAException, IllegalStateException, IOException, InterruptedException { LOGGER.info("Executing"); final UUID resUuid = UUID.randomUUID(); final TreeBasedTable<Long, UUID, DtxManager> testTable = TreeBasedTable.create(); long targetTxId = DtxTestHelper.nextTxId(); final Set<DtxManager> dtxMgrs = DTX_MGR_JOURNAL_MAP.keySet(); for (final DtxManager currDtxMgr : dtxMgrs) { targetTxId += NB_OF_TEST_TX; testTable.put(Long.valueOf(targetTxId), resUuid, currDtxMgr); } final Table<DtxResourceManager, Long, Path> resMgrMap = prepareExistingJournals(testTable, DTX_MGR_JOURNAL_MAP, SETUP_ROT_MGR); final HashBasedTable<UUID, DtxManager, CountDownLatch> upToDateTable = HashBasedTable.create(); for (final Long currIndex : testTable.rowKeySet()) { upToDateTable.put(resUuid, testTable.get(currIndex, resUuid), new CountDownLatch(1)); } final SeparateStateCountListener upToDateListener = new SeparateStateCountListener(upToDateTable, UP_TO_DATE); // gets the set of DtxManagers ordered by increasing last tx ID final Set<DtxManager> ordDtxMgrSet = upToDateTable.row(resUuid).keySet(); // initializes and starts all nodes for (final DtxManager currDtxMgr : ordDtxMgrSet) { currDtxMgr.init(); LOGGER.debug("Initialized; node=" + currDtxMgr.getNodeId()); currDtxMgr.start(); LOGGER.debug("Started; node=" + currDtxMgr.getNodeId() + ", last txId=" + currDtxMgr.getLastCompleteTxIdForResMgr(resUuid)); } for (final DtxResourceManager currResMgr : resMgrMap.rowKeySet()) { final Map<Long, Path> currRow = resMgrMap.row(currResMgr); for (final Long currIndex : currRow.keySet()) { final UUID currResId = currResMgr.getId(); final DtxManager currDtxMgr = testTable.get(currIndex, currResId); currDtxMgr.registerDtxEventListener(upToDateListener); currDtxMgr.registerResourceManager(currResMgr); } } // waits for all nodes to be up to date for (final DtxManager currDtxMgr : ordDtxMgrSet) { assertTrue(upToDateTable.get(resUuid, currDtxMgr).await(SYNC_WAIT_TIME_S, TimeUnit.SECONDS)); currDtxMgr.unregisterDtxEventListener(upToDateListener); } final ArrayList<DtxManager> dtxMgrKillList = new ArrayList<DtxManager>(dtxMgrs); Collections.shuffle(dtxMgrKillList); // stops one node out of order, executes some transactions and then lets it catch up for (final DtxManager currVictimDtxMgr : dtxMgrKillList) { final CountDownLatch catchUpLatch = new CountDownLatch(1); final HashMultimap<UUID, DtxManager> catchUpMap = HashMultimap.create(); catchUpMap.put(resUuid, currVictimDtxMgr); final StateCountListener resyncListener = new StateCountListener(catchUpLatch, UP_TO_DATE, catchUpMap); currVictimDtxMgr.registerDtxEventListener(resyncListener); final long lastOnlineTxId = currVictimDtxMgr.getLastCompleteTxIdForResMgr(resUuid); currVictimDtxMgr.stop(); assertEquals(INITIALIZED, currVictimDtxMgr.getStatus()); final int submitIndex = (dtxMgrKillList.indexOf(currVictimDtxMgr) + 1) % dtxMgrKillList.size(); final DtxManager submitNode = dtxMgrKillList.get(submitIndex); for (int i = 0; i < NB_OF_TEST_TX; i++) { final UUID taskId = submitNode.submit(resUuid, DEFAULT_PAYLOAD); while (DtxTaskStatus.COMMITTED != submitNode.getTask(taskId).getStatus()) { Thread.sleep(TX_WAIT_TIME_MS); } } final long newLastTxId = submitNode.getLastCompleteTxIdForResMgr(resUuid); assertTrue(lastOnlineTxId + NB_OF_TEST_TX <= newLastTxId); currVictimDtxMgr.start(); assertTrue(catchUpLatch.await(SYNC_WAIT_TIME_S, TimeUnit.SECONDS)); assertEquals(newLastTxId, currVictimDtxMgr.getLastCompleteTxIdForResMgr(resUuid)); currVictimDtxMgr.unregisterDtxEventListener(resyncListener); } for (final DtxManager currDtxMgr : dtxMgrs) { currDtxMgr.unregisterResourceManager(resUuid); } upToDateListener.checkForAssertErrors(LOGGER); } /** * Tests successful synchronization of nodes upon registering on a started cluster. * * @throws XAException * if mock setup fails, not part of this test * @throws IOException * if writing transactions to journals fails, not part of this test * @throws IllegalStateException * if setting up existing journals fails, not part of this test * @throws InterruptedException * if interrupted while waiting for completion, not part of this test */ @Test public final void testSyncPartialTempReplayFailure() throws XAException, IllegalStateException, IOException, InterruptedException { LOGGER.info("Executing"); final UUID resUuid = UUID.randomUUID(); final TreeBasedTable<Long, UUID, DtxManager> testTable = TreeBasedTable.create(); long targetTxId = DtxTestHelper.nextTxId(); final long firstTxId = targetTxId; final Set<DtxManager> dtxMgrs = DTX_MGR_JOURNAL_MAP.keySet(); for (final DtxManager currDtxMgr : dtxMgrs) { targetTxId += NB_OF_TEST_TX; testTable.put(Long.valueOf(targetTxId), resUuid, currDtxMgr); } final Table<DtxResourceManager, Long, Path> resMgrMap = prepareExistingJournals(testTable, DTX_MGR_JOURNAL_MAP, SETUP_ROT_MGR); final HashBasedTable<UUID, DtxManager, CountDownLatch> syncTable = HashBasedTable.create(); for (final Long currIndex : testTable.rowKeySet()) { if (currIndex.longValue() == targetTxId) { continue; } syncTable.put(resUuid, testTable.get(currIndex, resUuid), new CountDownLatch(NB_OF_SYNC_ATTEMPTS)); } final Set<DtxManager> lateDtxMgrs = syncTable.columnKeySet(); final SeparateStateCountListener syncListener = new SeparateStateCountListener(syncTable, SYNCHRONIZING); final HashBasedTable<UUID, DtxManager, CountDownLatch> upToDateTable = HashBasedTable.create(); for (final Long currIndex : testTable.rowKeySet()) { upToDateTable.put(resUuid, testTable.get(currIndex, resUuid), new CountDownLatch(1)); } final SeparateStateCountListener upToDateListener = new SeparateStateCountListener(upToDateTable, UP_TO_DATE); final ArrayList<ToggleAnswer<Void>> answerList = new ArrayList<ToggleAnswer<Void>>(); for (final DtxResourceManager currResMgr : resMgrMap.rowKeySet()) { for (final Long currIndex : resMgrMap.row(currResMgr).keySet()) { final DtxManager currDtxMgr = testTable.get(currIndex, resUuid); currDtxMgr.registerDtxEventListener(syncListener); currDtxMgr.registerDtxEventListener(upToDateListener); // register a persistently failing resource manager to keep synchronizing if (lateDtxMgrs.contains(currDtxMgr)) { final ToggleAnswer<Void> commitAns = new ToggleAnswer<Void>(true, XAER_RMERR, new DtxDummyRmFactory.DefaultCommitAnswer()); final ToggleAnswer<Void> rollbackAns = new ToggleAnswer<Void>(true, XAER_RMERR, new DtxDummyRmFactory.DefaultRollbackAnswer()); final DtxResourceManager faultyResourceManager = new DtxResourceManagerBuilder().setId(resUuid) .setCommit(null, commitAns).setRollback(null, rollbackAns).build(); answerList.add(commitAns); answerList.add(rollbackAns); currDtxMgr.registerResourceManager(faultyResourceManager); } else { currDtxMgr.registerResourceManager(currResMgr); } } } // waits for all late nodes to at least try synchronizing for (final DtxManager currDtxMgr : syncTable.row(resUuid).keySet()) { assertTrue(syncTable.get(resUuid, currDtxMgr).await(SYNC_WAIT_TIME_S, TimeUnit.SECONDS)); } // checks that no successful synchronization happened final ArrayList<TransactionManager> txMgrList = new ArrayList<TransactionManager>(); for (final DtxManager currDtxMgr : DTX_MGR_JOURNAL_MAP.keySet()) { final long lastCompleteTx = currDtxMgr.getLastCompleteTxIdForResMgr(resUuid); assertEquals(currDtxMgr, testTable.get(Long.valueOf(lastCompleteTx), resUuid)); txMgrList.add(currDtxMgr.getTxManager()); } // toggles error-generating behavior on each resource manager for (final ToggleAnswer<Void> currAnswer : answerList) { currAnswer.toggleErrorTrigger(); } // waits for all nodes to be up to date for (final DtxManager currDtxMgr : upToDateTable.row(resUuid).keySet()) { assertTrue(upToDateTable.get(resUuid, currDtxMgr).await(2 * SYNC_WAIT_TIME_S, TimeUnit.SECONDS)); } // delay verification by a few milliseconds to let things settle down Thread.sleep(PRE_VERIFICATION_LATENCY_MS); final int nbOfTx = Long.valueOf(targetTxId - firstTxId).intValue(); checkJournalSync(resUuid, txMgrList, nbOfTx); // verify a minimum of transactions actually got replayed for (final DtxManager lateDtxMgr : lateDtxMgrs) { verify(lateDtxMgr.getRegisteredResourceManager(resUuid), atLeast(NB_OF_TEST_TX)).prepare( any(DtxResourceManagerContext.class)); } for (final DtxManager currDtxMgr : dtxMgrs) { currDtxMgr.unregisterDtxEventListener(syncListener); currDtxMgr.unregisterDtxEventListener(upToDateListener); currDtxMgr.unregisterResourceManager(resUuid); } upToDateListener.checkForAssertErrors(LOGGER); syncListener.checkForAssertErrors(LOGGER); } /** * Tests successful synchronization of nodes including a post-sync callback upon registering on a started cluster. * * @throws XAException * if mock setup fails, not part of this test * @throws IOException * if writing transactions to journals fails, not part of this test * @throws IllegalStateException * if setting up existing journals fails, not part of this test * @throws InterruptedException * if interrupted while waiting for completion, not part of this test */ @Test public final void testSyncWithPostSync() throws IllegalStateException, IOException, XAException, InterruptedException { LOGGER.info("Executing"); final UUID resUuid = UUID.randomUUID(); final TreeBasedTable<Long, UUID, DtxManager> testTable = TreeBasedTable.create(); long targetTxId = DtxTestHelper.nextTxId(); final long firstTxId = targetTxId; final Set<DtxManager> dtxMgrs = DTX_MGR_JOURNAL_MAP.keySet(); for (final DtxManager currDtxMgr : dtxMgrs) { targetTxId += NB_OF_TEST_TX; testTable.put(Long.valueOf(targetTxId), resUuid, currDtxMgr); } final Table<DtxResourceManager, Long, Path> resMgrMap = prepareExistingJournals(testTable, DTX_MGR_JOURNAL_MAP, SETUP_ROT_MGR); final HashBasedTable<UUID, DtxManager, CountDownLatch> upToDateTable = HashBasedTable.create(); for (final Long currIndex : testTable.rowKeySet()) { upToDateTable.put(resUuid, testTable.get(currIndex, resUuid), new CountDownLatch(1)); } final SeparateStateCountListener upToDateListener = new SeparateStateCountListener(upToDateTable, UP_TO_DATE); final HashBasedTable<UUID, DtxManager, CountDownLatch> postProcTable = HashBasedTable.create(); for (final Long currIndex : testTable.rowKeySet()) { postProcTable.put(resUuid, testTable.get(currIndex, resUuid), new CountDownLatch(1)); } final SeparateStateCountListener postProcListener = new SeparateStateCountListener(postProcTable, POST_SYNC_PROCESSING); final ArrayList<TransactionManager> txMgrList = new ArrayList<TransactionManager>(); // gets the set of DtxManagers ordered by increasing last tx ID final Set<DtxManager> ordDtxMgrSet = upToDateTable.row(resUuid).keySet(); // initializes and starts all nodes for (final DtxManager currDtxMgr : ordDtxMgrSet) { currDtxMgr.init(); LOGGER.debug("Initialized; node=" + currDtxMgr.getNodeId()); currDtxMgr.start(); LOGGER.debug("Started; node=" + currDtxMgr.getNodeId() + ", last txId=" + currDtxMgr.getLastCompleteTxIdForResMgr(resUuid)); txMgrList.add(currDtxMgr.getTxManager()); } for (final DtxResourceManager currResMgr : resMgrMap.rowKeySet()) { final Map<Long, Path> currRow = resMgrMap.row(currResMgr); for (final Long currIndex : currRow.keySet()) { final UUID currResId = currResMgr.getId(); final DtxManager currDtxMgr = testTable.get(currIndex, currResId); currDtxMgr.registerDtxEventListener(upToDateListener); currDtxMgr.registerDtxEventListener(postProcListener); currDtxMgr.registerResourceManager(currResMgr); } } // waits for all nodes to be post-sync processing, then up-to-date and verifies post-sync processing was called for (final DtxManager currDtxMgr : upToDateTable.row(resUuid).keySet()) { assertTrue(postProcTable.get(resUuid, currDtxMgr).await(SYNC_WAIT_TIME_S, TimeUnit.SECONDS)); assertTrue(upToDateTable.get(resUuid, currDtxMgr).await(SYNC_WAIT_TIME_S, TimeUnit.SECONDS)); try { verify(currDtxMgr.getRegisteredResourceManager(resUuid), atLeast(1)).processPostSync(); } catch (final Exception e) { LOGGER.warn("Post-sync processing threw exception", e); } } final int nbOfTx = Long.valueOf(targetTxId - firstTxId).intValue(); checkJournalSync(resUuid, txMgrList, nbOfTx); // verify a minimum of transactions actually got replayed for (final Long currIndex : testTable.rowKeySet()) { final DtxManager currDtxMgr = testTable.get(currIndex, resUuid); final int expectedDiff = Long.valueOf(targetTxId - currIndex.longValue()).intValue(); verify(currDtxMgr.getRegisteredResourceManager(resUuid), atLeast(expectedDiff)).prepare( any(DtxResourceManagerContext.class)); } for (final DtxManager currDtxMgr : dtxMgrs) { currDtxMgr.unregisterDtxEventListener(upToDateListener); currDtxMgr.unregisterDtxEventListener(postProcListener); currDtxMgr.unregisterResourceManager(resUuid); } upToDateListener.checkForAssertErrors(LOGGER); postProcListener.checkForAssertErrors(LOGGER); } /** * Tests successful synchronization of nodes including a post-sync callback upon registering on a started cluster. * * @throws Exception * if mock setup fails, not part of this test */ @Test public final void testSyncWithTxDuringPostSync() throws Exception { LOGGER.info("Executing"); final UUID resUuid = UUID.randomUUID(); final TreeBasedTable<Long, UUID, DtxManager> testTable = TreeBasedTable.create(); long targetTxId = DtxTestHelper.nextTxId(); final long firstTxId = targetTxId; final Set<DtxManager> dtxMgrs = DTX_MGR_JOURNAL_MAP.keySet(); for (final DtxManager currDtxMgr : dtxMgrs) { targetTxId += NB_OF_TEST_TX; testTable.put(Long.valueOf(targetTxId), resUuid, currDtxMgr); } final Table<DtxResourceManager, Long, Path> resMgrMap = prepareExistingJournals(testTable, DTX_MGR_JOURNAL_MAP, SETUP_ROT_MGR); final HashBasedTable<UUID, DtxManager, CountDownLatch> upToDateTable = HashBasedTable.create(); for (final Long currIndex : testTable.rowKeySet()) { upToDateTable.put(resUuid, testTable.get(currIndex, resUuid), new CountDownLatch(1)); } final SeparateStateCountListener upToDateListener = new SeparateStateCountListener(upToDateTable, UP_TO_DATE); final HashBasedTable<UUID, DtxManager, CountDownLatch> postProcTable = HashBasedTable.create(); for (final Long currIndex : testTable.rowKeySet()) { postProcTable.put(resUuid, testTable.get(currIndex, resUuid), new CountDownLatch(1)); } final SeparateStateCountListener postProcListener = new SeparateStateCountListener(postProcTable, POST_SYNC_PROCESSING); final ArrayList<TransactionManager> txMgrList = new ArrayList<TransactionManager>(); // gets the set of DtxManagers ordered by increasing last tx ID final Set<DtxManager> ordDtxMgrSet = upToDateTable.row(resUuid).keySet(); // initializes and starts all nodes for (final DtxManager currDtxMgr : ordDtxMgrSet) { currDtxMgr.init(); LOGGER.debug("Initialized; node=" + currDtxMgr.getNodeId()); currDtxMgr.start(); LOGGER.debug("Started; node=" + currDtxMgr.getNodeId() + ", last txId=" + currDtxMgr.getLastCompleteTxIdForResMgr(resUuid)); txMgrList.add(currDtxMgr.getTxManager()); } for (final DtxResourceManager currResMgr : resMgrMap.rowKeySet()) { final Map<Long, Path> currRow = resMgrMap.row(currResMgr); for (final Long currIndex : currRow.keySet()) { final UUID currResId = currResMgr.getId(); final DtxManager currDtxMgr = testTable.get(currIndex, currResId); currDtxMgr.registerDtxEventListener(upToDateListener); currDtxMgr.registerDtxEventListener(postProcListener); final DtxResourceManager postSyncResMgr = new DtxDummyRmFactory.DtxResourceManagerBuilder() .setId(currResId).setPostSync(null, new Answer<Void>() { @Override public Void answer(final InvocationOnMock invocation) throws Throwable { final UUID taskId = currDtxMgr.submit(currResId, DtxDummyRmFactory.DEFAULT_PAYLOAD); final DtxTaskStatus targetStatus; if (currDtxMgr.isUpToDateOnClusterMapQuorum(currResId)) { targetStatus = COMMITTED; } else { targetStatus = ROLLED_BACK; } DtxTaskStatus currStatus; final ArrayList<DtxTaskStatus> statusList = new ArrayList<DtxTaskStatus>(); statusList.add(UNKNOWN); do { Thread.sleep(TX_WAIT_TIME_MS); currStatus = currDtxMgr.getTask(taskId).getStatus(); if (currStatus != statusList.get(statusList.size() - 1)) { statusList.add(currStatus); LOGGER.debug("taskId=" + taskId + ", status evolution=" + statusList + ", expected final=" + targetStatus); } } while (targetStatus != currStatus); return null; } }).build(); currDtxMgr.registerResourceManager(postSyncResMgr); } } // waits for all nodes to be post-sync processing for (final DtxManager currDtxMgr : upToDateTable.row(resUuid).keySet()) { assertTrue(postProcTable.get(resUuid, currDtxMgr).await(SYNC_WAIT_TIME_S, TimeUnit.SECONDS)); } // waits for all nodes to be up-to-date and verifies post-sync processing was called for (final DtxManager currDtxMgr : upToDateTable.row(resUuid).keySet()) { assertTrue(upToDateTable.get(resUuid, currDtxMgr).await(SYNC_WAIT_TIME_S, TimeUnit.SECONDS)); try { verify(currDtxMgr.getRegisteredResourceManager(resUuid), atLeast(1)).processPostSync(); } catch (final Exception e) { LOGGER.warn("Post-sync processing threw exception", e); } } // delay verification by a few milliseconds to let things settle down Thread.sleep(PRE_VERIFICATION_LATENCY_MS); final int nbOfTx = Long.valueOf(targetTxId - firstTxId).intValue(); checkJournalSync(resUuid, txMgrList, nbOfTx); // verify a minimum of transactions actually got replayed for (final Long currIndex : testTable.rowKeySet()) { final DtxManager currDtxMgr = testTable.get(currIndex, resUuid); final int expectedDiff = Long.valueOf(targetTxId - currIndex.longValue()).intValue(); verify(currDtxMgr.getRegisteredResourceManager(resUuid), atLeast(expectedDiff)).prepare( any(DtxResourceManagerContext.class)); } for (final DtxManager currDtxMgr : dtxMgrs) { currDtxMgr.unregisterDtxEventListener(upToDateListener); currDtxMgr.unregisterDtxEventListener(postProcListener); currDtxMgr.unregisterResourceManager(resUuid); } upToDateListener.checkForAssertErrors(LOGGER); postProcListener.checkForAssertErrors(LOGGER); } }