/** Copyright (C) SYSTAP, LLC DBA Blazegraph 2006-2016. All rights reserved. Contact: SYSTAP, LLC DBA Blazegraph 2501 Calvert ST NW #106 Washington, DC 20008 licenses@blazegraph.com This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; version 2 of the License. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA */ /* * Created on Jun 2, 2010 */ package com.bigdata.quorum; import java.util.Arrays; import java.util.Map; import java.util.UUID; import com.bigdata.quorum.MockQuorumFixture.MockQuorumMember; /** * Test the quorum semantics for a highly available quorum of 3 services. The * main points to test here are the particulars of events not observable with a * singleton quorum, including that all events are perceived by all clients via * the watcher for their quorum, that a service join which does not trigger a * quorum meet, that a service leave which does not trigger a quorum break, a * leader leave, etc. * <p> * These conditions arise because the quorum can meet with 2 out of 3 services * forming a consensus. Even if the third service has the same lastCommitTime, * if it votes after the other services, then it can be in the position of * joining a quorum which has already met. This leads naturally into an * exploration of the synchronization protocol for the persistent state of the * services when joining a met quorum. * * @author <a href="mailto:thompsonbry@users.sourceforge.net">Bryan Thompson</a> * * FIXME We must also test with k:=5 in order to see some situations * which do not appear with k:=3 (not sure which ones off hand, but I * remember noting that there are some). * * FIXME There will need to be unit tests for synchronization when a * service wants to join a met quorum. Persistent services need to * verify their root blocks in detail against the leader. If they are * not identical, then the services need to synchronize rather than * join. Also, services which attempt to join an already met quorum * must synchronize before they can join. All of this needs to be * tested, but we need to do those tests with live journals. * * FIXME There will need to be unit tests for hot spares. */ public class TestHA3QuorumSemantics extends AbstractQuorumTestCase { /** * */ public TestHA3QuorumSemantics() { } /** * @param name */ public TestHA3QuorumSemantics(String name) { super(name); } @Override protected void setUp() throws Exception { k = 3; super.setUp(); } /** * Unit test for quorum member add/remove. * * @throws InterruptedException */ public void test_memberAddRemove3() throws InterruptedException { final Quorum<?, ?> quorum0 = quorums[0]; final QuorumMember<?> client0 = clients[0]; final QuorumActor<?,?> actor0 = actors[0]; final UUID serviceId0 = client0.getServiceId(); final Quorum<?, ?> quorum1 = quorums[1]; final QuorumMember<?> client1 = clients[1]; final QuorumActor<?,?> actor1 = actors[1]; final UUID serviceId1 = client1.getServiceId(); final Quorum<?, ?> quorum2 = quorums[2]; final QuorumMember<?> client2 = clients[2]; final QuorumActor<?,?> actor2 = actors[2]; final UUID serviceId2 = client2.getServiceId(); // client is not a member. assertFalse(client0.isMember()); assertFalse(client1.isMember()); assertFalse(client2.isMember()); assertEquals(new UUID[] {}, quorum0.getMembers()); assertEquals(new UUID[] {}, quorum1.getMembers()); assertEquals(new UUID[] {}, quorum2.getMembers()); // instruct an actor to add its client as a member. actor0.memberAdd(); fixture.awaitDeque(); // client is a member. assertTrue(client0.isMember()); // assertFalse(client1.isMember()); // assertFalse(client2.isMember()); assertEquals(new UUID[] {serviceId0}, quorum0.getMembers()); // assertEquals(new UUID[] {serviceId0}, quorum1.getMembers()); // assertEquals(new UUID[] {serviceId0}, quorum2.getMembers()); // instruct an actor to add its client as a member. actor1.memberAdd(); fixture.awaitDeque(); // the client is now a member. // assertTrue(client0.isMember()); assertTrue(client1.isMember()); // assertFalse(client2.isMember()); // assertEquals(new UUID[] {serviceId0,serviceId1}, quorum0.getMembers()); assertEquals(new UUID[] {serviceId0,serviceId1}, quorum1.getMembers()); // assertEquals(new UUID[] {serviceId0,serviceId1}, quorum2.getMembers()); // instruct an actor to remove its client as a member. actor0.memberRemove(); fixture.awaitDeque(); // the client is no longer a member. assertFalse(client0.isMember()); // assertTrue(client1.isMember()); // assertFalse(client2.isMember()); assertEquals(new UUID[] {serviceId1}, quorum0.getMembers()); // assertEquals(new UUID[] {serviceId1}, quorum1.getMembers()); // assertEquals(new UUID[] {serviceId1}, quorum2.getMembers()); // instruct an actor to remove its client as a member. actor1.memberRemove(); fixture.awaitDeque(); // the client is no longer a member. // assertFalse(client0.isMember()); assertFalse(client1.isMember()); // assertFalse(client2.isMember()); // assertEquals(new UUID[] {}, quorum0.getMembers()); assertEquals(new UUID[] {}, quorum1.getMembers()); // assertEquals(new UUID[] {}, quorum2.getMembers()); } /** * Unit test for write pipeline add/remove, including the * {@link PipelineState} of the downstream service as maintained by the * {@link MockQuorumMember}. * * @throws InterruptedException */ public void test_pipelineAddRemove3() throws InterruptedException { final Quorum<?, ?> quorum0 = quorums[0]; final MockQuorumMember<?> client0 = clients[0]; final QuorumActor<?,?> actor0 = actors[0]; final UUID serviceId0 = client0.getServiceId(); final Quorum<?, ?> quorum1 = quorums[1]; final MockQuorumMember<?> client1 = clients[1]; final QuorumActor<?,?> actor1 = actors[1]; final UUID serviceId1 = client1.getServiceId(); final Quorum<?, ?> quorum2 = quorums[2]; final MockQuorumMember<?> client2 = clients[2]; final QuorumActor<?,?> actor2 = actors[2]; final UUID serviceId2 = client2.getServiceId(); // Verify the initial pipeline state. assertNull(client0.downStreamId); assertNull(client1.downStreamId); assertNull(client2.downStreamId); assertFalse(client0.isPipelineMember()); assertFalse(client1.isPipelineMember()); assertFalse(client2.isPipelineMember()); assertEquals(new UUID[]{},quorum0.getPipeline()); assertEquals(new UUID[]{},quorum1.getPipeline()); assertEquals(new UUID[]{},quorum2.getPipeline()); // Add the services to the quorum. actor0.memberAdd(); actor1.memberAdd(); actor2.memberAdd(); fixture.awaitDeque(); assertCondition(new Runnable() { public void run() { assertEquals(new UUID[] { serviceId0, serviceId1, serviceId2 }, quorum0.getMembers()); } }); /* * Add each service in turn to the pipeline, verifying the changes in * the pipeline state as we go. */ actor0.pipelineAdd(); fixture.awaitDeque(); assertTrue(client0.isPipelineMember()); // assertFalse(client1.isPipelineMember()); // assertFalse(client2.isPipelineMember()); assertEquals(new UUID[]{serviceId0},quorum0.getPipeline()); // assertEquals(new UUID[]{serviceId0},quorum1.getPipeline()); // assertEquals(new UUID[]{serviceId0},quorum2.getPipeline()); assertNull(client0.downStreamId); // assertNull(client1.downStreamId); // assertNull(client2.downStreamId); actor1.pipelineAdd(); fixture.awaitDeque(); // assertTrue(client0.isPipelineMember()); assertTrue(client1.isPipelineMember()); // assertFalse(client2.isPipelineMember()); // assertEquals(new UUID[]{serviceId0,serviceId1},quorum0.getPipeline()); assertEquals(new UUID[]{serviceId0,serviceId1},quorum1.getPipeline()); // assertEquals(new UUID[]{serviceId0,serviceId1},quorum2.getPipeline()); // assertEquals(serviceId1,client0.downStreamId); assertNull(client1.downStreamId); // assertNull(client2.downStreamId); actor2.pipelineAdd(); fixture.awaitDeque(); // assertTrue(client0.isPipelineMember()); // assertTrue(client1.isPipelineMember()); assertTrue(client2.isPipelineMember()); // assertEquals(new UUID[]{serviceId0,serviceId1,serviceId2},quorum0.getPipeline()); // assertEquals(new UUID[]{serviceId0,serviceId1,serviceId2},quorum1.getPipeline()); assertEquals(new UUID[]{serviceId0,serviceId1,serviceId2},quorum2.getPipeline()); // assertEquals(serviceId1,client0.downStreamId); // assertEquals(serviceId2,client1.downStreamId); assertNull(client2.downStreamId); assertCondition(new Runnable() { public void run() { // test prior/next. assertEquals(new UUID[] { null, serviceId1 }, quorum0 .getPipelinePriorAndNext(serviceId0)); assertEquals(new UUID[] { serviceId0, serviceId2 }, quorum0 .getPipelinePriorAndNext(serviceId1)); assertEquals(new UUID[] { serviceId1, null }, quorum0 .getPipelinePriorAndNext(serviceId2)); } }); // remove one from the pipeline. actor1.pipelineRemove(); fixture.awaitDeque(); // assertTrue(client0.isPipelineMember()); assertFalse(client1.isPipelineMember()); // assertTrue(client2.isPipelineMember()); // assertEquals(new UUID[]{serviceId0,serviceId2},quorum0.getPipeline()); assertEquals(new UUID[]{serviceId0,serviceId2},quorum1.getPipeline()); // assertEquals(new UUID[]{serviceId0,serviceId2},quorum2.getPipeline()); // assertEquals(serviceId2,client0.downStreamId); assertNull(client1.downStreamId); // assertNull(client2.downStreamId); // remove another from the pipeline. actor0.pipelineRemove(); fixture.awaitDeque(); assertFalse(client0.isPipelineMember()); // assertFalse(client1.isPipelineMember()); // assertTrue(client2.isPipelineMember()); assertEquals(new UUID[]{serviceId2},quorum0.getPipeline()); // assertEquals(new UUID[]{serviceId2},quorum1.getPipeline()); // assertEquals(new UUID[]{serviceId2},quorum2.getPipeline()); assertNull(client0.downStreamId); // assertNull(client1.downStreamId); // assertNull(client2.downStreamId); // remove the last service from the pipeline. actor2.pipelineRemove(); fixture.awaitDeque(); // assertFalse(client0.isPipelineMember()); // assertFalse(client1.isPipelineMember()); assertFalse(client2.isPipelineMember()); // assertEquals(new UUID[]{},quorum0.getPipeline()); // assertEquals(new UUID[]{},quorum1.getPipeline()); assertEquals(new UUID[]{},quorum2.getPipeline()); // assertNull(client0.downStreamId); // assertNull(client1.downStreamId); assertNull(client2.downStreamId); // remove the members from the quorum. actor0.memberRemove(); actor1.memberRemove(); actor2.memberRemove(); fixture.awaitDeque(); } /** * Unit test for the voting protocol. * * @throws InterruptedException */ public void test_voting3() throws InterruptedException { final AbstractQuorum<?, ?> quorum0 = quorums[0]; final MockQuorumMember<?> client0 = clients[0]; final QuorumActor<?, ?> actor0 = actors[0]; final UUID serviceId0 = client0.getServiceId(); final AbstractQuorum<?, ?> quorum1 = quorums[1]; final MockQuorumMember<?> client1 = clients[1]; final QuorumActor<?,?> actor1 = actors[1]; final UUID serviceId1 = client1.getServiceId(); final AbstractQuorum<?, ?> quorum2 = quorums[2]; final MockQuorumMember<?> client2 = clients[2]; final QuorumActor<?,?> actor2 = actors[2]; final UUID serviceId2 = client2.getServiceId(); final long lastCommitTime1 = 0L; final long lastCommitTime2 = 2L; // Verify that no consensus has been achieved yet. assertEquals(-1L, client0.lastConsensusValue); assertEquals(-1L, client1.lastConsensusValue); assertEquals(-1L, client2.lastConsensusValue); // add as member services. actor0.memberAdd(); actor1.memberAdd(); actor2.memberAdd(); fixture.awaitDeque(); // join the pipeline. actor0.pipelineAdd(); actor1.pipelineAdd(); actor2.pipelineAdd(); fixture.awaitDeque(); // Should not be any votes. assertEquals(0,quorum0.getVotes().size()); assertEquals(0,quorum1.getVotes().size()); assertEquals(0,quorum2.getVotes().size()); // Cast a vote. actor0.castVote(lastCommitTime1); fixture.awaitDeque(); // The last consensus timestamp should not have been updated. assertEquals(-1L, client0.lastConsensusValue); assertEquals(-1L, client1.lastConsensusValue); assertEquals(-1L, client2.lastConsensusValue); // Should be just one timestamp for which services have voted. assertEquals(1,quorum0.getVotes().size()); // assertEquals(1,quorum1.getVotes().size()); // assertEquals(1,quorum2.getVotes().size()); // Cast a vote for a different timestamp. actor1.castVote(lastCommitTime2); fixture.awaitDeque(); // The last consensus timestamp should not have been updated. assertEquals(-1L, client0.lastConsensusValue); assertEquals(-1L, client1.lastConsensusValue); assertEquals(-1L, client2.lastConsensusValue); /* * Should be two timestamps for which services have voted (but we can * only. check the one that enacted the change since that is the only * one for which the update is guaranteed to be visible without awaiting * the Condition). */ // assertEquals(2, quorum0.getVotes().size()); assertEquals(2, quorum1.getVotes().size()); // assertEquals(2, quorum2.getVotes().size()); /* * Cast another vote for for the same timestamp. This will trigger a * consensus (simple majority) and the quorum will meet. */ actor2.castVote(lastCommitTime1); fixture.awaitDeque(); // wait for quorums to meet (visibility guarantee). final long token1 = quorum0.awaitQuorum(); assertEquals(token1, quorum1.awaitQuorum()); assertEquals(token1, quorum2.awaitQuorum()); // The last consensus timestamp should have been updated for all quorum members. assertEquals(lastCommitTime1, client0.lastConsensusValue); assertEquals(lastCommitTime1, client1.lastConsensusValue); assertEquals(lastCommitTime1, client2.lastConsensusValue); // Should be two timestamps for which services have voted. assertEquals(2, quorum0.getVotes().size()); assertEquals(2, quorum1.getVotes().size()); assertEquals(2, quorum2.getVotes().size()); // Verify the specific services voting for each timestamp. assertEquals(new UUID[] { serviceId0, serviceId2 }, quorum0.getVotes() .get(lastCommitTime1)); assertEquals(new UUID[] { serviceId1 }, quorum0.getVotes().get( lastCommitTime2)); // The quorum met. // long token1 = quorum0.awaitQuorum(); assertEquals(Quorum.NO_QUORUM + 1, token1); assertEquals(token1, quorum1.awaitQuorum()); assertEquals(token1, quorum2.awaitQuorum()); assertTrue(quorum0.isQuorumMet()); assertTrue(quorum1.isQuorumMet()); assertTrue(quorum2.isQuorumMet()); // Remove as a member. This should not affect the consensus. actor1.memberRemove(); fixture.awaitDeque(); // The last consensus timestamp should not have changed. assertEquals(lastCommitTime1, client0.lastConsensusValue); assertEquals(lastCommitTime1, client1.lastConsensusValue); assertEquals(lastCommitTime1, client2.lastConsensusValue); // Should be just one timestamps for which services have voted (again, // only verify for service that makes the change). // assertEquals(1, quorum0.getVotes().size()); assertEquals(1, quorum1.getVotes().size()); // assertEquals(1, quorum2.getVotes().size()); // Verify the specific services voting for each timestamp (done with the // service which made the change). assertEquals(new UUID[] { serviceId0, serviceId2 }, quorum1.getVotes() .get(lastCommitTime1)); assertEquals(null, quorum1.getVotes().get(lastCommitTime2)); /* * Remove another service as a member. This should break the consensus * and (since the quorum was actually met) this will break the quorum. */ actor0.memberRemove(); fixture.awaitDeque(); // await break to provide visibility for changes (but changes not // required for a quorum break might still not be visible.) quorum0.awaitBreak(); quorum1.awaitBreak(); quorum2.awaitBreak(); // The quorum broke. assertFalse(quorum0.isQuorumMet()); assertFalse(quorum1.isQuorumMet()); assertFalse(quorum2.isQuorumMet()); assertCondition(new Runnable() { public void run() { // The last consensus on the mock clients should have been cleared. assertEquals(-1L, client0.lastConsensusValue); assertEquals(-1L, client1.lastConsensusValue); assertEquals(-1L, client2.lastConsensusValue); // Should be no timestamps for which services have voted. final Map<Long, UUID[]> votes0 = quorum0.getVotes(); final Map<Long, UUID[]> votes1 = quorum1.getVotes(); final Map<Long, UUID[]> votes2 = quorum2.getVotes(); assertEquals(AbstractQuorumTestCase.toString(votes0), 0, votes0.size()); assertEquals(AbstractQuorumTestCase.toString(votes1), 0, votes1.size()); assertEquals(AbstractQuorumTestCase.toString(votes2), 0, votes2.size()); // Verify the specific services voting for each timestamp. assertEquals(null, quorum0.getVotes().get(lastCommitTime1)); assertEquals(null, quorum0.getVotes().get(lastCommitTime2)); } }); // Remove the last member service. actor2.memberRemove(); fixture.awaitDeque(); assertCondition(new Runnable() { public void run() { // No change. assertEquals(-1L, client0.lastConsensusValue); assertEquals(-1L, client1.lastConsensusValue); assertEquals(-1L, client2.lastConsensusValue); // No votes left. assertEquals(0, quorum0.getVotes().size()); assertEquals(0, quorum1.getVotes().size()); assertEquals(0, quorum2.getVotes().size()); // Verify the specific services voting for each timestamp. assertEquals(null, quorum2.getVotes().get(lastCommitTime1)); assertEquals(null, quorum2.getVotes().get(lastCommitTime2)); } }); } /** * Unit test for service join/leave where services vote in the pipeline * order so the leader does not need to reorganize the pipeline when the * number of joined services reaches (k+1)/2. The quorum meets after two * services cast their vote. When the third service casts its vote, it joins * the met quorum. * * @throws InterruptedException */ public void test_serviceJoin3_simple() throws InterruptedException { final Quorum<?, ?> quorum0 = quorums[0]; final MockQuorumMember<?> client0 = clients[0]; final QuorumActor<?, ?> actor0 = actors[0]; final UUID serviceId0 = client0.getServiceId(); final Quorum<?, ?> quorum1 = quorums[1]; final MockQuorumMember<?> client1 = clients[1]; final QuorumActor<?, ?> actor1 = actors[1]; final UUID serviceId1 = client1.getServiceId(); final Quorum<?, ?> quorum2 = quorums[2]; final MockQuorumMember<?> client2 = clients[2]; final QuorumActor<?,?> actor2 = actors[2]; final UUID serviceId2 = client2.getServiceId(); // final long lastCommitTime1 = 0L; // // final long lastCommitTime2 = 2L; final long lastCommitTime = 0L; // declare the services as a quorum members. actor0.memberAdd(); actor1.memberAdd(); actor2.memberAdd(); fixture.awaitDeque(); assertCondition(new Runnable() { public void run() { assertEquals(3, quorum0.getMembers().length); assertEquals(3, quorum1.getMembers().length); assertEquals(3, quorum2.getMembers().length); } }); /* * Have the services join the pipeline. */ actor0.pipelineAdd(); actor1.pipelineAdd(); actor2.pipelineAdd(); fixture.awaitDeque(); assertCondition(new Runnable() { public void run() { assertEquals(new UUID[] { serviceId0, serviceId1, serviceId2 }, quorum0.getPipeline()); assertEquals(new UUID[] { serviceId0, serviceId1, serviceId2 }, quorum1.getPipeline()); assertEquals(new UUID[] { serviceId0, serviceId1, serviceId2 }, quorum2.getPipeline()); } }); /* * Have two services cast a vote for a lastCommitTime. This will cause * the quorum to meet. */ final long token1; { actor0.castVote(lastCommitTime); actor1.castVote(lastCommitTime); fixture.awaitDeque(); // validate the token was assigned (must wait for meet). token1 = quorum0.awaitQuorum(); assertEquals(Quorum.NO_QUORUM + 1, token1); assertEquals(Quorum.NO_QUORUM + 1, quorum0.token()); assertEquals(Quorum.NO_QUORUM + 1, quorum0.lastValidToken()); assertTrue(quorum0.isQuorumMet()); // wait for meet for other clients. assertEquals(token1, quorum1.awaitQuorum()); assertEquals(token1, quorum2.awaitQuorum()); assertEquals(token1, quorum1.lastValidToken()); assertEquals(token1, quorum2.lastValidToken()); assertCondition(new Runnable() { public void run() { // services have voted for a single lastCommitTime. assertEquals(1, quorum0.getVotes().size()); assertEquals(1, quorum1.getVotes().size()); assertEquals(1, quorum2.getVotes().size()); // verify the vote order. assertEquals(new UUID[] { serviceId0, serviceId1 }, quorum0 .getVotes().get(lastCommitTime)); // verify the consensus was updated. assertEquals(lastCommitTime, client0.lastConsensusValue); assertEquals(lastCommitTime, client1.lastConsensusValue); assertEquals(lastCommitTime, client2.lastConsensusValue); /* * Service join in the same order in which they cast their * votes. */ assertEquals(new UUID[] { serviceId0, serviceId1 }, quorum0 .getJoined()); assertEquals(new UUID[] { serviceId0, serviceId1 }, quorum1 .getJoined()); assertEquals(new UUID[] { serviceId0, serviceId1 }, quorum2 .getJoined()); // The pipeline order is the same as the vote order. assertEquals(new UUID[] { serviceId0, serviceId1, serviceId2 }, quorum0.getPipeline()); assertEquals(new UUID[] { serviceId0, serviceId1, serviceId2 }, quorum1.getPipeline()); assertEquals(new UUID[] { serviceId0, serviceId1, serviceId2 }, quorum2.getPipeline()); } }); } /* * Cast the last vote and verify that the last service joins. * * Note: The last service should join immediately since it does not have * to do any validation when it joins. */ { actor2.castVote(lastCommitTime); fixture.awaitDeque(); assertCondition(new Runnable() { public void run() { // services have voted for a single lastCommitTime. assertEquals(1, quorum0.getVotes().size()); // verify the vote order. assertEquals(new UUID[] { serviceId0, serviceId1, serviceId2 }, quorum0.getVotes().get(lastCommitTime)); // verify the consensus was NOT updated. assertEquals(lastCommitTime, client0.lastConsensusValue); assertEquals(lastCommitTime, client1.lastConsensusValue); assertEquals(lastCommitTime, client2.lastConsensusValue); // Service join in the same order in which they cast their votes. assertEquals(new UUID[] { serviceId0, serviceId1, serviceId2 }, quorum0.getJoined()); assertEquals(new UUID[] { serviceId0, serviceId1, serviceId2 }, quorum1.getJoined()); assertEquals(new UUID[] { serviceId0, serviceId1, serviceId2 }, quorum2.getJoined()); } }); // validate the token was NOT updated. assertEquals(token1, quorum0.lastValidToken()); assertEquals(token1, quorum1.lastValidToken()); assertEquals(token1, quorum2.lastValidToken()); assertEquals(token1, quorum0.token()); assertEquals(token1, quorum1.token()); assertEquals(token1, quorum2.token()); assertTrue(quorum0.isQuorumMet()); assertTrue(quorum1.isQuorumMet()); assertTrue(quorum2.isQuorumMet()); // The pipeline order is the same as the vote order. assertEquals(new UUID[] { serviceId0, serviceId1, serviceId2 }, quorum0.getPipeline()); assertEquals(new UUID[] { serviceId0, serviceId1, serviceId2 }, quorum1.getPipeline()); assertEquals(new UUID[] { serviceId0, serviceId1, serviceId2 }, quorum2.getPipeline()); } /* * Follower leave/join test. */ { /* * Fail the first follower. This will not cause a quorum break since * there are still (k+1)/2 services in the quorum. */ actor1.serviceLeave(); fixture.awaitDeque(); assertCondition(new Runnable() { public void run() { // services have voted for a single lastCommitTime. assertEquals(1, quorum0.getVotes().size()); // verify the vote order. assertEquals(new UUID[] { serviceId0, serviceId2 }, quorum0 .getVotes().get(lastCommitTime)); // verify the consensus was NOT updated. assertEquals(lastCommitTime, client0.lastConsensusValue); assertEquals(lastCommitTime, client1.lastConsensusValue); assertEquals(lastCommitTime, client2.lastConsensusValue); /* * Service join in the same order in which they cast their * votes. */ assertEquals(new UUID[] { serviceId0, serviceId2 }, quorum0 .getJoined()); assertEquals(new UUID[] { serviceId0, serviceId2 }, quorum1 .getJoined()); assertEquals(new UUID[] { serviceId0, serviceId2 }, quorum2 .getJoined()); } }); // validate the token was NOT updated. assertEquals(token1, quorum0.lastValidToken()); assertEquals(token1, quorum1.lastValidToken()); assertEquals(token1, quorum2.lastValidToken()); assertEquals(token1, quorum0.token()); assertEquals(token1, quorum1.token()); assertEquals(token1, quorum2.token()); assertTrue(quorum0.isQuorumMet()); assertTrue(quorum1.isQuorumMet()); assertTrue(quorum2.isQuorumMet()); assertCondition(new Runnable() { public void run() { // The pipeline order is the same as the vote order. assertEquals(new UUID[] { serviceId0, serviceId2 }, quorum0 .getPipeline()); assertEquals(new UUID[] { serviceId0, serviceId2 }, quorum1 .getPipeline()); assertEquals(new UUID[] { serviceId0, serviceId2 }, quorum2 .getPipeline()); } }); /* * Rejoin the service. */ actor1.pipelineAdd(); fixture.awaitDeque(); actor1.castVote(lastCommitTime); fixture.awaitDeque(); assertCondition(new Runnable() { public void run() { // services have voted for a single lastCommitTime. assertEquals(1, quorum0.getVotes().size()); assertEquals(1, quorum1.getVotes().size()); assertEquals(1, quorum2.getVotes().size()); // verify the vote order. assertEquals(new UUID[] { serviceId0, serviceId2, serviceId1 }, quorum0.getVotes() .get(lastCommitTime)); assertEquals(new UUID[] { serviceId0, serviceId2, serviceId1 }, quorum1.getVotes() .get(lastCommitTime)); assertEquals(new UUID[] { serviceId0, serviceId2, serviceId1 }, quorum2.getVotes() .get(lastCommitTime)); // verify the consensus was NOT updated. assertEquals(lastCommitTime, client0.lastConsensusValue); assertEquals(lastCommitTime, client1.lastConsensusValue); assertEquals(lastCommitTime, client2.lastConsensusValue); // Service join in the same order in which they cast their // votes. assertEquals(new UUID[] { serviceId0, serviceId2, serviceId1 }, quorum0.getJoined()); assertEquals(new UUID[] { serviceId0, serviceId2, serviceId1 }, quorum1.getJoined()); assertEquals(new UUID[] { serviceId0, serviceId2, serviceId1 }, quorum2.getJoined()); } }); // validate the token was NOT updated. assertEquals(token1, quorum0.lastValidToken()); assertEquals(token1, quorum1.lastValidToken()); assertEquals(token1, quorum2.lastValidToken()); assertEquals(token1, quorum0.token()); assertEquals(token1, quorum1.token()); assertEquals(token1, quorum2.token()); assertTrue(quorum0.isQuorumMet()); assertTrue(quorum1.isQuorumMet()); assertTrue(quorum2.isQuorumMet()); // The pipeline order is the same as the vote order. assertCondition(new Runnable() { public void run() { assertEquals(new UUID[] { serviceId0, serviceId2, serviceId1 }, quorum0.getPipeline()); assertEquals(new UUID[] { serviceId0, serviceId2, serviceId1 }, quorum1.getPipeline()); assertEquals(new UUID[] { serviceId0, serviceId2, serviceId1 }, quorum2.getPipeline()); } }); } /* * Leader leave test. * * This forces the quorum leader to do a serviceLeave(), which causes a * quorum break. All joined services should have left. Their votes were * withdrawn when they left and they were removed from the pipeline as * well. */ { actor0.serviceLeave(); fixture.awaitDeque(); // the votes were withdrawn. assertCondition(new Runnable() { public void run() { assertEquals(0, quorum0.getVotes().size()); assertEquals(0, quorum1.getVotes().size()); assertEquals(0, quorum2.getVotes().size()); } }); assertCondition(new Runnable() { public void run() { // the consensus was cleared. assertEquals(-1L, client0.lastConsensusValue); assertEquals(-1L, client1.lastConsensusValue); assertEquals(-1L, client2.lastConsensusValue); // No one is joined. assertEquals(new UUID[] {}, quorum0.getJoined()); assertEquals(new UUID[] {}, quorum1.getJoined()); assertEquals(new UUID[] {}, quorum2.getJoined()); // validate the token was cleared (lastValidToken is // unchanged). assertEquals(token1, quorum0.lastValidToken()); assertEquals(token1, quorum1.lastValidToken()); assertEquals(token1, quorum2.lastValidToken()); assertEquals(Quorum.NO_QUORUM, quorum0.token()); assertEquals(Quorum.NO_QUORUM, quorum1.token()); assertEquals(Quorum.NO_QUORUM, quorum2.token()); assertFalse(quorum0.isQuorumMet()); assertFalse(quorum1.isQuorumMet()); assertFalse(quorum2.isQuorumMet()); // No one is in the pipeline. assertEquals(new UUID[] {}, quorum0.getPipeline()); assertEquals(new UUID[] {}, quorum1.getPipeline()); assertEquals(new UUID[] {}, quorum2.getPipeline()); } }); } /* * Heal the quorum by rejoining all of the services. */ final long token2; { actor0.pipelineAdd(); actor1.pipelineAdd(); actor2.pipelineAdd(); fixture.awaitDeque(); actor0.castVote(lastCommitTime); actor1.castVote(lastCommitTime); actor2.castVote(lastCommitTime); fixture.awaitDeque(); assertCondition(new Runnable() { public void run() { // services have voted for a single lastCommitTime. assertEquals(1,quorum0.getVotes().size()); assertEquals(1,quorum1.getVotes().size()); assertEquals(1,quorum2.getVotes().size()); // verify the vote order. assertEquals(new UUID[] { serviceId0, serviceId1, serviceId2 }, quorum0.getVotes() .get(lastCommitTime)); assertEquals(new UUID[] { serviceId0, serviceId1, serviceId2 }, quorum1.getVotes() .get(lastCommitTime)); assertEquals(new UUID[] { serviceId0, serviceId1, serviceId2 }, quorum2.getVotes() .get(lastCommitTime)); // verify the consensus was updated. assertEquals(lastCommitTime, client0.lastConsensusValue); assertEquals(lastCommitTime, client1.lastConsensusValue); assertEquals(lastCommitTime, client2.lastConsensusValue); /* * Service join in the same order in which they cast their * votes. */ assertEquals(new UUID[] { serviceId0, serviceId1, serviceId2 }, quorum0.getJoined()); assertEquals(new UUID[] { serviceId0, serviceId1, serviceId2 }, quorum1.getJoined()); assertEquals(new UUID[] { serviceId0, serviceId1, serviceId2 }, quorum2.getJoined()); } }); // validate the token was updated. token2 = quorum0.awaitQuorum(); assertEquals(token2,quorum1.awaitQuorum()); assertEquals(token2,quorum2.awaitQuorum()); assertEquals(token2, quorum0.lastValidToken()); assertEquals(token2, quorum1.lastValidToken()); assertEquals(token2, quorum2.lastValidToken()); assertEquals(token2, quorum0.token()); assertEquals(token2, quorum1.token()); assertEquals(token2, quorum2.token()); assertTrue(quorum0.isQuorumMet()); assertTrue(quorum1.isQuorumMet()); assertTrue(quorum2.isQuorumMet()); // The pipeline order is the same as the vote order. assertCondition(new Runnable() { public void run() { assertEquals(new UUID[] { serviceId0, serviceId1, serviceId2 }, quorum0.getPipeline()); assertEquals(new UUID[] { serviceId0, serviceId1, serviceId2 }, quorum1.getPipeline()); assertEquals(new UUID[] { serviceId0, serviceId1, serviceId2 }, quorum2.getPipeline()); } }); } /* * Cause the quorum to break by failing both of the followers. */ { /* * Fail one follower. The quorum should not break. */ actor2.serviceLeave(); fixture.awaitDeque(); assertCondition(new Runnable() { public void run() { // services have voted for a single lastCommitTime. assertEquals(1, quorum0.getVotes().size()); assertEquals(1, quorum1.getVotes().size()); assertEquals(1, quorum2.getVotes().size()); // verify the vote order. assertEquals(new UUID[] { serviceId0, serviceId1 }, quorum0 .getVotes().get(lastCommitTime)); assertEquals(new UUID[] { serviceId0, serviceId1 }, quorum1 .getVotes().get(lastCommitTime)); assertEquals(new UUID[] { serviceId0, serviceId1 }, quorum2 .getVotes().get(lastCommitTime)); // verify the consensus was NOT updated. assertEquals(lastCommitTime, client0.lastConsensusValue); assertEquals(lastCommitTime, client1.lastConsensusValue); assertEquals(lastCommitTime, client2.lastConsensusValue); } }); /* * Service join in the same order in which they cast their votes. */ assertCondition(new Runnable() { public void run() { assertEquals(new UUID[] { serviceId0, serviceId1 }, quorum0 .getJoined()); assertEquals(new UUID[] { serviceId0, serviceId1 }, quorum1 .getJoined()); assertEquals(new UUID[] { serviceId0, serviceId1 }, quorum2 .getJoined()); } }); // validate the token was NOT updated. assertEquals(token2, quorum0.lastValidToken()); assertEquals(token2, quorum1.lastValidToken()); assertEquals(token2, quorum2.lastValidToken()); assertEquals(token2, quorum0.token()); assertEquals(token2, quorum1.token()); assertEquals(token2, quorum2.token()); assertTrue(quorum0.isQuorumMet()); assertTrue(quorum1.isQuorumMet()); assertTrue(quorum2.isQuorumMet()); // The pipeline order is the same as the vote order. assertCondition(new Runnable() { public void run() { assertEquals(new UUID[] { serviceId0, serviceId1 }, quorum0 .getPipeline()); assertEquals(new UUID[] { serviceId0, serviceId1 }, quorum1 .getPipeline()); assertEquals(new UUID[] { serviceId0, serviceId1 }, quorum2 .getPipeline()); } }); /* * Fail the remaining follower. The quorum will break. */ actor1.serviceLeave(); fixture.awaitDeque(); assertCondition(new Runnable() { public void run() { // Services have voted for a single lastCommitTime. assertEquals(0, quorum0.getVotes().size()); /** * TODO The assert above occasionally fails with this trace. * * <pre> * junit.framework.AssertionFailedError: expected:<0> but was:<1> * at junit.framework.Assert.fail(Assert.java:47) * at junit.framework.Assert.failNotEquals(Assert.java:282) * at junit.framework.Assert.assertEquals(Assert.java:64) * at junit.framework.Assert.assertEquals(Assert.java:201) * at junit.framework.Assert.assertEquals(Assert.java:207) * at com.bigdata.quorum.TestHA3QuorumSemantics$19.run(TestHA3QuorumSemantics.java:1034) * at com.bigdata.quorum.AbstractQuorumTestCase.assertCondition(AbstractQuorumTestCase.java:184) * at com.bigdata.quorum.AbstractQuorumTestCase.assertCondition(AbstractQuorumTestCase.java:225) * at com.bigdata.quorum.TestHA3QuorumSemantics.test_serviceJoin3_simple(TestHA3QuorumSemantics.java:1031) * </pre> */ // verify the vote order. assertEquals(null, quorum0.getVotes().get(lastCommitTime)); // verify the consensus was cleared. assertEquals(-1L, client0.lastConsensusValue); assertEquals(-1L, client1.lastConsensusValue); assertEquals(-1L, client2.lastConsensusValue); // no services are joined. assertEquals(new UUID[] {}, quorum0.getJoined()); assertEquals(new UUID[] {}, quorum1.getJoined()); assertEquals(new UUID[] {}, quorum2.getJoined()); } }); quorum0.awaitBreak(); quorum1.awaitBreak(); quorum2.awaitBreak(); // validate the token was cleared. assertEquals(token2, quorum0.lastValidToken()); assertEquals(token2, quorum1.lastValidToken()); assertEquals(token2, quorum2.lastValidToken()); assertEquals(Quorum.NO_QUORUM, quorum0.token()); assertEquals(Quorum.NO_QUORUM, quorum1.token()); assertEquals(Quorum.NO_QUORUM, quorum2.token()); assertFalse(quorum0.isQuorumMet()); assertFalse(quorum1.isQuorumMet()); assertFalse(quorum2.isQuorumMet()); assertCondition(new Runnable() { public void run() { // Service leaves forced pipeline leaves. assertEquals(new UUID[] {}, quorum0.getPipeline()); assertEquals(new UUID[] {}, quorum1.getPipeline()); assertEquals(new UUID[] {}, quorum2.getPipeline()); } }); } } public void test_serviceJoin3_simpleForceRemove() throws InterruptedException { final Quorum<?, ?> quorum0 = quorums[0]; final MockQuorumMember<?> client0 = clients[0]; final QuorumActor<?, ?> actor0 = actors[0]; final UUID serviceId0 = client0.getServiceId(); final Quorum<?, ?> quorum1 = quorums[1]; final MockQuorumMember<?> client1 = clients[1]; final QuorumActor<?, ?> actor1 = actors[1]; final UUID serviceId1 = client1.getServiceId(); final Quorum<?, ?> quorum2 = quorums[2]; final MockQuorumMember<?> client2 = clients[2]; final QuorumActor<?,?> actor2 = actors[2]; final UUID serviceId2 = client2.getServiceId(); // final long lastCommitTime1 = 0L; // // final long lastCommitTime2 = 2L; final long lastCommitTime = 0L; // declare the services as a quorum members. actor0.memberAdd(); actor1.memberAdd(); actor2.memberAdd(); fixture.awaitDeque(); assertCondition(new Runnable() { public void run() { assertEquals(3, quorum0.getMembers().length); assertEquals(3, quorum1.getMembers().length); assertEquals(3, quorum2.getMembers().length); } }); /* * Have the services join the pipeline. */ actor0.pipelineAdd(); actor1.pipelineAdd(); actor2.pipelineAdd(); fixture.awaitDeque(); assertCondition(new Runnable() { public void run() { assertEquals(new UUID[] { serviceId0, serviceId1, serviceId2 }, quorum0.getPipeline()); assertEquals(new UUID[] { serviceId0, serviceId1, serviceId2 }, quorum1.getPipeline()); assertEquals(new UUID[] { serviceId0, serviceId1, serviceId2 }, quorum2.getPipeline()); } }); /* * Have two services cast a vote for a lastCommitTime. This will cause * the quorum to meet. */ final long token1; { actor0.castVote(lastCommitTime); actor1.castVote(lastCommitTime); fixture.awaitDeque(); // validate the token was assigned (must wait for meet). token1 = quorum0.awaitQuorum(); assertEquals(Quorum.NO_QUORUM + 1, token1); assertEquals(Quorum.NO_QUORUM + 1, quorum0.token()); assertEquals(Quorum.NO_QUORUM + 1, quorum0.lastValidToken()); assertTrue(quorum0.isQuorumMet()); // wait for meet for other clients. assertEquals(token1, quorum1.awaitQuorum()); assertEquals(token1, quorum2.awaitQuorum()); assertEquals(token1, quorum1.lastValidToken()); assertEquals(token1, quorum2.lastValidToken()); assertCondition(new Runnable() { public void run() { // services have voted for a single lastCommitTime. assertEquals(1, quorum0.getVotes().size()); assertEquals(1, quorum1.getVotes().size()); assertEquals(1, quorum2.getVotes().size()); // verify the vote order. assertEquals(new UUID[] { serviceId0, serviceId1 }, quorum0 .getVotes().get(lastCommitTime)); // verify the consensus was updated. assertEquals(lastCommitTime, client0.lastConsensusValue); assertEquals(lastCommitTime, client1.lastConsensusValue); assertEquals(lastCommitTime, client2.lastConsensusValue); /* * Service join in the same order in which they cast their * votes. */ assertEquals(new UUID[] { serviceId0, serviceId1 }, quorum0 .getJoined()); assertEquals(new UUID[] { serviceId0, serviceId1 }, quorum1 .getJoined()); assertEquals(new UUID[] { serviceId0, serviceId1 }, quorum2 .getJoined()); // The pipeline order is the same as the vote order. assertEquals(new UUID[] { serviceId0, serviceId1, serviceId2 }, quorum0.getPipeline()); assertEquals(new UUID[] { serviceId0, serviceId1, serviceId2 }, quorum1.getPipeline()); assertEquals(new UUID[] { serviceId0, serviceId1, serviceId2 }, quorum2.getPipeline()); } }); } /* * Cast the last vote and verify that the last service joins. * * Note: The last service should join immediately since it does not have * to do any validation when it joins. */ { actor2.castVote(lastCommitTime); fixture.awaitDeque(); assertCondition(new Runnable() { public void run() { // services have voted for a single lastCommitTime. assertEquals(1, quorum0.getVotes().size()); // verify the vote order. assertEquals(new UUID[] { serviceId0, serviceId1, serviceId2 }, quorum0.getVotes().get(lastCommitTime)); // verify the consensus was NOT updated. assertEquals(lastCommitTime, client0.lastConsensusValue); assertEquals(lastCommitTime, client1.lastConsensusValue); assertEquals(lastCommitTime, client2.lastConsensusValue); // Service join in the same order in which they cast their votes. assertEquals(new UUID[] { serviceId0, serviceId1, serviceId2 }, quorum0.getJoined()); assertEquals(new UUID[] { serviceId0, serviceId1, serviceId2 }, quorum1.getJoined()); assertEquals(new UUID[] { serviceId0, serviceId1, serviceId2 }, quorum2.getJoined()); } }); // validate the token was NOT updated. assertEquals(token1, quorum0.lastValidToken()); assertEquals(token1, quorum1.lastValidToken()); assertEquals(token1, quorum2.lastValidToken()); assertEquals(token1, quorum0.token()); assertEquals(token1, quorum1.token()); assertEquals(token1, quorum2.token()); assertTrue(quorum0.isQuorumMet()); assertTrue(quorum1.isQuorumMet()); assertTrue(quorum2.isQuorumMet()); // The pipeline order is the same as the vote order. assertEquals(new UUID[] { serviceId0, serviceId1, serviceId2 }, quorum0.getPipeline()); assertEquals(new UUID[] { serviceId0, serviceId1, serviceId2 }, quorum1.getPipeline()); assertEquals(new UUID[] { serviceId0, serviceId1, serviceId2 }, quorum2.getPipeline()); } /* * Follower leave/join test. */ { /* * Fail the first follower. This will not cause a quorum break since * there are still (k+1)/2 services in the quorum. */ // actor1.serviceLeave(); actor0.forceRemoveService(actor1.getServiceId()); fixture.awaitDeque(); assertCondition(new Runnable() { public void run() { // services have voted for a single lastCommitTime. assertEquals(1, quorum0.getVotes().size()); // verify the vote order. assertEquals(new UUID[] { serviceId0, serviceId2 }, quorum0 .getVotes().get(lastCommitTime)); // verify the consensus was NOT updated. assertEquals(lastCommitTime, client0.lastConsensusValue); assertEquals(lastCommitTime, client1.lastConsensusValue); assertEquals(lastCommitTime, client2.lastConsensusValue); /* * Service join in the same order in which they cast their * votes. */ assertEquals(new UUID[] { serviceId0, serviceId2 }, quorum0 .getJoined()); assertEquals(new UUID[] { serviceId0, serviceId2 }, quorum1 .getJoined()); assertEquals(new UUID[] { serviceId0, serviceId2 }, quorum2 .getJoined()); } }); // validate the token was NOT updated. assertEquals(token1, quorum0.lastValidToken()); assertEquals(token1, quorum1.lastValidToken()); assertEquals(token1, quorum2.lastValidToken()); assertEquals(token1, quorum0.token()); assertEquals(token1, quorum1.token()); assertEquals(token1, quorum2.token()); assertTrue(quorum0.isQuorumMet()); assertTrue(quorum1.isQuorumMet()); assertTrue(quorum2.isQuorumMet()); assertCondition(new Runnable() { public void run() { // The pipeline order is the same as the vote order. assertEquals(new UUID[] { serviceId0, serviceId2 }, quorum0 .getPipeline()); assertEquals(new UUID[] { serviceId0, serviceId2 }, quorum1 .getPipeline()); assertEquals(new UUID[] { serviceId0, serviceId2 }, quorum2 .getPipeline()); } }); /* * Rejoin the service. */ actor1.memberAdd(); fixture.awaitDeque(); actor1.pipelineAdd(); fixture.awaitDeque(); actor1.castVote(lastCommitTime); fixture.awaitDeque(); assertCondition(new Runnable() { public void run() { // services have voted for a single lastCommitTime. assertEquals(1, quorum0.getVotes().size()); assertEquals(1, quorum1.getVotes().size()); assertEquals(1, quorum2.getVotes().size()); // verify the vote order. assertEquals(new UUID[] { serviceId0, serviceId2, serviceId1 }, quorum0.getVotes() .get(lastCommitTime)); assertEquals(new UUID[] { serviceId0, serviceId2, serviceId1 }, quorum1.getVotes() .get(lastCommitTime)); assertEquals(new UUID[] { serviceId0, serviceId2, serviceId1 }, quorum2.getVotes() .get(lastCommitTime)); // verify the consensus was NOT updated. assertEquals(lastCommitTime, client0.lastConsensusValue); assertEquals(lastCommitTime, client1.lastConsensusValue); assertEquals(lastCommitTime, client2.lastConsensusValue); // Service join in the same order in which they cast their // votes. assertEquals(new UUID[] { serviceId0, serviceId2, serviceId1 }, quorum0.getJoined()); assertEquals(new UUID[] { serviceId0, serviceId2, serviceId1 }, quorum1.getJoined()); assertEquals(new UUID[] { serviceId0, serviceId2, serviceId1 }, quorum2.getJoined()); } }); // validate the token was NOT updated. assertEquals(token1, quorum0.lastValidToken()); assertEquals(token1, quorum1.lastValidToken()); assertEquals(token1, quorum2.lastValidToken()); assertEquals(token1, quorum0.token()); assertEquals(token1, quorum1.token()); assertEquals(token1, quorum2.token()); assertTrue(quorum0.isQuorumMet()); assertTrue(quorum1.isQuorumMet()); assertTrue(quorum2.isQuorumMet()); // The pipeline order is the same as the vote order. assertCondition(new Runnable() { public void run() { assertEquals(new UUID[] { serviceId0, serviceId2, serviceId1 }, quorum0.getPipeline()); assertEquals(new UUID[] { serviceId0, serviceId2, serviceId1 }, quorum1.getPipeline()); assertEquals(new UUID[] { serviceId0, serviceId2, serviceId1 }, quorum2.getPipeline()); } }); } /* * Leader leave test. * * This forces the quorum leader to do a serviceLeave(), which causes a * quorum break. All joined services should have left. Their votes were * withdrawn when they left and they were removed from the pipeline as * well. */ { actor0.serviceLeave(); fixture.awaitDeque(); // the votes were withdrawn. assertCondition(new Runnable() { public void run() { assertEquals(0, quorum0.getVotes().size()); assertEquals(0, quorum1.getVotes().size()); assertEquals(0, quorum2.getVotes().size()); } }); assertCondition(new Runnable() { public void run() { // the consensus was cleared. assertEquals(-1L, client0.lastConsensusValue); assertEquals(-1L, client1.lastConsensusValue); assertEquals(-1L, client2.lastConsensusValue); // No one is joined. assertEquals(new UUID[] {}, quorum0.getJoined()); assertEquals(new UUID[] {}, quorum1.getJoined()); assertEquals(new UUID[] {}, quorum2.getJoined()); // validate the token was cleared (lastValidToken is // unchanged). assertEquals(token1, quorum0.lastValidToken()); assertEquals(token1, quorum1.lastValidToken()); assertEquals(token1, quorum2.lastValidToken()); assertEquals(Quorum.NO_QUORUM, quorum0.token()); assertEquals(Quorum.NO_QUORUM, quorum1.token()); assertEquals(Quorum.NO_QUORUM, quorum2.token()); assertFalse(quorum0.isQuorumMet()); assertFalse(quorum1.isQuorumMet()); assertFalse(quorum2.isQuorumMet()); // No one is in the pipeline. assertEquals(new UUID[] {}, quorum0.getPipeline()); assertEquals(new UUID[] {}, quorum1.getPipeline()); assertEquals(new UUID[] {}, quorum2.getPipeline()); } }); } /* * Heal the quorum by rejoining all of the services. */ final long token2; { actor0.pipelineAdd(); actor1.pipelineAdd(); actor2.pipelineAdd(); fixture.awaitDeque(); actor0.castVote(lastCommitTime); actor1.castVote(lastCommitTime); actor2.castVote(lastCommitTime); fixture.awaitDeque(); assertCondition(new Runnable() { public void run() { // services have voted for a single lastCommitTime. assertEquals(1,quorum0.getVotes().size()); assertEquals(1,quorum1.getVotes().size()); assertEquals(1,quorum2.getVotes().size()); // verify the vote order. assertEquals(new UUID[] { serviceId0, serviceId1, serviceId2 }, quorum0.getVotes() .get(lastCommitTime)); assertEquals(new UUID[] { serviceId0, serviceId1, serviceId2 }, quorum1.getVotes() .get(lastCommitTime)); assertEquals(new UUID[] { serviceId0, serviceId1, serviceId2 }, quorum2.getVotes() .get(lastCommitTime)); // verify the consensus was updated. assertEquals(lastCommitTime, client0.lastConsensusValue); assertEquals(lastCommitTime, client1.lastConsensusValue); assertEquals(lastCommitTime, client2.lastConsensusValue); /* * Service join in the same order in which they cast their * votes. */ assertEquals(new UUID[] { serviceId0, serviceId1, serviceId2 }, quorum0.getJoined()); assertEquals(new UUID[] { serviceId0, serviceId1, serviceId2 }, quorum1.getJoined()); assertEquals(new UUID[] { serviceId0, serviceId1, serviceId2 }, quorum2.getJoined()); } }); // validate the token was updated. token2 = quorum0.awaitQuorum(); assertEquals(token2,quorum1.awaitQuorum()); assertEquals(token2,quorum2.awaitQuorum()); assertEquals(token2, quorum0.lastValidToken()); assertEquals(token2, quorum1.lastValidToken()); assertEquals(token2, quorum2.lastValidToken()); assertEquals(token2, quorum0.token()); assertEquals(token2, quorum1.token()); assertEquals(token2, quorum2.token()); assertTrue(quorum0.isQuorumMet()); assertTrue(quorum1.isQuorumMet()); assertTrue(quorum2.isQuorumMet()); // The pipeline order is the same as the vote order. assertCondition(new Runnable() { public void run() { assertEquals(new UUID[] { serviceId0, serviceId1, serviceId2 }, quorum0.getPipeline()); assertEquals(new UUID[] { serviceId0, serviceId1, serviceId2 }, quorum1.getPipeline()); assertEquals(new UUID[] { serviceId0, serviceId1, serviceId2 }, quorum2.getPipeline()); } }); } /* * Cause the quorum to break by failing both of the followers. */ { /* * Fail one follower. The quorum should not break. */ // actor2.serviceLeave(); actor0.forceRemoveService(actor2.getServiceId()); fixture.awaitDeque(); assertCondition(new Runnable() { public void run() { // services have voted for a single lastCommitTime. assertEquals(1, quorum0.getVotes().size()); assertEquals(1, quorum1.getVotes().size()); assertEquals(1, quorum2.getVotes().size()); // verify the vote order. assertEquals(new UUID[] { serviceId0, serviceId1 }, quorum0 .getVotes().get(lastCommitTime)); assertEquals(new UUID[] { serviceId0, serviceId1 }, quorum1 .getVotes().get(lastCommitTime)); assertEquals(new UUID[] { serviceId0, serviceId1 }, quorum2 .getVotes().get(lastCommitTime)); // verify the consensus was NOT updated. assertEquals(lastCommitTime, client0.lastConsensusValue); assertEquals(lastCommitTime, client1.lastConsensusValue); assertEquals(lastCommitTime, client2.lastConsensusValue); } }); /* * Service join in the same order in which they cast their votes. */ assertCondition(new Runnable() { public void run() { assertEquals(new UUID[] { serviceId0, serviceId1 }, quorum0 .getJoined()); assertEquals(new UUID[] { serviceId0, serviceId1 }, quorum1 .getJoined()); assertEquals(new UUID[] { serviceId0, serviceId1 }, quorum2 .getJoined()); } }); // validate the token was NOT updated. assertEquals(token2, quorum0.lastValidToken()); assertEquals(token2, quorum1.lastValidToken()); assertEquals(token2, quorum2.lastValidToken()); assertEquals(token2, quorum0.token()); assertEquals(token2, quorum1.token()); assertEquals(token2, quorum2.token()); assertTrue(quorum0.isQuorumMet()); assertTrue(quorum1.isQuorumMet()); assertTrue(quorum2.isQuorumMet()); // The pipeline order is the same as the vote order. assertCondition(new Runnable() { public void run() { assertEquals(new UUID[] { serviceId0, serviceId1 }, quorum0 .getPipeline()); assertEquals(new UUID[] { serviceId0, serviceId1 }, quorum1 .getPipeline()); assertEquals(new UUID[] { serviceId0, serviceId1 }, quorum2 .getPipeline()); } }); /* * Fail the remaining follower. The quorum will break. */ // actor1.serviceLeave(); actor0.forceRemoveService(actor1.getServiceId()); fixture.awaitDeque(); assertCondition(new Runnable() { public void run() { // Services have voted for a single lastCommitTime. assertEquals(0, quorum0.getVotes().size()); /** * TODO The assert above occasionally fails with this trace. * * <pre> * junit.framework.AssertionFailedError: expected:<0> but was:<1> * at junit.framework.Assert.fail(Assert.java:47) * at junit.framework.Assert.failNotEquals(Assert.java:282) * at junit.framework.Assert.assertEquals(Assert.java:64) * at junit.framework.Assert.assertEquals(Assert.java:201) * at junit.framework.Assert.assertEquals(Assert.java:207) * at com.bigdata.quorum.TestHA3QuorumSemantics$19.run(TestHA3QuorumSemantics.java:1034) * at com.bigdata.quorum.AbstractQuorumTestCase.assertCondition(AbstractQuorumTestCase.java:184) * at com.bigdata.quorum.AbstractQuorumTestCase.assertCondition(AbstractQuorumTestCase.java:225) * at com.bigdata.quorum.TestHA3QuorumSemantics.test_serviceJoin3_simple(TestHA3QuorumSemantics.java:1031) * </pre> */ // verify the vote order. assertEquals(null, quorum0.getVotes().get(lastCommitTime)); // verify the consensus was cleared. assertEquals(-1L, client0.lastConsensusValue); assertEquals(-1L, client1.lastConsensusValue); assertEquals(-1L, client2.lastConsensusValue); // no services are joined. assertEquals(new UUID[] {}, quorum0.getJoined()); assertEquals(new UUID[] {}, quorum1.getJoined()); assertEquals(new UUID[] {}, quorum2.getJoined()); } }); quorum0.awaitBreak(); quorum1.awaitBreak(); quorum2.awaitBreak(); // validate the token was cleared. assertEquals(token2, quorum0.lastValidToken()); assertEquals(token2, quorum1.lastValidToken()); assertEquals(token2, quorum2.lastValidToken()); assertEquals(Quorum.NO_QUORUM, quorum0.token()); assertEquals(Quorum.NO_QUORUM, quorum1.token()); assertEquals(Quorum.NO_QUORUM, quorum2.token()); assertFalse(quorum0.isQuorumMet()); assertFalse(quorum1.isQuorumMet()); assertFalse(quorum2.isQuorumMet()); assertCondition(new Runnable() { public void run() { // Service leaves forced pipeline leaves. assertEquals(new UUID[] {}, quorum0.getPipeline()); assertEquals(new UUID[] {}, quorum1.getPipeline()); assertEquals(new UUID[] {}, quorum2.getPipeline()); } }); } } /** * Unit tests for pipeline reorganization when the leader is elected. This * tests the automatic reorganization of the pipeline order where the * service which will become the leader is not at the head of the pipeline. * When the leader is elected, the leader must cause the pipeline to be * reorganized such that the leader is at the head of the pipeline. In * general, we can not control the pipeline order imposed by the leader when * it reorganizes the pipeline and the leader that was aware of the network * topology could chose to reorganize the pipeline in order to optimize it * even though the leader was already at the head of the pipeline. * * @throws InterruptedException */ public void test_pipelineReorganization() throws InterruptedException { final Quorum<?, ?> quorum0 = quorums[0]; final MockQuorumMember<?> client0 = clients[0]; final QuorumActor<?, ?> actor0 = actors[0]; final UUID serviceId0 = client0.getServiceId(); final Quorum<?, ?> quorum1 = quorums[1]; final MockQuorumMember<?> client1 = clients[1]; final QuorumActor<?, ?> actor1 = actors[1]; final UUID serviceId1 = client1.getServiceId(); final Quorum<?, ?> quorum2 = quorums[2]; final MockQuorumMember<?> client2 = clients[2]; final QuorumActor<?,?> actor2 = actors[2]; final UUID serviceId2 = client2.getServiceId(); final long lastCommitTime = 0L; // declare the services as a quorum members. actor0.memberAdd(); actor1.memberAdd(); actor2.memberAdd(); fixture.awaitDeque(); /* * Have the services join the pipeline a different order from the order * in which they will cast their votes. * * Note: Only two of the members are added to the pipeline so the * behavior when the leader reorganizes the pipeline will be * deterministic. */ // actor2.pipelineAdd(); actor1.pipelineAdd(); actor0.pipelineAdd(); fixture.awaitDeque(); /* * The service which we will cause to vote first (and hence will become * the leader) is NOT at the head of the pipeline. */ assertCondition(new Runnable() { public void run() { assertEquals(new UUID[] { serviceId1, serviceId0 }, quorum0 .getPipeline()); assertEquals(new UUID[] { serviceId1, serviceId0 }, quorum1 .getPipeline()); assertEquals(new UUID[] { serviceId1, serviceId0 }, quorum2 .getPipeline()); } }); /* * Have two services cast a vote for a lastCommitTime. This will cause * the quorum to meet. */ final long token1; { actor0.castVote(lastCommitTime); actor1.castVote(lastCommitTime); fixture.awaitDeque(); // validate the token was assigned. token1 = quorum0.awaitQuorum(); assertEquals(token1, quorum1.awaitQuorum()); assertEquals(token1, quorum2.awaitQuorum()); assertEquals(token1, quorum1.lastValidToken()); assertEquals(token1, quorum2.lastValidToken()); assertEquals(Quorum.NO_QUORUM + 1, quorum0.lastValidToken()); assertEquals(Quorum.NO_QUORUM + 1, quorum0.token()); assertTrue(quorum0.isQuorumMet()); assertCondition(new Runnable() { public void run() { // services have voted for a single lastCommitTime. assertEquals(1, quorum0.getVotes().size()); assertEquals(1, quorum1.getVotes().size()); assertEquals(1, quorum2.getVotes().size()); // verify the vote order. assertEquals(new UUID[] { serviceId0, serviceId1 }, quorum0 .getVotes().get(lastCommitTime)); assertEquals(new UUID[] { serviceId0, serviceId1 }, quorum1 .getVotes().get(lastCommitTime)); assertEquals(new UUID[] { serviceId0, serviceId1 }, quorum2 .getVotes().get(lastCommitTime)); // verify the consensus was updated. assertEquals(lastCommitTime, client0.lastConsensusValue); assertEquals(lastCommitTime, client1.lastConsensusValue); assertEquals(lastCommitTime, client2.lastConsensusValue); /* * Service join in the same order in which they cast their * votes. */ assertEquals(new UUID[] { serviceId0, serviceId1 }, quorum0 .getJoined()); assertEquals(new UUID[] { serviceId0, serviceId1 }, quorum1 .getJoined()); assertEquals(new UUID[] { serviceId0, serviceId1 }, quorum2 .getJoined()); // The leader is now at the front of the pipeline. assertEquals(new UUID[] { serviceId0, serviceId1 }, quorum0 .getPipeline()); assertEquals(new UUID[] { serviceId0, serviceId1 }, quorum1 .getPipeline()); assertEquals(new UUID[] { serviceId0, serviceId1 }, quorum2 .getPipeline()); } }); } } /** * Variations on standard pipelineReorganization to test conditions that * occasionally fail in TestHA3 * * @throws InterruptedException */ public void test_pipelineReorganization2() throws InterruptedException { final Quorum<?, ?> quorum0 = quorums[0]; final MockQuorumMember<?> client0 = clients[0]; final QuorumActor<?, ?> actor0 = actors[0]; final UUID serviceId0 = client0.getServiceId(); final Quorum<?, ?> quorum1 = quorums[1]; final MockQuorumMember<?> client1 = clients[1]; final QuorumActor<?, ?> actor1 = actors[1]; final UUID serviceId1 = client1.getServiceId(); final Quorum<?, ?> quorum2 = quorums[2]; final MockQuorumMember<?> client2 = clients[2]; final QuorumActor<?, ?> actor2 = actors[2]; final UUID serviceId2 = client2.getServiceId(); final long lastCommitTime = 0L; // declare the services as a quorum members. actor0.memberAdd(); actor1.memberAdd(); actor2.memberAdd(); /* * Have the services join the pipeline a different order from the order * in which they will cast their votes. */ actor2.pipelineAdd(); actor1.pipelineAdd(); actor0.pipelineAdd(); assertCondition(new Runnable() { public void run() { /* * The service which we will cause to vote first (and hence will * become the leader) is NOT at the head of the pipeline. */ assertEquals(new UUID[] { serviceId2, serviceId1, serviceId0 }, quorum0.getPipeline()); assertEquals(new UUID[] { serviceId2, serviceId1, serviceId0 }, quorum1.getPipeline()); assertEquals(new UUID[] { serviceId2, serviceId1, serviceId0 }, quorum2.getPipeline()); } }); /* * Have two services cast a vote for a lastCommitTime. This will cause * the quorum to meet. * * But actor2 should be moved to end of pipeline */ final long token1; { actor1.castVote(lastCommitTime); actor0.castVote(lastCommitTime); // validate the token was assigned. log.warn("Awaiting quorums to report met."); token1 = quorum0.awaitQuorum(); log.warn("Quorum0 reports met."); assertEquals(token1, quorum1.awaitQuorum()); log.warn("Quorum1 reports met."); assertEquals(token1, quorum2.awaitQuorum()); log.warn("Quorum2 reports met."); assertEquals(token1, quorum1.lastValidToken()); assertEquals(token1, quorum2.lastValidToken()); assertEquals(Quorum.NO_QUORUM + 1, quorum0.lastValidToken()); assertEquals(Quorum.NO_QUORUM + 1, quorum0.token()); assertTrue(quorum0.isQuorumMet()); assertCondition(new Runnable() { public void run() { // services have voted for a single lastCommitTime. assertEquals(1, quorum0.getVotes().size()); assertEquals(1, quorum1.getVotes().size()); assertEquals(1, quorum2.getVotes().size()); // verify the vote order. log.warn("expected: " + Arrays.toString(new UUID[] { serviceId1, serviceId0 })); log.warn("actual : " + Arrays.toString(quorum0.getVotes().get( lastCommitTime))); assertEquals(new UUID[] { serviceId1, serviceId0 }, quorum0 .getVotes().get(lastCommitTime)); assertEquals(new UUID[] { serviceId1, serviceId0 }, quorum1 .getVotes().get(lastCommitTime)); assertEquals(new UUID[] { serviceId1, serviceId0 }, quorum2 .getVotes().get(lastCommitTime)); // verify the consensus was updated. assertEquals(lastCommitTime, client0.lastConsensusValue); assertEquals(lastCommitTime, client1.lastConsensusValue); assertEquals(lastCommitTime, client2.lastConsensusValue); /* * Service join in the same order in which they cast their * votes. */ assertEquals(new UUID[] { serviceId1, serviceId0 }, quorum0.getJoined()); assertEquals(new UUID[] { serviceId1, serviceId0 }, quorum1.getJoined()); assertEquals(new UUID[] { serviceId1, serviceId0 }, quorum2.getJoined()); // The leader is now at the front of the pipeline, with // service2 moved to end. assertEquals(new UUID[] { serviceId1, serviceId0, serviceId2 }, quorum0.getPipeline()); assertEquals(new UUID[] { serviceId1, serviceId0, serviceId2 }, quorum1.getPipeline()); assertEquals(new UUID[] { serviceId1, serviceId0, serviceId2 }, quorum2.getPipeline()); } }); // check votes cast assertEquals(quorum1.getCastVote(serviceId1), quorum1.getCastVoteIfConsensus(serviceId0)); assertNotSame(quorum1.getCastVote(serviceId1), quorum1.getCastVote(serviceId2)); actor2.castVote(lastCommitTime); assertEquals(quorum1.getCastVoteIfConsensus(serviceId1), quorum1.getCastVoteIfConsensus(serviceId2)); Thread.sleep(100); final long token = quorum2.awaitQuorum(); assertTrue(quorum2.isQuorumFullyMet(token)); assertEquals(new UUID[] { serviceId1, serviceId0, serviceId2 }, quorum0.getJoined()); assertEquals(new UUID[] { serviceId1, serviceId0, serviceId2 }, quorum1.getJoined()); assertEquals(new UUID[] { serviceId1, serviceId0, serviceId2 }, quorum2.getJoined()); } } }