package org.jgroups.tests; import org.jgroups.*; import org.jgroups.protocols.*; import org.jgroups.protocols.pbcast.FLUSH; import org.jgroups.protocols.pbcast.GMS; import org.jgroups.protocols.pbcast.NAKACK2; import org.jgroups.protocols.pbcast.STABLE; import org.jgroups.stack.Protocol; import org.jgroups.stack.ProtocolStack; import org.jgroups.util.Util; import org.testng.Assert; import org.testng.annotations.AfterMethod; import org.testng.annotations.Test; import java.io.*; import java.util.*; /** * Various tests for the FLUSH protocol * @author Bela Ban */ @Test(groups={Global.FLUSH,Global.EAP_EXCLUDED},singleThreaded=true) public class ReconciliationTest { protected List<JChannel> channels; protected List<MyReceiver> receivers; @AfterMethod void tearDown() throws Exception { if(channels != null) channels.forEach(Util::close); } /** * Test scenario: * <ul> * <li>3 members: A,B,C * <li>All members have DISCARD which does <em>not</em> discard any * messages ! * <li>B (in DISCARD) ignores all messages from C * <li>C multicasts 5 messages to the cluster, A and C receive them * <li>New member D joins * <li>Before installing view {A,B,C,D}, FLUSH updates B with all of C's 5 * messages * </ul> */ public void testReconciliationFlushTriggeredByNewMemberJoin() throws Exception { FlushTrigger t=() -> { System.out.println("Joining D, this will trigger FLUSH and a subsequent view change to {A,B,C,D}"); JChannel newChannel; try { newChannel=createChannel("X"); newChannel.connect("ReconciliationTest"); channels.add(newChannel); } catch(Exception e) { e.printStackTrace(); } }; reconciliationHelper(new String[]{"A", "B", "C"}, t); } /** * Test scenario: * <ul> * <li>3 members: A,B,C * <li>All members have DISCARD which does <em>not</em> discard any * messages ! * <li>B (in DISCARD) ignores all messages from C * <li>C multicasts 5 messages to the cluster, A and C receive them * <li>A then runs a manual flush by calling Channel.start/stopFlush() * <li>Before installing view {A,B}, FLUSH makes A sends its 5 messages * received from C to B * </ul> */ public void testReconciliationFlushTriggeredByManualFlush() throws Exception { FlushTrigger t=() -> { JChannel channel=channels.get(0); boolean rc=Util.startFlush(channel); System.out.println("manual flush success=" + rc); channel.stopFlush(); }; String apps[]={"A", "B", "C"}; reconciliationHelper(apps, t); } /** * Test scenario: * <ul> * <li>3 members: A,B,C * <li>All members have DISCARD which does <em>not</em> discard any * messages ! * <li>B (in DISCARD) ignores all messages from C * <li>C multicasts 5 messages to the cluster, A and C receive them * <li>C then 'crashes' (Channel.shutdown()) * <li>Before installing view {A,B}, FLUSH makes A sends its 5 messages * received from C to B * </ul> */ public void testReconciliationFlushTriggeredByMemberCrashing() throws Exception { FlushTrigger t=() -> { JChannel channel=channels.remove(channels.size() - 1); try { Util.shutdown(channel); } catch(Exception e) { e.printStackTrace(); } }; String apps[]={"A", "B", "C"}; reconciliationHelper(apps, t); } /** * Tests reconciliation. Creates N channels, based on 'names'. Say we have A, B and C. Then we have the second but * last node (B) discard all messages from the last node (C). Then the last node (C) multicasts 5 messages. We check * that the 5 messages have been received correctly by all nodes but the second-but-last node (B). Then we remove * DISCARD from B and trigger a manual flush. After the flush, B should also have received the 5 messages sent * by C. */ protected void reconciliationHelper(String[] names, FlushTrigger ft) throws Exception { // create channels and setup receivers int channelCount=names.length; channels=new ArrayList<>(names.length); receivers=new ArrayList<>(names.length); for(int i=0;i < channelCount;i++) { JChannel channel=createChannel(names[i]); modifyNAKACK(channel); MyReceiver r=new MyReceiver(channel, names[i]); receivers.add(r); channels.add(channel); channel.setReceiver(r); channel.connect("ReconciliationTest"); Util.sleep(i == 0? 1000 : 250); } View view=channels.get(channels.size() -1).getView(); System.out.println("view: " + view); assert view.size() == channels.size(); JChannel last=channels.get(channels.size() - 1); JChannel nextToLast=channels.get(channels.size() - 2); System.out.println(nextToLast.getAddress() + " is now discarding messages from " + last.getAddress()); insertDISCARD(nextToLast, last.getAddress()); String lastsName=names[names.length - 1]; String nextToLastName=names[names.length - 2]; printDigests(channels, "\nDigests before " + lastsName + " sends any messages:"); // now last sends 5 messages: System.out.println("\n" + lastsName + " sending 5 messages; " + nextToLastName + " will ignore them, but others will receive them"); for(int i=1;i <= 5;i++) last.send(null, i); Util.sleep(1000); // until al messages have been received, this is asynchronous so we need to wait a bit printDigests(channels, "\nDigests after " + lastsName + " sent messages:"); MyReceiver lastReceiver=receivers.get(receivers.size() - 1); MyReceiver nextToLastReceiver=receivers.get(receivers.size() - 2); // check last (must have received its own messages) Map<Address,List<Integer>> map=lastReceiver.getMsgs(); Assert.assertEquals(map.size(), 1, "we should have only 1 sender, namely C at this time"); List<Integer> list=map.get(last.getAddress()); System.out.println("\n" + lastsName + ": messages received from " + lastsName + ": " + list); Assert.assertEquals(list.size(), 5, "correct msgs: " + list); // check nextToLast (should have received none of last messages) map=nextToLastReceiver.getMsgs(); Assert.assertEquals(map.size(), 0, "we should have no sender at this time"); list=map.get(last.getAddress()); System.out.println(nextToLastName + ": messages received from " + lastsName + ": " + list); assert list == null; List<MyReceiver> otherReceivers=receivers.subList(0, receivers.size() - 2); // check other (should have received last's messages) for(MyReceiver receiver:otherReceivers) { map=receiver.getMsgs(); Assert.assertEquals(map.size(), 1, "we should have only 1 sender"); list=map.get(last.getAddress()); System.out.println(receiver.name + ": messages received from " + lastsName + ": " + list); Assert.assertEquals(list.size(), 5, "correct msgs" + list); } removeDISCARD(nextToLast); Address address=last.getAddress(); ft.triggerFlush(); int cnt=20; View v; while((v=channels.get(0).getView()) != null && cnt > 0) { cnt--; if(v.size() == channels.size()) break; Util.sleep(1000); } assert channels.get(0).getView().size() == channels.size(); printDigests(channels, "\nDigests after reconciliation (B should have received the 5 messages from B now):"); // check that member with discard (should have received all missing // messages map=nextToLastReceiver.getMsgs(); Assert.assertEquals(map.size(), 1, "we should have 1 sender at this time"); list=map.get(address); System.out.println("\n" + nextToLastName + ": messages received from " + lastsName + " : " + list); Assert.assertEquals(5, list.size()); } protected JChannel createChannel(String name) throws Exception { Protocol[] protocols={ new SHARED_LOOPBACK(), new SHARED_LOOPBACK_PING(), new FD_ALL().setValue("timeout", 3000).setValue("interval", 1000), new NAKACK2(), new UNICAST3(), new STABLE(), new GMS(), new FRAG2().fragSize(8000), new FLUSH() }; return new JChannel(protocols).name(name); } /** Sets discard_delivered_msgs to false */ protected void modifyNAKACK(JChannel ch) { if(ch == null) return; NAKACK2 nakack=(NAKACK2)ch.getProtocolStack().findProtocol(NAKACK2.class); if(nakack != null) nakack.setDiscardDeliveredMsgs(false); } private static void printDigests(List<JChannel> channels, String message) { System.out.println(message); for(JChannel channel:channels) { System.out.println("[" + channel.getAddress() + "] " + channel.down(Event.GET_DIGEST_EVT).toString()); } } private static void insertDISCARD(JChannel ch, Address exclude) throws Exception { DISCARD discard=new DISCARD().localAddress(ch.getAddress()); discard.setExcludeItself(true); discard.addIgnoreMember(exclude); // ignore messages from this member ch.getProtocolStack().insertProtocol(discard, ProtocolStack.Position.BELOW, NAKACK2.class); } private static void removeDISCARD(JChannel...channels) throws Exception { for(JChannel ch:channels) ch.getProtocolStack().removeProtocol(DISCARD.class); } private interface FlushTrigger { void triggerFlush(); } protected static class MyReceiver extends ReceiverAdapter { protected final Map<Address,List<Integer>> msgs=new HashMap<>(10); protected final JChannel channel; protected final String name; public MyReceiver(JChannel ch, String name) { this.channel=ch; this.name=name; } public Map<Address,List<Integer>> getMsgs() {return msgs;} public void reset() {msgs.clear();} public void receive(Message msg) { List<Integer> list=msgs.get(msg.getSrc()); if(list == null) { list=new ArrayList<>(); msgs.put(msg.getSrc(), list); } list.add((Integer)msg.getObject()); System.out.println(name + ": <-- " + msg.getObject() + " from " + msg.getSrc()); } } public void testVirtualSynchrony() throws Exception { JChannel a = createChannel("A"); Cache cache_1 = new Cache(a, "cache-1"); a.connect("testVirtualSynchrony"); JChannel b = createChannel("B"); Cache cache_2 = new Cache(b, "cache-2"); b.connect("testVirtualSynchrony"); Util.waitUntilAllChannelsHaveSameView(10000, 500, a, b); // start adding messages flush(a); // flush all pending message out of the system so everyone receives them for(int i = 1; i <= 20;i++) { if(i % 2 == 0) cache_1.put(i, true); // even numbers else cache_2.put(i, true); // odd numbers } System.out.println("Starting flush on C1"); flush(a); System.out.println("Starting flush on C2"); flush(b); System.out.println("flush done"); System.out.println("cache_1 (" + cache_1.size() + " elements): " + cache_1 + "\ncache_2 (" + cache_2.size() + " elements): " + cache_2); Assert.assertEquals(cache_1.size(), 20, "cache 1: " + cache_1); Assert.assertEquals(cache_2.size(), 20, "cache 2: " + cache_2); Util.close(b,a); } protected static void flush(JChannel channel) { try { assert Util.startFlush(channel); } finally { channel.stopFlush(); } } protected static class Cache extends ReceiverAdapter { protected final Map<Object,Object> data; protected JChannel ch; protected String name; public Cache(JChannel ch, String name) { this.data=new HashMap<>(); this.ch=ch; this.name=name; this.ch.setReceiver(this); } protected Object get(Object key) { synchronized(data) { return data.get(key); } } protected void put(Object key, Object val) throws Exception { ch.send(new Message(null, new Object[]{key,val})); } protected int size() { synchronized(data) { return data.size(); } } public void receive(Message msg) { Object[] modification=(Object[])msg.getObject(); synchronized(data) { data.put(modification[0], modification[1]); } } public void getState(OutputStream ostream) throws Exception { synchronized(data) { Util.objectToStream(data, new DataOutputStream(ostream)); } } @SuppressWarnings("unchecked") public void setState(InputStream istream) throws Exception { Map<Object,Object> m=(Map<Object,Object>)Util.objectFromStream(new DataInputStream(istream)); synchronized(data) { data.clear(); data.putAll(m); } } public String toString() { synchronized(data) { TreeMap<Object,Object> map=new TreeMap<>(data); return map.keySet().toString(); } } } }