package org.infinispan.test.fwk;
import static org.infinispan.commons.util.Immutables.immutableMapCopy;
import static org.infinispan.test.fwk.JGroupsConfigBuilder.ProtocolType.FD;
import static org.infinispan.test.fwk.JGroupsConfigBuilder.ProtocolType.FD_ALL;
import static org.infinispan.test.fwk.JGroupsConfigBuilder.ProtocolType.FD_ALL2;
import static org.infinispan.test.fwk.JGroupsConfigBuilder.ProtocolType.FD_SOCK;
import static org.infinispan.test.fwk.JGroupsConfigBuilder.ProtocolType.MERGE3;
import static org.infinispan.test.fwk.JGroupsConfigBuilder.ProtocolType.RELAY2;
import static org.infinispan.test.fwk.JGroupsConfigBuilder.ProtocolType.TCP;
import static org.infinispan.test.fwk.JGroupsConfigBuilder.ProtocolType.TCP_NIO2;
import static org.infinispan.test.fwk.JGroupsConfigBuilder.ProtocolType.TEST_PING;
import static org.infinispan.test.fwk.JGroupsConfigBuilder.ProtocolType.UDP;
import static org.infinispan.test.fwk.JGroupsConfigBuilder.ProtocolType.VERIFY_SUSPECT;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import org.infinispan.commons.util.LegacyKeySupportSystemProperties;
import org.jgroups.conf.ConfiguratorFactory;
import org.jgroups.conf.ProtocolConfiguration;
import org.jgroups.conf.ProtocolStackConfigurator;
import org.jgroups.conf.XmlConfigurator;
/**
* This class owns the logic of associating network resources(i.e. ports) with threads, in order to make sure that there
* won't be any clashes between multiple clusters running in parallel on same host. Used for parallel test suite.
*
* @author Mircea.Markus@jboss.com
* @author Galder ZamarreƱo
*/
public class JGroupsConfigBuilder {
public static final String JGROUPS_STACK;
// Load the XML just once
private static final ProtocolStackConfigurator tcpConfigurator = loadTcp();
private static final ProtocolStackConfigurator udpConfigurator = loadUdp();
public static final int TCP_PORT_RANGE_PER_THREAD = 100;
private static final ThreadLocal<Integer> threadTcpStartPort = new ThreadLocal<Integer>() {
private final AtomicInteger uniqueAddr = new AtomicInteger(7900);
@Override
protected Integer initialValue() {
return uniqueAddr.getAndAdd(TCP_PORT_RANGE_PER_THREAD);
}
};
/**
* Holds unique mcast_addr for each thread used for JGroups channel construction.
*/
private static final ThreadLocal<String> threadMcastIP = new ThreadLocal<String>() {
private final AtomicInteger uniqueAddr = new AtomicInteger(11);
@Override
protected String initialValue() {
return "228.10.10." + uniqueAddr.getAndIncrement();
}
};
/**
* Holds unique mcast_port for each thread used for JGroups channel construction.
*/
private static final ThreadLocal<Integer> threadMcastPort = new ThreadLocal<Integer>() {
private final AtomicInteger uniquePort = new AtomicInteger(45589);
@Override
protected Integer initialValue() {
return uniquePort.getAndIncrement();
}
};
static {
JGROUPS_STACK = LegacyKeySupportSystemProperties.getProperty("infinispan.test.jgroups.protocol", "protocol.stack", "tcp");
System.out.println("Transport protocol stack used = " + JGROUPS_STACK);
}
public static String getJGroupsConfig(String fullTestName, TransportFlags flags) {
if (JGROUPS_STACK.equalsIgnoreCase("tcp")) return getTcpConfig(fullTestName, flags);
if (JGROUPS_STACK.equalsIgnoreCase("udp")) return getUdpConfig(fullTestName, flags);
throw new IllegalStateException("Unknown protocol stack : " + JGROUPS_STACK);
}
public static String getTcpConfig(String fullTestName, TransportFlags flags) {
// With the XML already parsed, make a safe copy of the
// protocol stack configurator and use that accordingly.
JGroupsProtocolCfg jgroupsCfg =
getJGroupsProtocolCfg(tcpConfigurator.getProtocolStack());
if (!flags.withFD())
removeFailureDetection(jgroupsCfg);
if (!flags.isRelayRequired()) {
removeRelay2(jgroupsCfg);
} else {
ProtocolConfiguration protocol = jgroupsCfg.getProtocol(RELAY2);
protocol.getProperties().put("site", flags.siteName());
if (flags.relayConfig() != null) //if not specified, use default
protocol.getProperties().put("config", flags.relayConfig());
}
if (!flags.withMerge())
removeMerge(jgroupsCfg);
if (jgroupsCfg.containsProtocol(TEST_PING)) {
replaceTcpStartPort(jgroupsCfg, flags);
if (fullTestName == null)
return jgroupsCfg.toString(); // IDE run of test
else
return getTestPingDiscovery(fullTestName, jgroupsCfg); // Cmd line test run
} else {
return replaceMCastAddressAndPort(jgroupsCfg);
}
}
private static void removeMerge(JGroupsProtocolCfg jgroupsCfg) {
jgroupsCfg.removeProtocol(MERGE3);
}
public static String getUdpConfig(String fullTestName, TransportFlags flags) {
JGroupsProtocolCfg jgroupsCfg = getJGroupsProtocolCfg(udpConfigurator.getProtocolStack());
if (!flags.withFD())
removeFailureDetection(jgroupsCfg);
if (!flags.withMerge())
removeMerge(jgroupsCfg);
if (!flags.isRelayRequired()) {
removeRelay2(jgroupsCfg);
}
if (jgroupsCfg.containsProtocol(TEST_PING)) {
if (fullTestName != null)
return getTestPingDiscovery(fullTestName, jgroupsCfg); // Cmd line test run
}
return replaceMCastAddressAndPort(jgroupsCfg);
}
/**
* Remove all failure detection related
* protocols from the given JGroups stack.
*/
private static void removeFailureDetection(JGroupsProtocolCfg jgroupsCfg) {
jgroupsCfg.removeProtocol(FD).removeProtocol(FD_SOCK).removeProtocol(FD_ALL).removeProtocol(FD_ALL2)
.removeProtocol(VERIFY_SUSPECT);
}
private static void removeRelay2(JGroupsProtocolCfg jgroupsCfg) {
jgroupsCfg.removeProtocol(RELAY2);
}
private static String getTestPingDiscovery(String fullTestName, JGroupsProtocolCfg jgroupsCfg) {
ProtocolType type = TEST_PING;
Map<String, String> props = jgroupsCfg.getProtocol(type).getProperties();
props.put("testName", fullTestName);
return replaceProperties(jgroupsCfg, props, type);
}
private static String replaceMCastAddressAndPort(JGroupsProtocolCfg jgroupsCfg) {
ProtocolConfiguration udp = jgroupsCfg.getProtocol(UDP);
if (udp == null) return jgroupsCfg.toString();
Map<String, String> props = udp.getProperties();
props.put("mcast_addr", threadMcastIP.get());
props.put("mcast_port", threadMcastPort.get().toString());
return replaceProperties(jgroupsCfg, props, UDP);
}
private static String replaceTcpStartPort(JGroupsProtocolCfg jgroupsCfg, TransportFlags transportFlags) {
ProtocolType transportProtocol = jgroupsCfg.containsProtocol(TCP_NIO2) ? TCP_NIO2 : TCP;
Map<String, String> props = jgroupsCfg.getProtocol(transportProtocol).getProperties();
Integer startPort = threadTcpStartPort.get();
int portRange = TCP_PORT_RANGE_PER_THREAD;
if (transportFlags.isPortRangeSpecified()) {
portRange = 10;
int maxIndex = TCP_PORT_RANGE_PER_THREAD / portRange - 1;
if (transportFlags.portRange() > maxIndex) {
throw new IllegalStateException("Currently we only support " + (maxIndex + 1) + " ranges/sites!");
}
startPort += transportFlags.portRange() * portRange;
}
props.put("bind_port", startPort.toString());
// In JGroups, the port_range is inclusive
props.put("port_range", String.valueOf(portRange - 1));
return replaceProperties(jgroupsCfg, props, transportProtocol);
}
private static String replaceProperties(
JGroupsProtocolCfg cfg, Map<String, String> newProps, ProtocolType type) {
ProtocolConfiguration protocol = cfg.getProtocol(type);
ProtocolConfiguration newProtocol =
new ProtocolConfiguration(protocol.getProtocolName(), newProps);
cfg.replaceProtocol(type, newProtocol);
return cfg.toString();
}
private static ProtocolStackConfigurator loadTcp() {
try {
return ConfiguratorFactory.getStackConfigurator("stacks/tcp.xml");
} catch (Exception e) {
throw new RuntimeException(e);
}
}
private static ProtocolStackConfigurator loadUdp() {
try {
return ConfiguratorFactory.getStackConfigurator("stacks/udp.xml");
} catch (Exception e) {
throw new RuntimeException(e);
}
}
private static JGroupsProtocolCfg getJGroupsProtocolCfg(List<ProtocolConfiguration> baseStack) {
JGroupsXmxlConfigurator configurator = new JGroupsXmxlConfigurator(baseStack);
List<ProtocolConfiguration> protoStack = configurator.getProtocolStack();
Map<ProtocolType, ProtocolConfiguration> protoMap = new HashMap<ProtocolType, ProtocolConfiguration>(protoStack.size());
for (ProtocolConfiguration cfg : protoStack)
protoMap.put(getProtocolType(cfg.getProtocolName()), cfg);
return new JGroupsProtocolCfg(protoMap, configurator);
}
private static ProtocolType getProtocolType(String name) {
int dotIndex = name.lastIndexOf(".");
return ProtocolType.valueOf(
dotIndex == -1 ? name : name.substring(dotIndex + 1, name.length()));
}
static class JGroupsXmxlConfigurator extends XmlConfigurator {
protected JGroupsXmxlConfigurator(List<ProtocolConfiguration> protocols) {
super(copy(protocols));
}
static List<ProtocolConfiguration> copy(List<ProtocolConfiguration> protocols) {
// Make a safe copy of the protocol stack to avoid concurrent modification issues
List<ProtocolConfiguration> copy =
new ArrayList<ProtocolConfiguration>(protocols.size());
for (ProtocolConfiguration p : protocols)
copy.add(new ProtocolConfiguration(
p.getProtocolName(), immutableMapCopy(p.getProperties())));
return copy;
}
}
static class JGroupsProtocolCfg {
final Map<ProtocolType, ProtocolConfiguration> protoMap;
final XmlConfigurator configurator;
JGroupsProtocolCfg(Map<ProtocolType, ProtocolConfiguration> protoMap,
XmlConfigurator configurator) {
this.protoMap = protoMap;
this.configurator = configurator;
}
JGroupsProtocolCfg addProtocol(ProtocolType type, ProtocolConfiguration cfg, int position) {
protoMap.put(type, cfg);
configurator.getProtocolStack().add(position, cfg);
return this;
}
JGroupsProtocolCfg removeProtocol(ProtocolType type) {
// Update the stack and map
configurator.getProtocolStack().remove(protoMap.remove(type));
return this;
}
ProtocolConfiguration getProtocol(ProtocolType type) {
return protoMap.get(type);
}
boolean containsProtocol(ProtocolType type) {
return getProtocol(type) != null;
}
JGroupsProtocolCfg replaceProtocol(ProtocolType type, ProtocolConfiguration newCfg) {
ProtocolConfiguration oldCfg = protoMap.get(type);
int position = configurator.getProtocolStack().indexOf(oldCfg);
// Remove protocol and put new configuration in same position
return removeProtocol(type).addProtocol(type, newCfg, position);
}
@Override
public String toString() {
return configurator.getProtocolStackString(true);
}
}
enum ProtocolType {
TCP, TCP_NIO2, UDP, SHARED_LOOPBACK,
MPING, PING, TCPPING, TEST_PING, SHARED_LOOPBACK_PING,
MERGE2, MERGE3,
FD_SOCK, FD, VERIFY_SUSPECT, FD_ALL, FD_ALL2,
BARRIER,
UNICAST, UNICAST2, UNICAST3,
NAKACK, NAKACK2,
RSVP,
STABLE,
GMS,
UFC, MFC, FC,
FRAG2, FRAG3,
STREAMING_STATE_TRANSFER,
RELAY2,
TOA
}
}