RandomConnectionStrategyTest.java

/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 *  Unless required by applicable law or agreed to in writing, software
 *  distributed under the License is distributed on an "AS IS" BASIS,
 *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 *  See the License for the specific language governing permissions and
 *  limitations under the License.
 */
package org.apache.openejb.itest.failover;

import org.apache.openejb.client.Client;
import org.apache.openejb.client.RemoteInitialContextFactory;
import org.apache.openejb.client.event.ClusterMetaDataUpdated;
import org.apache.openejb.client.event.Observes;
import org.apache.openejb.itest.failover.ejb.Calculator;
import org.apache.openejb.loader.Files;
import org.apache.openejb.loader.IO;
import org.apache.openejb.loader.Zips;
import org.apache.openejb.util.OpenEjbVersion;
import org.apache.openejb.server.control.StandaloneServer;
import org.junit.Assert;
import org.junit.Test;

import javax.ejb.EJBException;
import javax.naming.Context;
import javax.naming.InitialContext;
import java.io.File;
import java.net.URI;
import java.util.Arrays;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.locks.Condition;
import java.util.concurrent.locks.ReentrantLock;
import java.util.logging.ConsoleHandler;
import java.util.logging.Level;
import java.util.logging.Logger;

import static org.apache.openejb.util.NetworkUtil.getNextAvailablePort;

public class RandomConnectionStrategyTest {

    static final Logger logger = Logger.getLogger("org.apache.openejb.client");

    private static final int CLIENT_DELAY = 60;

    static {
        final ConsoleHandler consoleHandler = new ConsoleHandler();
        consoleHandler.setLevel(Level.FINER);
        logger.addHandler(consoleHandler);
        logger.setLevel(Level.FINER);
        logger.setUseParentHandlers(false);
    }

    @Test
    public void test() throws Exception {

//        To run in an IDE, uncomment and update this line
//        System.setProperty("version", OpenEjbVersion.get().getVersion());
        System.setProperty("openejb.client.connection.strategy", "roundrobin");

        final File zip = Repository.getArtifact("org.apache.tomee", "openejb-standalone", "zip");
        final File app = Repository.getArtifact("org.apache.openejb.itests", "failover-ejb", "jar");

        final File dir = Files.tmpdir();

        final StandaloneServer root;
        {
            final String name = "root";
            final File home = new File(dir, name);

            Files.mkdir(home);
            Zips.unzip(zip, home, true);

            root = new StandaloneServer(home, home);
            root.killOnExit();
            root.getJvmOpts().add("-Dopenejb.classloader.forced-load=org.apache.openejb");
            root.ignoreOut();
            root.setProperty("name", name);
            root.setProperty("openejb.extract.configuration", "false");

            final StandaloneServer.ServerService multipoint = root.getServerService("multipoint");
            multipoint.setBind("localhost");
            multipoint.setPort(getNextAvailablePort());
            multipoint.setDisabled(false);
            multipoint.set("discoveryName", name);

            logger.info("Starting Root server");

            // Wait for it to start before continuing, otherwise test may fail in slower machines.
            root.start(1, TimeUnit.MINUTES);
        }

        final Services services = new Services();
        Client.addEventObserver(services);

        final Map<String, StandaloneServer> servers = new HashMap<String, StandaloneServer>();
        for (final String name : new String[]{"red", "green", "blue"}) {

            final File home = new File(dir, name);
            Files.mkdir(home);
            Zips.unzip(zip, home, true);

            final StandaloneServer server = new StandaloneServer(home, home);
            server.killOnExit();
            server.ignoreOut();
            server.setProperty("name", name);
            server.setProperty("openejb.extract.configuration", "false");
            server.getJvmOpts().add("-Dopenejb.classloader.forced-load=org.apache.openejb");

            IO.copy(app, Files.path(home, "apps", "itest.jar"));
            IO.copy(IO.read("<openejb><Deployments dir=\"apps/\"/></openejb>"), Files.path(home, "conf", "openejb.xml"));

            final StandaloneServer.ServerService ejbd = server.getServerService("ejbd");
            ejbd.setBind("localhost");
            ejbd.setDisabled(false);
            ejbd.setPort(getNextAvailablePort());
            ejbd.setThreads(5);

            final URI uri = URI.create(String.format("ejbd://%s:%s/%s", ejbd.getBind(), ejbd.getPort(), name));
            ejbd.set("discovery", "ejb:" + uri);
            services.add(uri);
            server.getContext().set(URI.class, uri);

            final StandaloneServer.ServerService multipoint = server.getServerService("multipoint");
            multipoint.setPort(getNextAvailablePort());
            multipoint.setDisabled(false);
            multipoint.set("discoveryName", name);
            multipoint.set("initialServers", "localhost:" + root.getServerService("multipoint").getPort());

            servers.put(name, server);

            logger.info(String.format("Starting %s server", name));

            server.start(1, TimeUnit.MINUTES);
        }


        System.setProperty("openejb.client.requestretry", "true");
        System.setProperty("openejb.client.connection.strategy", "random");

        logger.info("Beginning Test");

        final Properties environment = new Properties();
        environment.put(Context.INITIAL_CONTEXT_FACTORY, RemoteInitialContextFactory.class.getName());
        environment.put(Context.PROVIDER_URL, "ejbd://localhost:" + servers.values().iterator().next().getServerService("ejbd").getPort() + "/provider");

        final InitialContext context = new InitialContext(environment);
        final Calculator bean = (Calculator) context.lookup("CalculatorBeanRemote");

        for (final Map.Entry<String, StandaloneServer> entry : servers.entrySet()) {
            final String name = entry.getKey();
            final StandaloneServer server = entry.getValue();
            final URI serverURI = server.getContext().get(URI.class);

            logger.info("Waiting for updated list");
            services.assertServices(CLIENT_DELAY, TimeUnit.SECONDS, new CalculatorCallable(bean), 500);

            logger.info("Asserting balance");
            assertBalance(bean, services.get().size());

            logger.info("Shutting down " + name);
            server.kill();
            services.remove(serverURI);
        }

        logger.info("All Servers Shutdown");

        try {
            logger.info("Making one last request, expecting complete failover");

            final String name = bean.name();
            Assert.fail("Server should be destroyed: " + name);
        } catch (final EJBException e) {
            logger.info(String.format("Pass.  Request resulted in %s: %s", e.getCause().getClass().getSimpleName(), e.getMessage()));
            // good
        }


        for (final Map.Entry<String, StandaloneServer> entry : servers.entrySet()) {
            final String name = entry.getKey();
            final StandaloneServer server = entry.getValue();
            final URI serverURI = server.getContext().get(URI.class);

            logger.info(String.format("Starting %s server", name));

            server.start(1, TimeUnit.MINUTES);
            services.add(serverURI);

            logger.info("Waiting for updated list");
            services.assertServices(CLIENT_DELAY, TimeUnit.SECONDS, new CalculatorCallable(bean), 500);

            logger.info("Asserting balance");
            assertBalance(bean, services.get().size());
        }
    }

    private void assertBalance(final Calculator bean, final int size) {
        final int expectedInvocations = 1000;
        final double percent = 0.10;
        final int totalInvocations = size * expectedInvocations;

        // Verify the work reached all servers
        final Set<Map.Entry<String, AtomicInteger>> entries = invoke(bean, totalInvocations).entrySet();

        Assert.assertEquals("Bad number of invocations for the bean \"" + bean.name() + "\".", size, entries.size());

        // And each server got a minimum of %10 percent of the traffic
        for (final Map.Entry<String, AtomicInteger> entry : entries) {

            final int actualInvocations = entry.getValue().get();

            Assert.assertTrue(String.format("%s out of %s is too low", actualInvocations, expectedInvocations), actualInvocations > expectedInvocations * percent);
        }
    }


    private Map<String, AtomicInteger> invoke(final Calculator bean, final int max) {
        final Map<String, AtomicInteger> invocations = new HashMap<String, AtomicInteger>();
        for (int i = 0; i < max; i++) {
            final String name = bean.name();

            if (!invocations.containsKey(name)) {
                invocations.put(name, new AtomicInteger());
                logger.info(String.format("Tracking new Server %s", name));
            }
            invocations.get(name).incrementAndGet();
        }

        for (final Map.Entry<String, AtomicInteger> entry : invocations.entrySet()) {
            logger.info(String.format("Server %s invoked %s times", entry.getKey(), entry.getValue()));
        }

        return invocations;
    }

    public static class Services {
        static final Logger logger = Logger.getLogger(Services.class.getName());

        private final ReentrantLock lock = new ReentrantLock();
        private final Condition condition = lock.newCondition();

        private final Set<URI> expected = new HashSet<URI>();

        public Services() {
        }

        public Set<URI> get() {
            return expected;
        }

        public boolean add(final URI uri) {
            return expected.add(uri);
        }

        public boolean remove(final URI o) {
            return expected.remove(o);
        }

        public void observe(@Observes final ClusterMetaDataUpdated updated) {
            final URI[] locations = updated.getClusterMetaData().getLocations();
            final Set<URI> found = new HashSet<URI>(Arrays.asList(locations));

            if (expected.equals(found)) {
                lock.lock();
                try {
                    condition.signal();
                } finally {
                    lock.unlock();
                }
            }
        }

        public Set<URI> diff(final Set<URI> a, final Set<URI> b) {
            final Set<URI> diffs = new HashSet<URI>();
            for (final URI uri : b) {
                if (!a.contains(uri)) diffs.add(uri);
            }

            return diffs;
        }

        public void assertServices(final long timeout, final TimeUnit unit, final Callable callable) {
            assertServices(timeout, unit, callable, 10);
        }

        public void assertServices(final long timeout, final TimeUnit unit, final Callable callable, final int delay) {
            final ClientThread client = new ClientThread(callable);
            client.delay(delay);
            client.start();
            try {
                Assert.assertTrue(String.format("services failed to come online: waited %s %s", timeout, unit), await(timeout, unit));
            } catch (final InterruptedException e) {
                Thread.interrupted();
                Assert.fail("Interrupted");
            } finally {
                client.stop();
            }
        }

        public boolean await(final long timeout, final TimeUnit unit) throws InterruptedException {
            lock.lock();
            try {
                return condition.await(timeout, unit);
            } finally {
                lock.unlock();
            }
        }
    }

    private static class CalculatorCallable implements Callable {
        private final Calculator bean;

        public CalculatorCallable(final Calculator bean) {
            this.bean = bean;
        }

        @Override
        public Object call() throws Exception {
            return bean.name();
        }
    }
}