/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.solr.cloud; import java.lang.invoke.MethodHandles; import java.net.URL; import java.util.Collections; import java.util.List; import java.util.Objects; import java.util.concurrent.TimeUnit; import java.util.function.Predicate; import org.apache.solr.client.solrj.embedded.JettySolrRunner; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.cloud.overseer.OverseerAction; import org.apache.solr.common.cloud.ZkNodeProps; import org.apache.solr.common.util.Utils; import org.apache.solr.util.TimeOut; import org.apache.zookeeper.KeeperException; import org.junit.BeforeClass; import org.junit.Test; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import static org.apache.solr.cloud.OverseerCollectionConfigSetProcessor.getLeaderNode; import static org.apache.solr.cloud.OverseerTaskProcessor.getSortedElectionNodes; public class OverseerRolesTest extends SolrCloudTestCase { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); @BeforeClass public static void setupCluster() throws Exception { configureCluster(4) .addConfig("conf", configset("cloud-minimal")) .configure(); } private void waitForNewOverseer(int seconds, Predicate<String> state) throws Exception { TimeOut timeout = new TimeOut(seconds, TimeUnit.SECONDS); String current = null; while (timeout.hasTimedOut() == false) { current = OverseerCollectionConfigSetProcessor.getLeaderNode(zkClient()); if (state.test(current)) return; Thread.sleep(100); } fail("Timed out waiting for overseer state change"); } private void waitForNewOverseer(int seconds, String expected) throws Exception { waitForNewOverseer(seconds, s -> Objects.equals(s, expected)); } private JettySolrRunner getOverseerJetty() throws Exception { String overseer = getLeaderNode(zkClient()); URL overseerUrl = new URL("http://" + overseer.substring(0, overseer.indexOf('_'))); int hostPort = overseerUrl.getPort(); for (JettySolrRunner jetty : cluster.getJettySolrRunners()) { if (jetty.getBaseUrl().getPort() == hostPort) return jetty; } fail("Couldn't find overseer node " + overseer); return null; // to keep the compiler happy } private void logOverseerState() throws KeeperException, InterruptedException { log.info("Overseer: {}", getLeaderNode(zkClient())); log.info("Election queue: ", getSortedElectionNodes(zkClient(), "/overseer_elect/election")); } @Test public void testOverseerRole() throws Exception { logOverseerState(); List<String> nodes = OverseerCollectionConfigSetProcessor.getSortedOverseerNodeNames(zkClient()); String overseer1 = OverseerCollectionConfigSetProcessor.getLeaderNode(zkClient()); nodes.remove(overseer1); Collections.shuffle(nodes, random()); String overseer2 = nodes.get(0); log.info("### Setting overseer designate {}", overseer2); CollectionAdminRequest.addRole(overseer2, "overseer").process(cluster.getSolrClient()); waitForNewOverseer(15, overseer2); //add another node as overseer nodes.remove(overseer2); Collections.shuffle(nodes, random()); String overseer3 = nodes.get(0); log.info("### Adding another overseer designate {}", overseer3); CollectionAdminRequest.addRole(overseer3, "overseer").process(cluster.getSolrClient()); // kill the current overseer, and check that the new designate becomes the new overseer JettySolrRunner leaderJetty = getOverseerJetty(); logOverseerState(); ChaosMonkey.stop(leaderJetty); waitForNewOverseer(10, overseer3); // add another node as overseer nodes.remove(overseer3); Collections.shuffle(nodes, random()); String overseer4 = nodes.get(0); log.info("### Adding last overseer designate {}", overseer4); CollectionAdminRequest.addRole(overseer4, "overseer").process(cluster.getSolrClient()); logOverseerState(); // remove the overseer role from the current overseer CollectionAdminRequest.removeRole(overseer3, "overseer").process(cluster.getSolrClient()); waitForNewOverseer(15, overseer4); // Add it back again - we now have two delegates, 4 and 3 CollectionAdminRequest.addRole(overseer3, "overseer").process(cluster.getSolrClient()); // explicitly tell the overseer to quit String leaderId = OverseerCollectionConfigSetProcessor.getLeaderId(zkClient()); String leader = OverseerCollectionConfigSetProcessor.getLeaderNode(zkClient()); log.info("### Sending QUIT to overseer {}", leader); Overseer.getStateUpdateQueue(zkClient()) .offer(Utils.toJSON(new ZkNodeProps(Overseer.QUEUE_OPERATION, OverseerAction.QUIT.toLower(), "id", leaderId))); waitForNewOverseer(15, s -> Objects.equals(leader, s) == false); Thread.sleep(1000); logOverseerState(); assertTrue("The old leader should have rejoined election", OverseerCollectionConfigSetProcessor.getSortedOverseerNodeNames(zkClient()).contains(leader)); } }