/*
* Licensed to Elasticsearch under one or more contributor
* license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright
* ownership. Elasticsearch licenses this file to you under
* the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.elasticsearch.tribe;
import org.elasticsearch.action.admin.cluster.health.ClusterHealthResponse;
import org.elasticsearch.action.support.DestructiveOperations;
import org.elasticsearch.client.Client;
import org.elasticsearch.cluster.ClusterName;
import org.elasticsearch.cluster.ClusterStateUpdateTask;
import org.elasticsearch.cluster.ClusterState;
import org.elasticsearch.cluster.NamedDiff;
import org.elasticsearch.cluster.block.ClusterBlockException;
import org.elasticsearch.cluster.health.ClusterHealthStatus;
import org.elasticsearch.cluster.metadata.MetaData;
import org.elasticsearch.cluster.node.DiscoveryNode;
import org.elasticsearch.cluster.service.ClusterService;
import org.elasticsearch.common.Priority;
import org.elasticsearch.common.UUIDs;
import org.elasticsearch.common.io.stream.NamedWriteableRegistry;
import org.elasticsearch.common.io.stream.Writeable;
import org.elasticsearch.common.lease.Releasable;
import org.elasticsearch.common.network.NetworkModule;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.common.util.set.Sets;
import org.elasticsearch.discovery.DiscoverySettings;
import org.elasticsearch.discovery.MasterNotDiscoveredException;
import org.elasticsearch.node.Node;
import org.elasticsearch.plugins.Plugin;
import org.elasticsearch.test.ESIntegTestCase;
import org.elasticsearch.test.InternalTestCluster;
import org.elasticsearch.test.NodeConfigurationSource;
import org.elasticsearch.test.TestCustomMetaData;
import org.elasticsearch.transport.MockTcpTransportPlugin;
import org.elasticsearch.tribe.TribeServiceTests.MergableCustomMetaData1;
import org.elasticsearch.tribe.TribeServiceTests.MergableCustomMetaData2;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.function.Consumer;
import java.util.function.Function;
import java.util.function.Predicate;
import java.util.function.UnaryOperator;
import java.util.stream.Stream;
import java.util.stream.StreamSupport;
import static java.util.stream.Collectors.toSet;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertAcked;
import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount;
import static org.hamcrest.Matchers.containsInAnyOrder;
import static org.hamcrest.Matchers.containsString;
import static org.hamcrest.Matchers.equalTo;
import static org.hamcrest.Matchers.notNullValue;
import static org.hamcrest.core.Is.is;
/**
* Note, when talking to tribe client, no need to set the local flag on master read operations, it
* does it by default.
*/
@ESIntegTestCase.ClusterScope(scope = ESIntegTestCase.Scope.SUITE, numDataNodes = 0, numClientNodes = 0, transportClientRatio = 0.0)
public class TribeIT extends ESIntegTestCase {
private static final String TRIBE_NODE = "tribe_node";
private static InternalTestCluster cluster1;
private static InternalTestCluster cluster2;
/**
* A predicate that is used to select none of the remote clusters
**/
private static final Predicate<InternalTestCluster> NONE = c -> false;
/**
* A predicate that is used to select the remote cluster 1 only
**/
private static final Predicate<InternalTestCluster> CLUSTER1_ONLY = c -> c.getClusterName().equals(cluster1.getClusterName());
/**
* A predicate that is used to select the remote cluster 2 only
**/
private static final Predicate<InternalTestCluster> CLUSTER2_ONLY = c -> c.getClusterName().equals(cluster2.getClusterName());
/**
* A predicate that is used to select the the two remote clusters
**/
private static final Predicate<InternalTestCluster> ALL = c -> true;
@Override
protected Settings nodeSettings(int nodeOrdinal) {
return Settings.builder()
.put(super.nodeSettings(nodeOrdinal))
// Required to delete _all indices on remote clusters
.put(DestructiveOperations.REQUIRES_NAME_SETTING.getKey(), false)
.build();
}
public static class TestCustomMetaDataPlugin extends Plugin {
private final List<NamedWriteableRegistry.Entry> namedWritables = new ArrayList<>();
public TestCustomMetaDataPlugin() {
registerBuiltinWritables();
}
private <T extends MetaData.Custom> void registerMetaDataCustom(String name, Writeable.Reader<? extends T> reader,
Writeable.Reader<NamedDiff> diffReader) {
namedWritables.add(new NamedWriteableRegistry.Entry(MetaData.Custom.class, name, reader));
namedWritables.add(new NamedWriteableRegistry.Entry(NamedDiff.class, name, diffReader));
}
private void registerBuiltinWritables() {
registerMetaDataCustom(MergableCustomMetaData1.TYPE, MergableCustomMetaData1::readFrom, MergableCustomMetaData1::readDiffFrom);
registerMetaDataCustom(MergableCustomMetaData2.TYPE, MergableCustomMetaData2::readFrom, MergableCustomMetaData2::readDiffFrom);
}
@Override
public List<NamedWriteableRegistry.Entry> getNamedWriteables() {
return namedWritables;
}
}
@Override
protected Collection<Class<? extends Plugin>> nodePlugins() {
ArrayList<Class<? extends Plugin>> plugins = new ArrayList<>();
plugins.addAll(getMockPlugins());
plugins.add(TestCustomMetaDataPlugin.class);
return plugins;
}
@Before
public void startRemoteClusters() {
final int minNumDataNodes = 2;
final int maxNumDataNodes = 4;
final NodeConfigurationSource nodeConfigurationSource = getNodeConfigSource();
final Collection<Class<? extends Plugin>> plugins = nodePlugins();
if (cluster1 == null) {
cluster1 = new InternalTestCluster(randomLong(), createTempDir(), true, true, minNumDataNodes, maxNumDataNodes,
UUIDs.randomBase64UUID(random()), nodeConfigurationSource, 0, false, "cluster_1",
plugins, Function.identity());
}
if (cluster2 == null) {
cluster2 = new InternalTestCluster(randomLong(), createTempDir(), true, true, minNumDataNodes, maxNumDataNodes,
UUIDs.randomBase64UUID(random()), nodeConfigurationSource, 0, false, "cluster_2",
plugins, Function.identity());
}
doWithAllClusters(c -> {
try {
c.beforeTest(random(), 0.1);
c.ensureAtLeastNumDataNodes(minNumDataNodes);
} catch (Exception e) {
throw new RuntimeException("Failed to set up remote cluster [" + c.getClusterName() + "]", e);
}
});
}
@After
public void wipeRemoteClusters() {
doWithAllClusters(c -> {
final String clusterName = c.getClusterName();
try {
c.client().admin().indices().prepareDelete(MetaData.ALL).get();
c.afterTest();
} catch (IOException e) {
throw new RuntimeException("Failed to clean up remote cluster [" + clusterName + "]", e);
}
});
}
@AfterClass
public static void stopRemoteClusters() {
try {
doWithAllClusters(InternalTestCluster::close);
} finally {
cluster1 = null;
cluster2 = null;
}
}
private Releasable startTribeNode() throws Exception {
return startTribeNode(ALL, Settings.EMPTY);
}
private Releasable startTribeNode(Predicate<InternalTestCluster> filter, Settings settings) throws Exception {
final String node = internalCluster().startNode(createTribeSettings(filter).put(settings).build());
// wait for node to be connected to all tribe clusters
final Set<String> expectedNodes = Sets.newHashSet(internalCluster().getNodeNames());
doWithAllClusters(filter, c -> {
// Adds the tribe client node dedicated to this remote cluster
for (String tribeNode : internalCluster().getNodeNames()) {
expectedNodes.add(tribeNode + "/" + c.getClusterName());
}
// Adds the remote clusters nodes names
Collections.addAll(expectedNodes, c.getNodeNames());
});
assertBusy(() -> {
ClusterState state = client().admin().cluster().prepareState().setNodes(true).get().getState();
Set<String> nodes = StreamSupport.stream(state.getNodes().spliterator(), false).map(DiscoveryNode::getName).collect(toSet());
assertThat(nodes, containsInAnyOrder(expectedNodes.toArray()));
});
// wait for join to be fully applied on all nodes in the tribe clusters, see https://github.com/elastic/elasticsearch/issues/23695
doWithAllClusters(filter, c -> {
assertFalse(c.client().admin().cluster().prepareHealth().setWaitForEvents(Priority.LANGUID).get().isTimedOut());
});
return () -> {
try {
while(internalCluster().getNodeNames().length > 0) {
internalCluster().stopRandomNode(s -> true);
}
} catch (Exception e) {
throw new RuntimeException("Failed to close tribe node [" + node + "]", e);
}
};
}
private Settings.Builder createTribeSettings(Predicate<InternalTestCluster> filter) {
assertNotNull(filter);
final Settings.Builder settings = Settings.builder();
settings.put(Node.NODE_NAME_SETTING.getKey(), TRIBE_NODE);
settings.put(Node.NODE_DATA_SETTING.getKey(), false);
settings.put(Node.NODE_MASTER_SETTING.getKey(), true);
settings.put(NetworkModule.HTTP_ENABLED.getKey(), false);
settings.put(NetworkModule.TRANSPORT_TYPE_SETTING.getKey(), MockTcpTransportPlugin.MOCK_TCP_TRANSPORT_NAME);
doWithAllClusters(filter, c -> {
String tribeSetting = "tribe." + c.getClusterName() + ".";
settings.put(tribeSetting + ClusterName.CLUSTER_NAME_SETTING.getKey(), c.getClusterName());
settings.put(tribeSetting + DiscoverySettings.INITIAL_STATE_TIMEOUT_SETTING.getKey(), "100ms");
settings.put(tribeSetting + NetworkModule.TRANSPORT_TYPE_SETTING.getKey(), MockTcpTransportPlugin.MOCK_TCP_TRANSPORT_NAME);
});
return settings;
}
public void testGlobalReadWriteBlocks() throws Exception {
Settings additionalSettings = Settings.builder()
.put("tribe.blocks.write", true)
.put("tribe.blocks.metadata", true)
.build();
try (Releasable tribeNode = startTribeNode(ALL, additionalSettings)) {
// Creates 2 indices, test1 on cluster1 and test2 on cluster2
assertAcked(cluster1.client().admin().indices().prepareCreate("test1"));
ensureGreen(cluster1.client());
assertAcked(cluster2.client().admin().indices().prepareCreate("test2"));
ensureGreen(cluster2.client());
// Wait for the tribe node to retrieve the indices into its cluster state
assertIndicesExist(client(), "test1", "test2");
// Writes not allowed through the tribe node
ClusterBlockException e = expectThrows(ClusterBlockException.class, () -> {
client().prepareIndex("test1", "type1").setSource("field", "value").get();
});
assertThat(e.getMessage(), containsString("blocked by: [BAD_REQUEST/11/tribe node, write not allowed]"));
e = expectThrows(ClusterBlockException.class, () -> client().prepareIndex("test2", "type2").setSource("field", "value").get());
assertThat(e.getMessage(), containsString("blocked by: [BAD_REQUEST/11/tribe node, write not allowed]"));
e = expectThrows(ClusterBlockException.class, () -> client().admin().indices().prepareForceMerge("test1").get());
assertThat(e.getMessage(), containsString("blocked by: [BAD_REQUEST/10/tribe node, metadata not allowed]"));
e = expectThrows(ClusterBlockException.class, () -> client().admin().indices().prepareForceMerge("test2").get());
assertThat(e.getMessage(), containsString("blocked by: [BAD_REQUEST/10/tribe node, metadata not allowed]"));
}
}
public void testIndexWriteBlocks() throws Exception {
Settings additionalSettings = Settings.builder()
.put("tribe.blocks.write.indices", "block_*")
.build();
try (Releasable tribeNode = startTribeNode(ALL, additionalSettings)) {
// Creates 2 indices on each remote cluster, test1 and block_test1 on cluster1 and test2 and block_test2 on cluster2
assertAcked(cluster1.client().admin().indices().prepareCreate("test1"));
assertAcked(cluster1.client().admin().indices().prepareCreate("block_test1"));
ensureGreen(cluster1.client());
assertAcked(cluster2.client().admin().indices().prepareCreate("test2"));
assertAcked(cluster2.client().admin().indices().prepareCreate("block_test2"));
ensureGreen(cluster2.client());
// Wait for the tribe node to retrieve the indices into its cluster state
assertIndicesExist(client(), "test1", "test2", "block_test1", "block_test2");
// Writes allowed through the tribe node for test1/test2 indices
client().prepareIndex("test1", "type1").setSource("field", "value").get();
client().prepareIndex("test2", "type2").setSource("field", "value").get();
ClusterBlockException e;
e = expectThrows(ClusterBlockException.class, () -> client().prepareIndex("block_test1", "type1").setSource("foo", 0).get());
assertThat(e.getMessage(), containsString("blocked by: [FORBIDDEN/8/index write (api)]"));
e = expectThrows(ClusterBlockException.class, () -> client().prepareIndex("block_test2", "type2").setSource("foo", 0).get());
assertThat(e.getMessage(), containsString("blocked by: [FORBIDDEN/8/index write (api)]"));
}
}
public void testOnConflictDrop() throws Exception {
Settings additionalSettings = Settings.builder()
.put("tribe.on_conflict", "drop")
.build();
try (Releasable tribeNode = startTribeNode(ALL, additionalSettings)) {
// Creates 2 indices on each remote cluster, test1 and conflict on cluster1 and test2 and also conflict on cluster2
assertAcked(cluster1.client().admin().indices().prepareCreate("test1"));
assertAcked(cluster1.client().admin().indices().prepareCreate("conflict"));
ensureGreen(cluster1.client());
assertAcked(cluster2.client().admin().indices().prepareCreate("test2"));
assertAcked(cluster2.client().admin().indices().prepareCreate("conflict"));
ensureGreen(cluster2.client());
// Wait for the tribe node to retrieve the indices into its cluster state
assertIndicesExist(client(), "test1", "test2");
ClusterState clusterState = client().admin().cluster().prepareState().get().getState();
assertThat(clusterState.getMetaData().hasIndex("test1"), is(true));
assertThat(clusterState.getMetaData().index("test1").getSettings().get("tribe.name"), equalTo(cluster1.getClusterName()));
assertThat(clusterState.getMetaData().hasIndex("test2"), is(true));
assertThat(clusterState.getMetaData().index("test2").getSettings().get("tribe.name"), equalTo(cluster2.getClusterName()));
assertThat(clusterState.getMetaData().hasIndex("conflict"), is(false));
}
}
public void testOnConflictPrefer() throws Exception {
final String preference = randomFrom(cluster1, cluster2).getClusterName();
Settings additionalSettings = Settings.builder()
.put("tribe.on_conflict", "prefer_" + preference)
.build();
try (Releasable tribeNode = startTribeNode(ALL, additionalSettings)) {
assertAcked(cluster1.client().admin().indices().prepareCreate("test1"));
assertAcked(cluster1.client().admin().indices().prepareCreate("shared"));
ensureGreen(cluster1.client());
assertAcked(cluster2.client().admin().indices().prepareCreate("test2"));
assertAcked(cluster2.client().admin().indices().prepareCreate("shared"));
ensureGreen(cluster2.client());
// Wait for the tribe node to retrieve the indices into its cluster state
assertIndicesExist(client(), "test1", "test2", "shared");
ClusterState clusterState = client().admin().cluster().prepareState().get().getState();
assertThat(clusterState.getMetaData().hasIndex("test1"), is(true));
assertThat(clusterState.getMetaData().index("test1").getSettings().get("tribe.name"), equalTo(cluster1.getClusterName()));
assertThat(clusterState.getMetaData().hasIndex("test2"), is(true));
assertThat(clusterState.getMetaData().index("test2").getSettings().get("tribe.name"), equalTo(cluster2.getClusterName()));
assertThat(clusterState.getMetaData().hasIndex("shared"), is(true));
assertThat(clusterState.getMetaData().index("shared").getSettings().get("tribe.name"), equalTo(preference));
}
}
public void testTribeOnOneCluster() throws Exception {
try (Releasable tribeNode = startTribeNode()) {
// Creates 2 indices, test1 on cluster1 and test2 on cluster2
assertAcked(cluster1.client().admin().indices().prepareCreate("test1").setSettings("index.mapping.single_type", false));
ensureGreen(cluster1.client());
assertAcked(cluster2.client().admin().indices().prepareCreate("test2").setSettings("index.mapping.single_type", false));
ensureGreen(cluster2.client());
// Wait for the tribe node to retrieve the indices into its cluster state
assertIndicesExist(client(), "test1", "test2");
// Creates two docs using the tribe node
indexRandom(true,
client().prepareIndex("test1", "type1", "1").setSource("field1", "value1"),
client().prepareIndex("test2", "type1", "1").setSource("field1", "value1")
);
// Verify that documents are searchable using the tribe node
assertHitCount(client().prepareSearch().get(), 2L);
// Using assertBusy to check that the mappings are in the tribe node cluster state
assertBusy(() -> {
ClusterState clusterState = client().admin().cluster().prepareState().get().getState();
assertThat(clusterState.getMetaData().index("test1").mapping("type1"), notNullValue());
assertThat(clusterState.getMetaData().index("test2").mapping("type1"), notNullValue());
});
// More documents with another type
indexRandom(true,
client().prepareIndex("test1", "type2", "1").setSource("field1", "value1"),
client().prepareIndex("test2", "type2", "1").setSource("field1", "value1")
);
assertHitCount(client().prepareSearch().get(), 4L);
assertBusy(() -> {
ClusterState clusterState = client().admin().cluster().prepareState().get().getState();
assertThat(clusterState.getMetaData().index("test1").mapping("type1"), notNullValue());
assertThat(clusterState.getMetaData().index("test1").mapping("type2"), notNullValue());
assertThat(clusterState.getMetaData().index("test2").mapping("type1"), notNullValue());
assertThat(clusterState.getMetaData().index("test2").mapping("type2"), notNullValue());
});
// Make sure master level write operations fail... (we don't really have a master)
expectThrows(MasterNotDiscoveredException.class, () -> {
client().admin().indices().prepareCreate("tribe_index").setMasterNodeTimeout("10ms").get();
});
// Now delete an index and makes sure it's reflected in cluster state
cluster2.client().admin().indices().prepareDelete("test2").get();
assertBusy(() -> {
ClusterState clusterState = client().admin().cluster().prepareState().get().getState();
assertFalse(clusterState.getMetaData().hasIndex("test2"));
assertFalse(clusterState.getRoutingTable().hasIndex("test2"));
});
}
}
public void testCloseAndOpenIndex() throws Exception {
// Creates an index on remote cluster 1
assertTrue(cluster1.client().admin().indices().prepareCreate("first").get().isAcknowledged());
ensureGreen(cluster1.client());
// Closes the index
assertTrue(cluster1.client().admin().indices().prepareClose("first").get().isAcknowledged());
try (Releasable tribeNode = startTribeNode()) {
// The closed index is not part of the tribe node cluster state
ClusterState clusterState = client().admin().cluster().prepareState().get().getState();
assertFalse(clusterState.getMetaData().hasIndex("first"));
// Open the index, it becomes part of the tribe node cluster state
assertTrue(cluster1.client().admin().indices().prepareOpen("first").get().isAcknowledged());
assertIndicesExist(client(), "first");
// Create a second index, wait till it is seen from within the tribe node
assertTrue(cluster2.client().admin().indices().prepareCreate("second").get().isAcknowledged());
assertIndicesExist(client(), "first", "second");
ensureGreen(cluster2.client());
// Close the second index, wait till it gets removed from the tribe node cluster state
assertTrue(cluster2.client().admin().indices().prepareClose("second").get().isAcknowledged());
assertIndicesExist(client(), "first");
// Open the second index, wait till it gets added back to the tribe node cluster state
assertTrue(cluster2.client().admin().indices().prepareOpen("second").get().isAcknowledged());
assertIndicesExist(client(), "first", "second");
ensureGreen(cluster2.client());
}
}
/**
* Test that the tribe node's cluster state correctly reflect the number of nodes
* of the remote clusters the tribe node is connected to.
*/
public void testClusterStateNodes() throws Exception {
List<Predicate<InternalTestCluster>> predicates = Arrays.asList(NONE, CLUSTER1_ONLY, CLUSTER2_ONLY, ALL);
Collections.shuffle(predicates, random());
for (Predicate<InternalTestCluster> predicate : predicates) {
try (Releasable tribeNode = startTribeNode(predicate, Settings.EMPTY)) {
}
}
}
public void testMergingRemovedCustomMetaData() throws Exception {
removeCustomMetaData(cluster1, MergableCustomMetaData1.TYPE);
removeCustomMetaData(cluster2, MergableCustomMetaData1.TYPE);
MergableCustomMetaData1 customMetaData1 = new MergableCustomMetaData1("a");
MergableCustomMetaData1 customMetaData2 = new MergableCustomMetaData1("b");
try (Releasable tribeNode = startTribeNode()) {
putCustomMetaData(cluster1, customMetaData1);
putCustomMetaData(cluster2, customMetaData2);
assertCustomMetaDataUpdated(internalCluster(), customMetaData2);
removeCustomMetaData(cluster2, customMetaData2.getWriteableName());
assertCustomMetaDataUpdated(internalCluster(), customMetaData1);
}
}
public void testMergingCustomMetaData() throws Exception {
removeCustomMetaData(cluster1, MergableCustomMetaData1.TYPE);
removeCustomMetaData(cluster2, MergableCustomMetaData1.TYPE);
MergableCustomMetaData1 customMetaData1 = new MergableCustomMetaData1(randomAlphaOfLength(10));
MergableCustomMetaData1 customMetaData2 = new MergableCustomMetaData1(randomAlphaOfLength(10));
List<MergableCustomMetaData1> customMetaDatas = Arrays.asList(customMetaData1, customMetaData2);
Collections.sort(customMetaDatas, (cm1, cm2) -> cm2.getData().compareTo(cm1.getData()));
final MergableCustomMetaData1 tribeNodeCustomMetaData = customMetaDatas.get(0);
try (Releasable tribeNode = startTribeNode()) {
putCustomMetaData(cluster1, customMetaData1);
assertCustomMetaDataUpdated(internalCluster(), customMetaData1);
putCustomMetaData(cluster2, customMetaData2);
assertCustomMetaDataUpdated(internalCluster(), tribeNodeCustomMetaData);
}
}
public void testMergingMultipleCustomMetaData() throws Exception {
removeCustomMetaData(cluster1, MergableCustomMetaData1.TYPE);
removeCustomMetaData(cluster2, MergableCustomMetaData1.TYPE);
MergableCustomMetaData1 firstCustomMetaDataType1 = new MergableCustomMetaData1(randomAlphaOfLength(10));
MergableCustomMetaData1 secondCustomMetaDataType1 = new MergableCustomMetaData1(randomAlphaOfLength(10));
MergableCustomMetaData2 firstCustomMetaDataType2 = new MergableCustomMetaData2(randomAlphaOfLength(10));
MergableCustomMetaData2 secondCustomMetaDataType2 = new MergableCustomMetaData2(randomAlphaOfLength(10));
List<MergableCustomMetaData1> mergedCustomMetaDataType1 = Arrays.asList(firstCustomMetaDataType1, secondCustomMetaDataType1);
List<MergableCustomMetaData2> mergedCustomMetaDataType2 = Arrays.asList(firstCustomMetaDataType2, secondCustomMetaDataType2);
Collections.sort(mergedCustomMetaDataType1, (cm1, cm2) -> cm2.getData().compareTo(cm1.getData()));
Collections.sort(mergedCustomMetaDataType2, (cm1, cm2) -> cm2.getData().compareTo(cm1.getData()));
try (Releasable tribeNode = startTribeNode()) {
// test putting multiple custom md types propagates to tribe
putCustomMetaData(cluster1, firstCustomMetaDataType1);
putCustomMetaData(cluster1, firstCustomMetaDataType2);
assertCustomMetaDataUpdated(internalCluster(), firstCustomMetaDataType1);
assertCustomMetaDataUpdated(internalCluster(), firstCustomMetaDataType2);
// test multiple same type custom md is merged and propagates to tribe
putCustomMetaData(cluster2, secondCustomMetaDataType1);
assertCustomMetaDataUpdated(internalCluster(), firstCustomMetaDataType2);
assertCustomMetaDataUpdated(internalCluster(), mergedCustomMetaDataType1.get(0));
// test multiple same type custom md is merged and propagates to tribe
putCustomMetaData(cluster2, secondCustomMetaDataType2);
assertCustomMetaDataUpdated(internalCluster(), mergedCustomMetaDataType1.get(0));
assertCustomMetaDataUpdated(internalCluster(), mergedCustomMetaDataType2.get(0));
// test removing custom md is propagates to tribe
removeCustomMetaData(cluster2, secondCustomMetaDataType1.getWriteableName());
assertCustomMetaDataUpdated(internalCluster(), firstCustomMetaDataType1);
assertCustomMetaDataUpdated(internalCluster(), mergedCustomMetaDataType2.get(0));
removeCustomMetaData(cluster2, secondCustomMetaDataType2.getWriteableName());
assertCustomMetaDataUpdated(internalCluster(), firstCustomMetaDataType1);
assertCustomMetaDataUpdated(internalCluster(), firstCustomMetaDataType2);
}
}
private static void assertCustomMetaDataUpdated(InternalTestCluster cluster,
TestCustomMetaData expectedCustomMetaData) throws Exception {
assertBusy(() -> {
ClusterState tribeState = cluster.getInstance(ClusterService.class, cluster.getNodeNames()[0]).state();
MetaData.Custom custom = tribeState.metaData().custom(expectedCustomMetaData.getWriteableName());
assertNotNull(custom);
assertThat(custom, equalTo(expectedCustomMetaData));
});
}
private void removeCustomMetaData(InternalTestCluster cluster, final String customMetaDataType) {
logger.info("removing custom_md type [{}] from [{}]", customMetaDataType, cluster.getClusterName());
updateMetaData(cluster, builder -> builder.removeCustom(customMetaDataType));
}
private void putCustomMetaData(InternalTestCluster cluster, final TestCustomMetaData customMetaData) {
logger.info("putting custom_md type [{}] with data[{}] from [{}]", customMetaData.getWriteableName(),
customMetaData.getData(), cluster.getClusterName());
updateMetaData(cluster, builder -> builder.putCustom(customMetaData.getWriteableName(), customMetaData));
}
private static void updateMetaData(InternalTestCluster cluster, UnaryOperator<MetaData.Builder> addCustoms) {
ClusterService clusterService = cluster.getInstance(ClusterService.class, cluster.getMasterName());
final CountDownLatch latch = new CountDownLatch(1);
clusterService.submitStateUpdateTask("update customMetaData", new ClusterStateUpdateTask(Priority.IMMEDIATE) {
@Override
public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
latch.countDown();
}
@Override
public ClusterState execute(ClusterState currentState) throws Exception {
MetaData.Builder builder = MetaData.builder(currentState.metaData());
builder = addCustoms.apply(builder);
return ClusterState.builder(currentState).metaData(builder).build();
}
@Override
public void onFailure(String source, Exception e) {
fail("failed to apply cluster state from [" + source + "] with " + e.getMessage());
}
});
try {
latch.await(1, TimeUnit.MINUTES);
} catch (InterruptedException e) {
fail("latch waiting on publishing custom md interrupted [" + e.getMessage() + "]");
}
assertThat("timed out trying to add custom metadata to " + cluster.getClusterName(), latch.getCount(), equalTo(0L));
}
private void assertIndicesExist(Client client, String... indices) throws Exception {
assertBusy(() -> {
ClusterState state = client.admin().cluster().prepareState().setRoutingTable(true).setMetaData(true).get().getState();
assertThat(state.getMetaData().getIndices().size(), equalTo(indices.length));
for (String index : indices) {
assertTrue(state.getMetaData().hasIndex(index));
assertTrue(state.getRoutingTable().hasIndex(index));
}
});
}
private void ensureGreen(Client client) throws Exception {
assertBusy(() -> {
ClusterHealthResponse clusterHealthResponse = client.admin().cluster() .prepareHealth()
.setWaitForActiveShards(0)
.setWaitForEvents(Priority.LANGUID)
.setWaitForNoRelocatingShards(true)
.get();
assertThat(clusterHealthResponse.getStatus(), equalTo(ClusterHealthStatus.GREEN));
assertFalse(clusterHealthResponse.isTimedOut());
});
}
private static void doWithAllClusters(Consumer<InternalTestCluster> consumer) {
doWithAllClusters(cluster -> cluster != null, consumer);
}
private static void doWithAllClusters(Predicate<InternalTestCluster> predicate, Consumer<InternalTestCluster> consumer) {
Stream.of(cluster1, cluster2).filter(predicate).forEach(consumer);
}
}