/* * Licensed to the Apache Software Foundation (ASF) under one or more * contributor license agreements. See the NOTICE file distributed with * this work for additional information regarding copyright ownership. * The ASF licenses this file to You under the Apache License, Version 2.0 * (the "License"); you may not use this file except in compliance with * the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package org.apache.kafka.connect.runtime.distributed; import org.apache.kafka.clients.CommonClientConfigs; import org.apache.kafka.common.config.Config; import org.apache.kafka.common.config.ConfigDef; import org.apache.kafka.common.config.ConfigValue; import org.apache.kafka.common.utils.MockTime; import org.apache.kafka.connect.connector.Connector; import org.apache.kafka.connect.connector.ConnectorContext; import org.apache.kafka.connect.errors.AlreadyExistsException; import org.apache.kafka.connect.errors.NotFoundException; import org.apache.kafka.connect.runtime.ConnectorConfig; import org.apache.kafka.connect.runtime.Herder; import org.apache.kafka.connect.runtime.SinkConnectorConfig; import org.apache.kafka.connect.runtime.TargetState; import org.apache.kafka.connect.runtime.TaskConfig; import org.apache.kafka.connect.runtime.Worker; import org.apache.kafka.connect.runtime.WorkerConfig; import org.apache.kafka.connect.runtime.isolation.DelegatingClassLoader; import org.apache.kafka.connect.runtime.isolation.PluginClassLoader; import org.apache.kafka.connect.runtime.isolation.Plugins; import org.apache.kafka.connect.runtime.rest.entities.ConnectorInfo; import org.apache.kafka.connect.runtime.rest.entities.TaskInfo; import org.apache.kafka.connect.runtime.rest.errors.BadRequestException; import org.apache.kafka.connect.sink.SinkConnector; import org.apache.kafka.connect.source.SourceConnector; import org.apache.kafka.connect.source.SourceTask; import org.apache.kafka.connect.storage.ConfigBackingStore; import org.apache.kafka.connect.storage.StatusBackingStore; import org.apache.kafka.connect.util.Callback; import org.apache.kafka.connect.util.ConnectorTaskId; import org.apache.kafka.connect.util.FutureCallback; import org.easymock.Capture; import org.easymock.EasyMock; import org.easymock.IAnswer; import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; import org.powermock.api.easymock.PowerMock; import org.powermock.api.easymock.annotation.Mock; import org.powermock.core.classloader.annotations.PowerMockIgnore; import org.powermock.core.classloader.annotations.PrepareForTest; import org.powermock.modules.junit4.PowerMockRunner; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.concurrent.ExecutionException; import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import static java.util.Collections.singletonList; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; import static org.junit.Assert.fail; @RunWith(PowerMockRunner.class) @PrepareForTest({DistributedHerder.class, Plugins.class}) @PowerMockIgnore("javax.management.*") public class DistributedHerderTest { private static final Map<String, String> HERDER_CONFIG = new HashMap<>(); static { HERDER_CONFIG.put(DistributedConfig.STATUS_STORAGE_TOPIC_CONFIG, "status-topic"); HERDER_CONFIG.put(DistributedConfig.CONFIG_TOPIC_CONFIG, "config-topic"); HERDER_CONFIG.put(CommonClientConfigs.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092"); HERDER_CONFIG.put(DistributedConfig.GROUP_ID_CONFIG, "connect-test-group"); // The WorkerConfig base class has some required settings without defaults HERDER_CONFIG.put(WorkerConfig.KEY_CONVERTER_CLASS_CONFIG, "org.apache.kafka.connect.json.JsonConverter"); HERDER_CONFIG.put(WorkerConfig.VALUE_CONVERTER_CLASS_CONFIG, "org.apache.kafka.connect.json.JsonConverter"); HERDER_CONFIG.put(WorkerConfig.INTERNAL_KEY_CONVERTER_CLASS_CONFIG, "org.apache.kafka.connect.json.JsonConverter"); HERDER_CONFIG.put(WorkerConfig.INTERNAL_VALUE_CONVERTER_CLASS_CONFIG, "org.apache.kafka.connect.json.JsonConverter"); HERDER_CONFIG.put(DistributedConfig.OFFSET_STORAGE_TOPIC_CONFIG, "connect-offsets"); } private static final String MEMBER_URL = "memberUrl"; private static final String CONN1 = "sourceA"; private static final String CONN2 = "sourceB"; private static final ConnectorTaskId TASK0 = new ConnectorTaskId(CONN1, 0); private static final ConnectorTaskId TASK1 = new ConnectorTaskId(CONN1, 1); private static final ConnectorTaskId TASK2 = new ConnectorTaskId(CONN1, 2); private static final Integer MAX_TASKS = 3; private static final Map<String, String> CONN1_CONFIG = new HashMap<>(); static { CONN1_CONFIG.put(ConnectorConfig.NAME_CONFIG, CONN1); CONN1_CONFIG.put(ConnectorConfig.TASKS_MAX_CONFIG, MAX_TASKS.toString()); CONN1_CONFIG.put(SinkConnectorConfig.TOPICS_CONFIG, "foo,bar"); CONN1_CONFIG.put(ConnectorConfig.CONNECTOR_CLASS_CONFIG, BogusSourceConnector.class.getName()); } private static final Map<String, String> CONN1_CONFIG_UPDATED = new HashMap<>(CONN1_CONFIG); static { CONN1_CONFIG_UPDATED.put(SinkConnectorConfig.TOPICS_CONFIG, "foo,bar,baz"); } private static final Map<String, String> CONN2_CONFIG = new HashMap<>(); static { CONN2_CONFIG.put(ConnectorConfig.NAME_CONFIG, CONN2); CONN2_CONFIG.put(ConnectorConfig.TASKS_MAX_CONFIG, MAX_TASKS.toString()); CONN2_CONFIG.put(SinkConnectorConfig.TOPICS_CONFIG, "foo,bar"); CONN2_CONFIG.put(ConnectorConfig.CONNECTOR_CLASS_CONFIG, BogusSourceConnector.class.getName()); } private static final Map<String, String> TASK_CONFIG = new HashMap<>(); static { TASK_CONFIG.put(TaskConfig.TASK_CLASS_CONFIG, BogusSourceTask.class.getName()); } private static final List<Map<String, String>> TASK_CONFIGS = new ArrayList<>(); static { TASK_CONFIGS.add(TASK_CONFIG); TASK_CONFIGS.add(TASK_CONFIG); TASK_CONFIGS.add(TASK_CONFIG); } private static final HashMap<ConnectorTaskId, Map<String, String>> TASK_CONFIGS_MAP = new HashMap<>(); static { TASK_CONFIGS_MAP.put(TASK0, TASK_CONFIG); TASK_CONFIGS_MAP.put(TASK1, TASK_CONFIG); TASK_CONFIGS_MAP.put(TASK2, TASK_CONFIG); } private static final ClusterConfigState SNAPSHOT = new ClusterConfigState(1, Collections.singletonMap(CONN1, 3), Collections.singletonMap(CONN1, CONN1_CONFIG), Collections.singletonMap(CONN1, TargetState.STARTED), TASK_CONFIGS_MAP, Collections.<String>emptySet()); private static final ClusterConfigState SNAPSHOT_PAUSED_CONN1 = new ClusterConfigState(1, Collections.singletonMap(CONN1, 3), Collections.singletonMap(CONN1, CONN1_CONFIG), Collections.singletonMap(CONN1, TargetState.PAUSED), TASK_CONFIGS_MAP, Collections.<String>emptySet()); private static final ClusterConfigState SNAPSHOT_UPDATED_CONN1_CONFIG = new ClusterConfigState(1, Collections.singletonMap(CONN1, 3), Collections.singletonMap(CONN1, CONN1_CONFIG_UPDATED), Collections.singletonMap(CONN1, TargetState.STARTED), TASK_CONFIGS_MAP, Collections.<String>emptySet()); private static final String WORKER_ID = "localhost:8083"; @Mock private ConfigBackingStore configBackingStore; @Mock private StatusBackingStore statusBackingStore; @Mock private WorkerGroupMember member; private MockTime time; private DistributedHerder herder; @Mock private Worker worker; @Mock private Callback<Herder.Created<ConnectorInfo>> putConnectorCallback; @Mock private Plugins plugins; @Mock private PluginClassLoader pluginLoader; @Mock private DelegatingClassLoader delegatingLoader; private ConfigBackingStore.UpdateListener configUpdateListener; private WorkerRebalanceListener rebalanceListener; @Before public void setUp() throws Exception { worker = PowerMock.createMock(Worker.class); EasyMock.expect(worker.isSinkConnector(CONN1)).andStubReturn(Boolean.FALSE); time = new MockTime(); herder = PowerMock.createPartialMock(DistributedHerder.class, new String[]{"backoff", "updateDeletedConnectorStatus"}, new DistributedConfig(HERDER_CONFIG), worker, WORKER_ID, statusBackingStore, configBackingStore, member, MEMBER_URL, time); configUpdateListener = herder.new ConfigUpdateListener(); rebalanceListener = herder.new RebalanceListener(); plugins = PowerMock.createMock(Plugins.class); pluginLoader = PowerMock.createMock(PluginClassLoader.class); delegatingLoader = PowerMock.createMock(DelegatingClassLoader.class); PowerMock.mockStatic(Plugins.class); PowerMock.expectPrivate(herder, "updateDeletedConnectorStatus").andVoid().anyTimes(); } @Test public void testJoinAssignment() throws Exception { // Join group and get assignment EasyMock.expect(member.memberId()).andStubReturn("member"); EasyMock.expect(worker.getPlugins()).andReturn(plugins); expectRebalance(1, Arrays.asList(CONN1), Arrays.asList(TASK1)); expectPostRebalanceCatchup(SNAPSHOT); worker.startConnector(EasyMock.eq(CONN1), EasyMock.<Map<String, String>>anyObject(), EasyMock.<ConnectorContext>anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED)); PowerMock.expectLastCall().andReturn(true); EasyMock.expect(worker.isRunning(CONN1)).andReturn(true); EasyMock.expect(worker.connectorTaskConfigs(CONN1, MAX_TASKS, null)).andReturn(TASK_CONFIGS); worker.startTask(EasyMock.eq(TASK1), EasyMock.<Map<String, String>>anyObject(), EasyMock.<Map<String, String>>anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED)); PowerMock.expectLastCall().andReturn(true); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); PowerMock.replayAll(); herder.tick(); PowerMock.verifyAll(); } @Test public void testRebalance() throws Exception { // Join group and get assignment EasyMock.expect(member.memberId()).andStubReturn("member"); EasyMock.expect(worker.getPlugins()).andReturn(plugins); expectRebalance(1, Arrays.asList(CONN1), Arrays.asList(TASK1)); expectPostRebalanceCatchup(SNAPSHOT); worker.startConnector(EasyMock.eq(CONN1), EasyMock.<Map<String, String>>anyObject(), EasyMock.<ConnectorContext>anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED)); PowerMock.expectLastCall().andReturn(true); EasyMock.expect(worker.isRunning(CONN1)).andReturn(true); EasyMock.expect(worker.connectorTaskConfigs(CONN1, MAX_TASKS, null)).andReturn(TASK_CONFIGS); worker.startTask(EasyMock.eq(TASK1), EasyMock.<Map<String, String>>anyObject(), EasyMock.<Map<String, String>>anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED)); PowerMock.expectLastCall().andReturn(true); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); EasyMock.expect(worker.getPlugins()).andReturn(plugins); expectRebalance(Arrays.asList(CONN1), Arrays.asList(TASK1), ConnectProtocol.Assignment.NO_ERROR, 1, Arrays.asList(CONN1), Arrays.<ConnectorTaskId>asList()); // and the new assignment started worker.startConnector(EasyMock.eq(CONN1), EasyMock.<Map<String, String>>anyObject(), EasyMock.<ConnectorContext>anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED)); PowerMock.expectLastCall().andReturn(true); EasyMock.expect(worker.isRunning(CONN1)).andReturn(true); EasyMock.expect(worker.connectorTaskConfigs(CONN1, MAX_TASKS, null)).andReturn(TASK_CONFIGS); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); PowerMock.replayAll(); herder.tick(); herder.tick(); PowerMock.verifyAll(); } @Test public void testRebalanceFailedConnector() throws Exception { // Join group and get assignment EasyMock.expect(member.memberId()).andStubReturn("member"); EasyMock.expect(worker.getPlugins()).andReturn(plugins); expectRebalance(1, Arrays.asList(CONN1), Arrays.asList(TASK1)); expectPostRebalanceCatchup(SNAPSHOT); worker.startConnector(EasyMock.eq(CONN1), EasyMock.<Map<String, String>>anyObject(), EasyMock.<ConnectorContext>anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED)); PowerMock.expectLastCall().andReturn(true); EasyMock.expect(worker.isRunning(CONN1)).andReturn(true); EasyMock.expect(worker.connectorTaskConfigs(CONN1, MAX_TASKS, null)).andReturn(TASK_CONFIGS); worker.startTask(EasyMock.eq(TASK1), EasyMock.<Map<String, String>>anyObject(), EasyMock.<Map<String, String>>anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED)); PowerMock.expectLastCall().andReturn(true); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); expectRebalance(Arrays.asList(CONN1), Arrays.asList(TASK1), ConnectProtocol.Assignment.NO_ERROR, 1, Arrays.asList(CONN1), Arrays.<ConnectorTaskId>asList()); // and the new assignment started worker.startConnector(EasyMock.eq(CONN1), EasyMock.<Map<String, String>>anyObject(), EasyMock.<ConnectorContext>anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED)); PowerMock.expectLastCall().andReturn(true); EasyMock.expect(worker.isRunning(CONN1)).andReturn(false); // worker is not running, so we should see no call to connectorTaskConfigs() member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); PowerMock.replayAll(); herder.tick(); herder.tick(); PowerMock.verifyAll(); } @Test public void testHaltCleansUpWorker() { EasyMock.expect(worker.connectorNames()).andReturn(Collections.singleton(CONN1)); worker.stopConnector(CONN1); PowerMock.expectLastCall().andReturn(true); EasyMock.expect(worker.taskIds()).andReturn(Collections.singleton(TASK1)); worker.stopAndAwaitTask(TASK1); PowerMock.expectLastCall(); member.stop(); PowerMock.expectLastCall(); configBackingStore.stop(); PowerMock.expectLastCall(); statusBackingStore.stop(); PowerMock.expectLastCall(); worker.stop(); PowerMock.expectLastCall(); PowerMock.replayAll(); herder.halt(); PowerMock.verifyAll(); } @Test public void testCreateConnector() throws Exception { EasyMock.expect(member.memberId()).andStubReturn("leader"); expectRebalance(1, Collections.<String>emptyList(), Collections.<ConnectorTaskId>emptyList()); expectPostRebalanceCatchup(SNAPSHOT); member.wakeup(); PowerMock.expectLastCall(); // config validation Connector connectorMock = PowerMock.createMock(Connector.class); EasyMock.expect(worker.getPlugins()).andReturn(plugins).times(4); EasyMock.expect(plugins.compareAndSwapLoaders(connectorMock)).andReturn(delegatingLoader); EasyMock.expect(plugins.newConnector(EasyMock.anyString())).andReturn(connectorMock); EasyMock.expect(connectorMock.config()).andReturn(new ConfigDef()); EasyMock.expect(connectorMock.validate(CONN2_CONFIG)).andReturn(new Config(Collections.<ConfigValue>emptyList())); EasyMock.expect(Plugins.compareAndSwapLoaders(delegatingLoader)).andReturn(pluginLoader); // CONN2 is new, should succeed configBackingStore.putConnectorConfig(CONN2, CONN2_CONFIG); PowerMock.expectLastCall(); ConnectorInfo info = new ConnectorInfo(CONN2, CONN2_CONFIG, Collections.<ConnectorTaskId>emptyList()); putConnectorCallback.onCompletion(null, new Herder.Created<>(true, info)); PowerMock.expectLastCall(); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); // No immediate action besides this -- change will be picked up via the config log PowerMock.replayAll(); herder.putConnectorConfig(CONN2, CONN2_CONFIG, false, putConnectorCallback); herder.tick(); PowerMock.verifyAll(); } @Test public void testCreateConnectorFailedBasicValidation() throws Exception { EasyMock.expect(member.memberId()).andStubReturn("leader"); expectRebalance(1, Collections.<String>emptyList(), Collections.<ConnectorTaskId>emptyList()); expectPostRebalanceCatchup(SNAPSHOT); HashMap<String, String> config = new HashMap<>(CONN2_CONFIG); config.remove(ConnectorConfig.NAME_CONFIG); member.wakeup(); PowerMock.expectLastCall(); // config validation Connector connectorMock = PowerMock.createMock(Connector.class); EasyMock.expect(worker.getPlugins()).andReturn(plugins).times(3); EasyMock.expect(plugins.compareAndSwapLoaders(connectorMock)).andReturn(delegatingLoader); EasyMock.expect(plugins.newConnector(EasyMock.anyString())).andReturn(connectorMock); EasyMock.expect(Plugins.compareAndSwapLoaders(delegatingLoader)).andReturn(pluginLoader); // CONN2 creation should fail Capture<Throwable> error = EasyMock.newCapture(); putConnectorCallback.onCompletion(EasyMock.capture(error), EasyMock.<Herder.Created<ConnectorInfo>>isNull()); PowerMock.expectLastCall(); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); // No immediate action besides this -- change will be picked up via the config log PowerMock.replayAll(); herder.putConnectorConfig(CONN2, config, false, putConnectorCallback); herder.tick(); assertTrue(error.hasCaptured()); assertTrue(error.getValue() instanceof BadRequestException); PowerMock.verifyAll(); } @Test public void testCreateConnectorFailedCustomValidation() throws Exception { EasyMock.expect(member.memberId()).andStubReturn("leader"); expectRebalance(1, Collections.<String>emptyList(), Collections.<ConnectorTaskId>emptyList()); expectPostRebalanceCatchup(SNAPSHOT); member.wakeup(); PowerMock.expectLastCall(); // config validation Connector connectorMock = PowerMock.createMock(Connector.class); EasyMock.expect(worker.getPlugins()).andReturn(plugins).times(4); EasyMock.expect(plugins.compareAndSwapLoaders(connectorMock)).andReturn(delegatingLoader); EasyMock.expect(plugins.newConnector(EasyMock.anyString())).andReturn(connectorMock); ConfigDef configDef = new ConfigDef(); configDef.define("foo.bar", ConfigDef.Type.STRING, ConfigDef.Importance.HIGH, "foo.bar doc"); EasyMock.expect(connectorMock.config()).andReturn(configDef); ConfigValue validatedValue = new ConfigValue("foo.bar"); validatedValue.addErrorMessage("Failed foo.bar validation"); EasyMock.expect(connectorMock.validate(CONN2_CONFIG)).andReturn(new Config(singletonList(validatedValue))); EasyMock.expect(Plugins.compareAndSwapLoaders(delegatingLoader)).andReturn(pluginLoader); // CONN2 creation should fail Capture<Throwable> error = EasyMock.newCapture(); putConnectorCallback.onCompletion(EasyMock.capture(error), EasyMock.<Herder.Created<ConnectorInfo>>isNull()); PowerMock.expectLastCall(); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); // No immediate action besides this -- change will be picked up via the config log PowerMock.replayAll(); herder.putConnectorConfig(CONN2, CONN2_CONFIG, false, putConnectorCallback); herder.tick(); assertTrue(error.hasCaptured()); assertTrue(error.getValue() instanceof BadRequestException); PowerMock.verifyAll(); } @Test public void testConnectorNameConflictsWithWorkerGroupId() throws Exception { EasyMock.expect(member.memberId()).andStubReturn("leader"); expectRebalance(1, Collections.<String>emptyList(), Collections.<ConnectorTaskId>emptyList()); expectPostRebalanceCatchup(SNAPSHOT); member.wakeup(); PowerMock.expectLastCall(); Map<String, String> config = new HashMap<>(CONN2_CONFIG); config.put(ConnectorConfig.NAME_CONFIG, "test-group"); // config validation Connector connectorMock = PowerMock.createMock(SinkConnector.class); EasyMock.expect(worker.getPlugins()).andReturn(plugins).times(4); EasyMock.expect(plugins.compareAndSwapLoaders(connectorMock)).andReturn(delegatingLoader); EasyMock.expect(plugins.newConnector(EasyMock.anyString())).andReturn(connectorMock); EasyMock.expect(connectorMock.config()).andReturn(new ConfigDef()); EasyMock.expect(connectorMock.validate(config)).andReturn(new Config(Collections.<ConfigValue>emptyList())); EasyMock.expect(Plugins.compareAndSwapLoaders(delegatingLoader)).andReturn(pluginLoader); // CONN2 creation should fail because the worker group id (connect-test-group) conflicts with // the consumer group id we would use for this sink Capture<Throwable> error = EasyMock.newCapture(); putConnectorCallback.onCompletion(EasyMock.capture(error), EasyMock.isNull(Herder.Created.class)); PowerMock.expectLastCall(); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); // No immediate action besides this -- change will be picked up via the config log PowerMock.replayAll(); herder.putConnectorConfig(CONN2, config, false, putConnectorCallback); herder.tick(); assertTrue(error.hasCaptured()); assertTrue(error.getValue() instanceof BadRequestException); PowerMock.verifyAll(); } @Test public void testCreateConnectorAlreadyExists() throws Exception { EasyMock.expect(member.memberId()).andStubReturn("leader"); expectRebalance(1, Collections.<String>emptyList(), Collections.<ConnectorTaskId>emptyList()); expectPostRebalanceCatchup(SNAPSHOT); member.wakeup(); PowerMock.expectLastCall(); // CONN1 already exists putConnectorCallback.onCompletion(EasyMock.<AlreadyExistsException>anyObject(), EasyMock.<Herder.Created<ConnectorInfo>>isNull()); PowerMock.expectLastCall(); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); // No immediate action besides this -- change will be picked up via the config log PowerMock.replayAll(); herder.putConnectorConfig(CONN1, CONN1_CONFIG, false, putConnectorCallback); herder.tick(); PowerMock.verifyAll(); } @Test public void testDestroyConnector() throws Exception { EasyMock.expect(member.memberId()).andStubReturn("leader"); // Start with one connector EasyMock.expect(worker.getPlugins()).andReturn(plugins); expectRebalance(1, Arrays.asList(CONN1), Collections.<ConnectorTaskId>emptyList()); expectPostRebalanceCatchup(SNAPSHOT); worker.startConnector(EasyMock.eq(CONN1), EasyMock.<Map<String, String>>anyObject(), EasyMock.<ConnectorContext>anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED)); PowerMock.expectLastCall().andReturn(true); EasyMock.expect(worker.isRunning(CONN1)).andReturn(true); EasyMock.expect(worker.connectorTaskConfigs(CONN1, MAX_TASKS, null)).andReturn(TASK_CONFIGS); // And delete the connector member.wakeup(); PowerMock.expectLastCall(); configBackingStore.removeConnectorConfig(CONN1); PowerMock.expectLastCall(); putConnectorCallback.onCompletion(null, new Herder.Created<ConnectorInfo>(false, null)); PowerMock.expectLastCall(); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); // No immediate action besides this -- change will be picked up via the config log PowerMock.replayAll(); herder.deleteConnectorConfig(CONN1, putConnectorCallback); herder.tick(); PowerMock.verifyAll(); } @Test public void testRestartConnector() throws Exception { EasyMock.expect(worker.connectorTaskConfigs(CONN1, MAX_TASKS, null)).andStubReturn(TASK_CONFIGS); // get the initial assignment EasyMock.expect(member.memberId()).andStubReturn("leader"); EasyMock.expect(worker.getPlugins()).andReturn(plugins); expectRebalance(1, singletonList(CONN1), Collections.<ConnectorTaskId>emptyList()); expectPostRebalanceCatchup(SNAPSHOT); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); worker.startConnector(EasyMock.eq(CONN1), EasyMock.<Map<String, String>>anyObject(), EasyMock.<ConnectorContext>anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED)); PowerMock.expectLastCall().andReturn(true); EasyMock.expect(worker.isRunning(CONN1)).andReturn(true); // now handle the connector restart member.wakeup(); PowerMock.expectLastCall(); member.ensureActive(); PowerMock.expectLastCall(); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); worker.stopConnector(CONN1); PowerMock.expectLastCall().andReturn(true); EasyMock.expect(worker.getPlugins()).andReturn(plugins); worker.startConnector(EasyMock.eq(CONN1), EasyMock.<Map<String, String>>anyObject(), EasyMock.<ConnectorContext>anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED)); PowerMock.expectLastCall().andReturn(true); EasyMock.expect(worker.isRunning(CONN1)).andReturn(true); PowerMock.replayAll(); herder.tick(); FutureCallback<Void> callback = new FutureCallback<>(); herder.restartConnector(CONN1, callback); herder.tick(); callback.get(1000L, TimeUnit.MILLISECONDS); PowerMock.verifyAll(); } @Test public void testRestartUnknownConnector() throws Exception { // get the initial assignment EasyMock.expect(member.memberId()).andStubReturn("leader"); expectRebalance(1, Collections.<String>emptyList(), Collections.<ConnectorTaskId>emptyList()); expectPostRebalanceCatchup(SNAPSHOT); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); // now handle the connector restart member.wakeup(); PowerMock.expectLastCall(); member.ensureActive(); PowerMock.expectLastCall(); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); PowerMock.replayAll(); herder.tick(); FutureCallback<Void> callback = new FutureCallback<>(); herder.restartConnector(CONN2, callback); herder.tick(); try { callback.get(1000L, TimeUnit.MILLISECONDS); fail("Expected NotLeaderException to be raised"); } catch (ExecutionException e) { assertTrue(e.getCause() instanceof NotFoundException); } PowerMock.verifyAll(); } @Test public void testRestartConnectorRedirectToLeader() throws Exception { // get the initial assignment EasyMock.expect(member.memberId()).andStubReturn("member"); expectRebalance(1, Collections.<String>emptyList(), Collections.<ConnectorTaskId>emptyList()); expectPostRebalanceCatchup(SNAPSHOT); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); // now handle the connector restart member.wakeup(); PowerMock.expectLastCall(); member.ensureActive(); PowerMock.expectLastCall(); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); PowerMock.replayAll(); herder.tick(); FutureCallback<Void> callback = new FutureCallback<>(); herder.restartConnector(CONN1, callback); herder.tick(); try { callback.get(1000L, TimeUnit.MILLISECONDS); fail("Expected NotLeaderException to be raised"); } catch (ExecutionException e) { assertTrue(e.getCause() instanceof NotLeaderException); } PowerMock.verifyAll(); } @Test public void testRestartConnectorRedirectToOwner() throws Exception { // get the initial assignment EasyMock.expect(member.memberId()).andStubReturn("leader"); expectRebalance(1, Collections.<String>emptyList(), Collections.<ConnectorTaskId>emptyList()); expectPostRebalanceCatchup(SNAPSHOT); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); // now handle the connector restart member.wakeup(); PowerMock.expectLastCall(); member.ensureActive(); PowerMock.expectLastCall(); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); String ownerUrl = "ownerUrl"; EasyMock.expect(member.ownerUrl(CONN1)).andReturn(ownerUrl); PowerMock.replayAll(); herder.tick(); FutureCallback<Void> callback = new FutureCallback<>(); herder.restartConnector(CONN1, callback); herder.tick(); try { callback.get(1000L, TimeUnit.MILLISECONDS); fail("Expected NotLeaderException to be raised"); } catch (ExecutionException e) { assertTrue(e.getCause() instanceof NotAssignedException); NotAssignedException notAssignedException = (NotAssignedException) e.getCause(); assertEquals(ownerUrl, notAssignedException.forwardUrl()); } PowerMock.verifyAll(); } @Test public void testRestartTask() throws Exception { EasyMock.expect(worker.connectorTaskConfigs(CONN1, MAX_TASKS, null)).andStubReturn(TASK_CONFIGS); // get the initial assignment EasyMock.expect(member.memberId()).andStubReturn("leader"); expectRebalance(1, Collections.<String>emptyList(), singletonList(TASK0)); expectPostRebalanceCatchup(SNAPSHOT); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); worker.startTask(EasyMock.eq(TASK0), EasyMock.<Map<String, String>>anyObject(), EasyMock.<Map<String, String>>anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED)); PowerMock.expectLastCall().andReturn(true); // now handle the task restart member.wakeup(); PowerMock.expectLastCall(); member.ensureActive(); PowerMock.expectLastCall(); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); worker.stopAndAwaitTask(TASK0); PowerMock.expectLastCall(); worker.startTask(EasyMock.eq(TASK0), EasyMock.<Map<String, String>>anyObject(), EasyMock.<Map<String, String>>anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED)); PowerMock.expectLastCall().andReturn(true); PowerMock.replayAll(); herder.tick(); FutureCallback<Void> callback = new FutureCallback<>(); herder.restartTask(TASK0, callback); herder.tick(); callback.get(1000L, TimeUnit.MILLISECONDS); PowerMock.verifyAll(); } @Test public void testRestartUnknownTask() throws Exception { // get the initial assignment EasyMock.expect(member.memberId()).andStubReturn("member"); expectRebalance(1, Collections.<String>emptyList(), Collections.<ConnectorTaskId>emptyList()); expectPostRebalanceCatchup(SNAPSHOT); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); member.wakeup(); PowerMock.expectLastCall(); member.ensureActive(); PowerMock.expectLastCall(); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); PowerMock.replayAll(); FutureCallback<Void> callback = new FutureCallback<>(); herder.tick(); herder.restartTask(new ConnectorTaskId("blah", 0), callback); herder.tick(); try { callback.get(1000L, TimeUnit.MILLISECONDS); fail("Expected NotLeaderException to be raised"); } catch (ExecutionException e) { assertTrue(e.getCause() instanceof NotFoundException); } PowerMock.verifyAll(); } @Test public void testRequestProcessingOrder() throws Exception { final DistributedHerder.HerderRequest req1 = herder.addRequest(100, null, null); final DistributedHerder.HerderRequest req2 = herder.addRequest(10, null, null); final DistributedHerder.HerderRequest req3 = herder.addRequest(200, null, null); final DistributedHerder.HerderRequest req4 = herder.addRequest(200, null, null); assertEquals(req2, herder.requests.pollFirst()); // lowest delay assertEquals(req1, herder.requests.pollFirst()); // next lowest delay assertEquals(req3, herder.requests.pollFirst()); // same delay as req4, but added first assertEquals(req4, herder.requests.pollFirst()); } @Test public void testRestartTaskRedirectToLeader() throws Exception { // get the initial assignment EasyMock.expect(member.memberId()).andStubReturn("member"); expectRebalance(1, Collections.<String>emptyList(), Collections.<ConnectorTaskId>emptyList()); expectPostRebalanceCatchup(SNAPSHOT); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); // now handle the task restart member.wakeup(); PowerMock.expectLastCall(); member.ensureActive(); PowerMock.expectLastCall(); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); PowerMock.replayAll(); herder.tick(); FutureCallback<Void> callback = new FutureCallback<>(); herder.restartTask(TASK0, callback); herder.tick(); try { callback.get(1000L, TimeUnit.MILLISECONDS); fail("Expected NotLeaderException to be raised"); } catch (ExecutionException e) { assertTrue(e.getCause() instanceof NotLeaderException); } PowerMock.verifyAll(); } @Test public void testRestartTaskRedirectToOwner() throws Exception { // get the initial assignment EasyMock.expect(member.memberId()).andStubReturn("leader"); expectRebalance(1, Collections.<String>emptyList(), Collections.<ConnectorTaskId>emptyList()); expectPostRebalanceCatchup(SNAPSHOT); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); // now handle the task restart String ownerUrl = "ownerUrl"; EasyMock.expect(member.ownerUrl(TASK0)).andReturn(ownerUrl); member.wakeup(); PowerMock.expectLastCall(); member.ensureActive(); PowerMock.expectLastCall(); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); PowerMock.replayAll(); herder.tick(); FutureCallback<Void> callback = new FutureCallback<>(); herder.restartTask(TASK0, callback); herder.tick(); try { callback.get(1000L, TimeUnit.MILLISECONDS); fail("Expected NotLeaderException to be raised"); } catch (ExecutionException e) { assertTrue(e.getCause() instanceof NotAssignedException); NotAssignedException notAssignedException = (NotAssignedException) e.getCause(); assertEquals(ownerUrl, notAssignedException.forwardUrl()); } PowerMock.verifyAll(); } @Test public void testConnectorConfigAdded() { // If a connector was added, we need to rebalance EasyMock.expect(member.memberId()).andStubReturn("member"); // join, no configs so no need to catch up on config topic expectRebalance(-1, Collections.<String>emptyList(), Collections.<ConnectorTaskId>emptyList()); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); // apply config member.wakeup(); member.ensureActive(); PowerMock.expectLastCall(); // Checks for config updates and starts rebalance EasyMock.expect(configBackingStore.snapshot()).andReturn(SNAPSHOT); member.requestRejoin(); PowerMock.expectLastCall(); // Performs rebalance and gets new assignment expectRebalance(Collections.<String>emptyList(), Collections.<ConnectorTaskId>emptyList(), ConnectProtocol.Assignment.NO_ERROR, 1, Arrays.asList(CONN1), Collections.<ConnectorTaskId>emptyList()); worker.startConnector(EasyMock.eq(CONN1), EasyMock.<Map<String, String>>anyObject(), EasyMock.<ConnectorContext>anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED)); PowerMock.expectLastCall().andReturn(true); EasyMock.expect(worker.getPlugins()).andReturn(plugins); EasyMock.expect(worker.isRunning(CONN1)).andReturn(true); EasyMock.expect(worker.connectorTaskConfigs(CONN1, MAX_TASKS, null)).andReturn(TASK_CONFIGS); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); PowerMock.replayAll(); herder.tick(); // join configUpdateListener.onConnectorConfigUpdate(CONN1); // read updated config herder.tick(); // apply config herder.tick(); // do rebalance PowerMock.verifyAll(); } @Test public void testConnectorConfigUpdate() throws Exception { // Connector config can be applied without any rebalance EasyMock.expect(member.memberId()).andStubReturn("member"); EasyMock.expect(worker.connectorNames()).andStubReturn(Collections.singleton(CONN1)); // join expectRebalance(1, Arrays.asList(CONN1), Collections.<ConnectorTaskId>emptyList()); expectPostRebalanceCatchup(SNAPSHOT); worker.startConnector(EasyMock.eq(CONN1), EasyMock.<Map<String, String>>anyObject(), EasyMock.<ConnectorContext>anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED)); PowerMock.expectLastCall().andReturn(true); EasyMock.expect(worker.getPlugins()).andReturn(plugins); EasyMock.expect(worker.isRunning(CONN1)).andReturn(true); EasyMock.expect(worker.connectorTaskConfigs(CONN1, MAX_TASKS, null)).andReturn(TASK_CONFIGS); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); // apply config member.wakeup(); member.ensureActive(); PowerMock.expectLastCall(); EasyMock.expect(configBackingStore.snapshot()).andReturn(SNAPSHOT); // for this test, it doesn't matter if we use the same config snapshot worker.stopConnector(CONN1); PowerMock.expectLastCall().andReturn(true); worker.startConnector(EasyMock.eq(CONN1), EasyMock.<Map<String, String>>anyObject(), EasyMock.<ConnectorContext>anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED)); PowerMock.expectLastCall().andReturn(true); EasyMock.expect(worker.getPlugins()).andReturn(plugins); EasyMock.expect(worker.isRunning(CONN1)).andReturn(true); EasyMock.expect(worker.connectorTaskConfigs(CONN1, MAX_TASKS, null)).andReturn(TASK_CONFIGS); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); PowerMock.replayAll(); herder.tick(); // join configUpdateListener.onConnectorConfigUpdate(CONN1); // read updated config herder.tick(); // apply config PowerMock.verifyAll(); } @Test public void testConnectorPaused() throws Exception { // ensure that target state changes are propagated to the worker EasyMock.expect(member.memberId()).andStubReturn("member"); EasyMock.expect(worker.connectorNames()).andStubReturn(Collections.singleton(CONN1)); // join expectRebalance(1, Arrays.asList(CONN1), Collections.<ConnectorTaskId>emptyList()); expectPostRebalanceCatchup(SNAPSHOT); worker.startConnector(EasyMock.eq(CONN1), EasyMock.<Map<String, String>>anyObject(), EasyMock.<ConnectorContext>anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED)); PowerMock.expectLastCall().andReturn(true); EasyMock.expect(worker.getPlugins()).andReturn(plugins); EasyMock.expect(worker.isRunning(CONN1)).andReturn(true); EasyMock.expect(worker.connectorTaskConfigs(CONN1, MAX_TASKS, null)).andReturn(TASK_CONFIGS); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); // handle the state change member.wakeup(); member.ensureActive(); PowerMock.expectLastCall(); EasyMock.expect(configBackingStore.snapshot()).andReturn(SNAPSHOT_PAUSED_CONN1); PowerMock.expectLastCall(); worker.setTargetState(CONN1, TargetState.PAUSED); PowerMock.expectLastCall(); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); PowerMock.replayAll(); herder.tick(); // join configUpdateListener.onConnectorTargetStateChange(CONN1); // state changes to paused herder.tick(); // worker should apply the state change PowerMock.verifyAll(); } @Test public void testConnectorResumed() throws Exception { EasyMock.expect(member.memberId()).andStubReturn("member"); EasyMock.expect(worker.connectorNames()).andStubReturn(Collections.singleton(CONN1)); // start with the connector paused expectRebalance(1, Arrays.asList(CONN1), Collections.<ConnectorTaskId>emptyList()); expectPostRebalanceCatchup(SNAPSHOT_PAUSED_CONN1); worker.startConnector(EasyMock.eq(CONN1), EasyMock.<Map<String, String>>anyObject(), EasyMock.<ConnectorContext>anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.PAUSED)); PowerMock.expectLastCall().andReturn(true); EasyMock.expect(worker.getPlugins()).andReturn(plugins); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); // handle the state change member.wakeup(); member.ensureActive(); PowerMock.expectLastCall(); EasyMock.expect(configBackingStore.snapshot()).andReturn(SNAPSHOT); PowerMock.expectLastCall(); // we expect reconfiguration after resuming EasyMock.expect(worker.isRunning(CONN1)).andReturn(true); EasyMock.expect(worker.connectorTaskConfigs(CONN1, MAX_TASKS, null)).andReturn(TASK_CONFIGS); worker.setTargetState(CONN1, TargetState.STARTED); PowerMock.expectLastCall(); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); PowerMock.replayAll(); herder.tick(); // join configUpdateListener.onConnectorTargetStateChange(CONN1); // state changes to started herder.tick(); // apply state change PowerMock.verifyAll(); } @Test public void testUnknownConnectorPaused() throws Exception { EasyMock.expect(member.memberId()).andStubReturn("member"); EasyMock.expect(worker.connectorNames()).andStubReturn(Collections.singleton(CONN1)); // join expectRebalance(1, Collections.<String>emptyList(), singletonList(TASK0)); expectPostRebalanceCatchup(SNAPSHOT); worker.startTask(EasyMock.eq(TASK0), EasyMock.<Map<String, String>>anyObject(), EasyMock.<Map<String, String>>anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED)); PowerMock.expectLastCall().andReturn(true); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); // state change is ignored since we have no target state member.wakeup(); member.ensureActive(); PowerMock.expectLastCall(); EasyMock.expect(configBackingStore.snapshot()).andReturn(SNAPSHOT); PowerMock.expectLastCall(); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); PowerMock.replayAll(); herder.tick(); // join configUpdateListener.onConnectorTargetStateChange("unknown-connector"); herder.tick(); // continue PowerMock.verifyAll(); } @Test public void testConnectorPausedRunningTaskOnly() throws Exception { // even if we don't own the connector, we should still propagate target state // changes to the worker so that tasks will transition correctly EasyMock.expect(member.memberId()).andStubReturn("member"); EasyMock.expect(worker.connectorNames()).andStubReturn(Collections.<String>emptySet()); // join expectRebalance(1, Collections.<String>emptyList(), singletonList(TASK0)); expectPostRebalanceCatchup(SNAPSHOT); worker.startTask(EasyMock.eq(TASK0), EasyMock.<Map<String, String>>anyObject(), EasyMock.<Map<String, String>>anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED)); PowerMock.expectLastCall().andReturn(true); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); // handle the state change member.wakeup(); member.ensureActive(); PowerMock.expectLastCall(); EasyMock.expect(configBackingStore.snapshot()).andReturn(SNAPSHOT_PAUSED_CONN1); PowerMock.expectLastCall(); worker.setTargetState(CONN1, TargetState.PAUSED); PowerMock.expectLastCall(); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); PowerMock.replayAll(); herder.tick(); // join configUpdateListener.onConnectorTargetStateChange(CONN1); // state changes to paused herder.tick(); // apply state change PowerMock.verifyAll(); } @Test public void testConnectorResumedRunningTaskOnly() throws Exception { // even if we don't own the connector, we should still propagate target state // changes to the worker so that tasks will transition correctly EasyMock.expect(member.memberId()).andStubReturn("member"); EasyMock.expect(worker.connectorNames()).andStubReturn(Collections.<String>emptySet()); // join expectRebalance(1, Collections.<String>emptyList(), singletonList(TASK0)); expectPostRebalanceCatchup(SNAPSHOT_PAUSED_CONN1); worker.startTask(EasyMock.eq(TASK0), EasyMock.<Map<String, String>>anyObject(), EasyMock.<Map<String, String>>anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.PAUSED)); PowerMock.expectLastCall().andReturn(true); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); // handle the state change member.wakeup(); member.ensureActive(); PowerMock.expectLastCall(); EasyMock.expect(configBackingStore.snapshot()).andReturn(SNAPSHOT); PowerMock.expectLastCall(); worker.setTargetState(CONN1, TargetState.STARTED); PowerMock.expectLastCall(); EasyMock.expect(worker.isRunning(CONN1)).andReturn(false); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); PowerMock.replayAll(); herder.tick(); // join configUpdateListener.onConnectorTargetStateChange(CONN1); // state changes to paused herder.tick(); // apply state change PowerMock.verifyAll(); } @Test public void testTaskConfigAdded() { // Task config always requires rebalance EasyMock.expect(member.memberId()).andStubReturn("member"); // join expectRebalance(-1, Collections.<String>emptyList(), Collections.<ConnectorTaskId>emptyList()); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); // apply config member.wakeup(); member.ensureActive(); PowerMock.expectLastCall(); // Checks for config updates and starts rebalance EasyMock.expect(configBackingStore.snapshot()).andReturn(SNAPSHOT); member.requestRejoin(); PowerMock.expectLastCall(); // Performs rebalance and gets new assignment expectRebalance(Collections.<String>emptyList(), Collections.<ConnectorTaskId>emptyList(), ConnectProtocol.Assignment.NO_ERROR, 1, Collections.<String>emptyList(), Arrays.asList(TASK0)); worker.startTask(EasyMock.eq(TASK0), EasyMock.<Map<String, String>>anyObject(), EasyMock.<Map<String, String>>anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED)); PowerMock.expectLastCall().andReturn(true); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); PowerMock.replayAll(); herder.tick(); // join configUpdateListener.onTaskConfigUpdate(Arrays.asList(TASK0, TASK1, TASK2)); // read updated config herder.tick(); // apply config herder.tick(); // do rebalance PowerMock.verifyAll(); } @Test public void testJoinLeaderCatchUpFails() throws Exception { // Join group and as leader fail to do assignment EasyMock.expect(member.memberId()).andStubReturn("leader"); expectRebalance(Collections.<String>emptyList(), Collections.<ConnectorTaskId>emptyList(), ConnectProtocol.Assignment.CONFIG_MISMATCH, 1, Collections.<String>emptyList(), Collections.<ConnectorTaskId>emptyList()); // Reading to end of log times out configBackingStore.refresh(EasyMock.anyLong(), EasyMock.anyObject(TimeUnit.class)); EasyMock.expectLastCall().andThrow(new TimeoutException()); member.maybeLeaveGroup(); EasyMock.expectLastCall(); PowerMock.expectPrivate(herder, "backoff", DistributedConfig.WORKER_UNSYNC_BACKOFF_MS_DEFAULT); member.requestRejoin(); // After backoff, restart the process and this time succeed expectRebalance(1, Arrays.asList(CONN1), Arrays.asList(TASK1)); expectPostRebalanceCatchup(SNAPSHOT); worker.startConnector(EasyMock.eq(CONN1), EasyMock.<Map<String, String>>anyObject(), EasyMock.<ConnectorContext>anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED)); PowerMock.expectLastCall().andReturn(true); EasyMock.expect(worker.getPlugins()).andReturn(plugins); EasyMock.expect(worker.connectorTaskConfigs(CONN1, MAX_TASKS, null)).andReturn(TASK_CONFIGS); worker.startTask(EasyMock.eq(TASK1), EasyMock.<Map<String, String>>anyObject(), EasyMock.<Map<String, String>>anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED)); PowerMock.expectLastCall().andReturn(true); EasyMock.expect(worker.isRunning(CONN1)).andReturn(true); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); PowerMock.replayAll(); herder.tick(); herder.tick(); PowerMock.verifyAll(); } @Test public void testAccessors() throws Exception { EasyMock.expect(member.memberId()).andStubReturn("leader"); expectRebalance(1, Collections.<String>emptyList(), Collections.<ConnectorTaskId>emptyList()); expectPostRebalanceCatchup(SNAPSHOT); member.wakeup(); PowerMock.expectLastCall().anyTimes(); // list connectors, get connector info, get connector config, get task configs member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); PowerMock.replayAll(); FutureCallback<Collection<String>> listConnectorsCb = new FutureCallback<>(); herder.connectors(listConnectorsCb); FutureCallback<ConnectorInfo> connectorInfoCb = new FutureCallback<>(); herder.connectorInfo(CONN1, connectorInfoCb); FutureCallback<Map<String, String>> connectorConfigCb = new FutureCallback<>(); herder.connectorConfig(CONN1, connectorConfigCb); FutureCallback<List<TaskInfo>> taskConfigsCb = new FutureCallback<>(); herder.taskConfigs(CONN1, taskConfigsCb); herder.tick(); assertTrue(listConnectorsCb.isDone()); assertEquals(Collections.singleton(CONN1), listConnectorsCb.get()); assertTrue(connectorInfoCb.isDone()); ConnectorInfo info = new ConnectorInfo(CONN1, CONN1_CONFIG, Arrays.asList(TASK0, TASK1, TASK2)); assertEquals(info, connectorInfoCb.get()); assertTrue(connectorConfigCb.isDone()); assertEquals(CONN1_CONFIG, connectorConfigCb.get()); assertTrue(taskConfigsCb.isDone()); assertEquals(Arrays.asList( new TaskInfo(TASK0, TASK_CONFIG), new TaskInfo(TASK1, TASK_CONFIG), new TaskInfo(TASK2, TASK_CONFIG)), taskConfigsCb.get()); PowerMock.verifyAll(); } @Test public void testPutConnectorConfig() throws Exception { EasyMock.expect(member.memberId()).andStubReturn("leader"); expectRebalance(1, Arrays.asList(CONN1), Collections.<ConnectorTaskId>emptyList()); expectPostRebalanceCatchup(SNAPSHOT); worker.startConnector(EasyMock.eq(CONN1), EasyMock.<Map<String, String>>anyObject(), EasyMock.<ConnectorContext>anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED)); PowerMock.expectLastCall().andReturn(true); EasyMock.expect(worker.isRunning(CONN1)).andReturn(true); EasyMock.expect(worker.connectorTaskConfigs(CONN1, MAX_TASKS, null)).andReturn(TASK_CONFIGS); // list connectors, get connector info, get connector config, get task configs member.wakeup(); PowerMock.expectLastCall().anyTimes(); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); // Poll loop for second round of calls member.ensureActive(); PowerMock.expectLastCall(); // config validation Connector connectorMock = PowerMock.createMock(Connector.class); EasyMock.expect(worker.getPlugins()).andReturn(plugins).times(6); EasyMock.expect(plugins.compareAndSwapLoaders(connectorMock)).andReturn(delegatingLoader); EasyMock.expect(plugins.newConnector(EasyMock.anyString())).andReturn(connectorMock); EasyMock.expect(connectorMock.config()).andReturn(new ConfigDef()); EasyMock.expect(connectorMock.validate(CONN1_CONFIG_UPDATED)).andReturn(new Config(Collections.<ConfigValue>emptyList())); EasyMock.expect(Plugins.compareAndSwapLoaders(delegatingLoader)).andReturn(pluginLoader); configBackingStore.putConnectorConfig(CONN1, CONN1_CONFIG_UPDATED); PowerMock.expectLastCall().andAnswer(new IAnswer<Object>() { @Override public Object answer() throws Throwable { // Simulate response to writing config + waiting until end of log to be read configUpdateListener.onConnectorConfigUpdate(CONN1); return null; } }); // As a result of reconfig, should need to update snapshot. With only connector updates, we'll just restart // connector without rebalance EasyMock.expect(configBackingStore.snapshot()).andReturn(SNAPSHOT_UPDATED_CONN1_CONFIG); worker.stopConnector(CONN1); PowerMock.expectLastCall().andReturn(true); worker.startConnector(EasyMock.eq(CONN1), EasyMock.<Map<String, String>>anyObject(), EasyMock.<ConnectorContext>anyObject(), EasyMock.eq(herder), EasyMock.eq(TargetState.STARTED)); PowerMock.expectLastCall().andReturn(true); EasyMock.expect(worker.isRunning(CONN1)).andReturn(true); EasyMock.expect(worker.connectorTaskConfigs(CONN1, MAX_TASKS, null)).andReturn(TASK_CONFIGS); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); // Third tick just to read the config member.ensureActive(); PowerMock.expectLastCall(); member.poll(EasyMock.anyInt()); PowerMock.expectLastCall(); PowerMock.replayAll(); // Should pick up original config FutureCallback<Map<String, String>> connectorConfigCb = new FutureCallback<>(); herder.connectorConfig(CONN1, connectorConfigCb); herder.tick(); assertTrue(connectorConfigCb.isDone()); assertEquals(CONN1_CONFIG, connectorConfigCb.get()); // Apply new config. FutureCallback<Herder.Created<ConnectorInfo>> putConfigCb = new FutureCallback<>(); herder.putConnectorConfig(CONN1, CONN1_CONFIG_UPDATED, true, putConfigCb); herder.tick(); assertTrue(putConfigCb.isDone()); ConnectorInfo updatedInfo = new ConnectorInfo(CONN1, CONN1_CONFIG_UPDATED, Arrays.asList(TASK0, TASK1, TASK2)); assertEquals(new Herder.Created<>(false, updatedInfo), putConfigCb.get()); // Check config again to validate change connectorConfigCb = new FutureCallback<>(); herder.connectorConfig(CONN1, connectorConfigCb); herder.tick(); assertTrue(connectorConfigCb.isDone()); assertEquals(CONN1_CONFIG_UPDATED, connectorConfigCb.get()); PowerMock.verifyAll(); } @Test public void testInconsistentConfigs() throws Exception { // FIXME: if we have inconsistent configs, we need to request forced reconfig + write of the connector's task configs // This requires inter-worker communication, so needs the REST API } private void expectRebalance(final long offset, final List<String> assignedConnectors, final List<ConnectorTaskId> assignedTasks) { expectRebalance(null, null, ConnectProtocol.Assignment.NO_ERROR, offset, assignedConnectors, assignedTasks); } // Handles common initial part of rebalance callback. Does not handle instantiation of connectors and tasks. private void expectRebalance(final Collection<String> revokedConnectors, final List<ConnectorTaskId> revokedTasks, final short error, final long offset, final List<String> assignedConnectors, final List<ConnectorTaskId> assignedTasks) { member.ensureActive(); PowerMock.expectLastCall().andAnswer(new IAnswer<Object>() { @Override public Object answer() throws Throwable { if (revokedConnectors != null) rebalanceListener.onRevoked("leader", revokedConnectors, revokedTasks); ConnectProtocol.Assignment assignment = new ConnectProtocol.Assignment( error, "leader", "leaderUrl", offset, assignedConnectors, assignedTasks); rebalanceListener.onAssigned(assignment, 0); return null; } }); if (revokedConnectors != null) { for (String connector : revokedConnectors) { worker.stopConnector(connector); PowerMock.expectLastCall().andReturn(true); } } if (revokedTasks != null && !revokedTasks.isEmpty()) { worker.stopAndAwaitTask(EasyMock.anyObject(ConnectorTaskId.class)); PowerMock.expectLastCall(); } if (revokedConnectors != null) { statusBackingStore.flush(); PowerMock.expectLastCall(); } member.wakeup(); PowerMock.expectLastCall(); } private void expectPostRebalanceCatchup(final ClusterConfigState readToEndSnapshot) throws TimeoutException { configBackingStore.refresh(EasyMock.anyLong(), EasyMock.anyObject(TimeUnit.class)); EasyMock.expectLastCall(); EasyMock.expect(configBackingStore.snapshot()).andReturn(readToEndSnapshot); } // We need to use a real class here due to some issue with mocking java.lang.Class private abstract class BogusSourceConnector extends SourceConnector { } private abstract class BogusSourceTask extends SourceTask { } }