/*
Copyright 2015 Red Hat, Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package org.jboss.as.test.integration.domain.suites;
import static org.jboss.as.controller.descriptions.ModelDescriptionConstants.ACTIVE_OPERATION;
import static org.jboss.as.controller.descriptions.ModelDescriptionConstants.CHILD_TYPE;
import static org.jboss.as.controller.descriptions.ModelDescriptionConstants.CORE_SERVICE;
import static org.jboss.as.controller.descriptions.ModelDescriptionConstants.DOMAIN_FAILURE_DESCRIPTION;
import static org.jboss.as.controller.descriptions.ModelDescriptionConstants.EXTENSION;
import static org.jboss.as.controller.descriptions.ModelDescriptionConstants.FAILED;
import static org.jboss.as.controller.descriptions.ModelDescriptionConstants.FAILURE_DESCRIPTION;
import static org.jboss.as.controller.descriptions.ModelDescriptionConstants.HOST;
import static org.jboss.as.controller.descriptions.ModelDescriptionConstants.HOST_FAILURE_DESCRIPTIONS;
import static org.jboss.as.controller.descriptions.ModelDescriptionConstants.IN_SERIES;
import static org.jboss.as.controller.descriptions.ModelDescriptionConstants.MANAGEMENT;
import static org.jboss.as.controller.descriptions.ModelDescriptionConstants.MANAGEMENT_OPERATIONS;
import static org.jboss.as.controller.descriptions.ModelDescriptionConstants.MAX_FAILED_SERVERS;
import static org.jboss.as.controller.descriptions.ModelDescriptionConstants.NAME;
import static org.jboss.as.controller.descriptions.ModelDescriptionConstants.OP;
import static org.jboss.as.controller.descriptions.ModelDescriptionConstants.OPERATION_HEADERS;
import static org.jboss.as.controller.descriptions.ModelDescriptionConstants.OUTCOME;
import static org.jboss.as.controller.descriptions.ModelDescriptionConstants.PROFILE;
import static org.jboss.as.controller.descriptions.ModelDescriptionConstants.READ_ATTRIBUTE_OPERATION;
import static org.jboss.as.controller.descriptions.ModelDescriptionConstants.READ_CHILDREN_RESOURCES_OPERATION;
import static org.jboss.as.controller.descriptions.ModelDescriptionConstants.REMOVE;
import static org.jboss.as.controller.descriptions.ModelDescriptionConstants.RESPONSE;
import static org.jboss.as.controller.descriptions.ModelDescriptionConstants.RESULT;
import static org.jboss.as.controller.descriptions.ModelDescriptionConstants.ROLLOUT_PLAN;
import static org.jboss.as.controller.descriptions.ModelDescriptionConstants.SERVER;
import static org.jboss.as.controller.descriptions.ModelDescriptionConstants.SERVER_GROUP;
import static org.jboss.as.controller.descriptions.ModelDescriptionConstants.SERVER_GROUPS;
import static org.jboss.as.controller.descriptions.ModelDescriptionConstants.SERVICE;
import static org.jboss.as.controller.descriptions.ModelDescriptionConstants.SUBSYSTEM;
import static org.jboss.as.controller.descriptions.ModelDescriptionConstants.SUCCESS;
import static org.jboss.as.test.integration.management.extension.blocker.BlockerExtension.CALLER;
import static org.jboss.as.test.integration.management.extension.blocker.BlockerExtension.TARGET_HOST;
import static org.jboss.as.test.integration.management.extension.blocker.BlockerExtension.TARGET_SERVER;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import org.jboss.as.controller.PathAddress;
import org.jboss.as.controller.PathElement;
import org.jboss.as.controller.client.ModelControllerClient;
import org.jboss.as.controller.client.OperationMessageHandler;
import org.jboss.as.controller.client.helpers.domain.DomainClient;
import org.jboss.as.controller.operations.common.Util;
import org.jboss.as.test.integration.domain.extension.ExtensionSetup;
import org.jboss.as.test.integration.domain.management.util.DomainTestSupport;
import org.jboss.as.test.integration.management.extension.error.ErrorExtension;
import org.jboss.as.test.integration.management.util.MgmtOperationException;
import org.jboss.as.test.shared.TimeoutUtil;
import org.jboss.dmr.ModelNode;
import org.jboss.dmr.ModelType;
import org.jboss.dmr.Property;
import org.jboss.logging.Logger;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
/**
* Test of handling of java.lang.Error thrown while running operations in a domain.
*
* @author Brian Stansberry (c) 2015 Red Hat Inc.
*/
public class OperationErrorTestCase {
private static final Logger log = Logger.getLogger(OperationErrorTestCase.class);
private static final PathAddress SUBSYSTEM_ADDRESS = PathAddress.pathAddress(
PathElement.pathElement(PROFILE, "default"),
PathElement.pathElement(SUBSYSTEM, ErrorExtension.SUBSYSTEM_NAME));
private static final ModelNode ERROR_OP = Util.createEmptyOperation("error", SUBSYSTEM_ADDRESS);
private static final PathAddress MGMT_CONTROLLER = PathAddress.pathAddress(
PathElement.pathElement(CORE_SERVICE, MANAGEMENT),
PathElement.pathElement(SERVICE, MANAGEMENT_OPERATIONS)
);
private static final long GET_TIMEOUT = TimeoutUtil.adjust(10000);
private static final ModelNode ROLLOUT_HEADER;
static {
ROLLOUT_HEADER = new ModelNode();
ModelNode groupHeader = ROLLOUT_HEADER.get(ROLLOUT_PLAN, IN_SERIES).add();
groupHeader.get(SERVER_GROUP, "main-server-group", MAX_FAILED_SERVERS).set(1);
ROLLOUT_HEADER.protect();
}
private static DomainTestSupport testSupport;
private static DomainClient masterClient;
@BeforeClass
public static void setupDomain() throws Exception {
testSupport = DomainTestSuite.createSupport(OperationCancellationTestCase.class.getSimpleName());
masterClient = testSupport.getDomainMasterLifecycleUtil().getDomainClient();
// Initialize the test extension
ExtensionSetup.initializeErrorExtension(testSupport);
ModelNode addExtension = Util.createAddOperation(PathAddress.pathAddress(PathElement.pathElement(EXTENSION, ErrorExtension.MODULE_NAME)));
executeForResponse(addExtension, masterClient);
ModelNode addSubsystem = Util.createAddOperation(PathAddress.pathAddress(
PathElement.pathElement(PROFILE, "default"),
PathElement.pathElement(SUBSYSTEM, ErrorExtension.SUBSYSTEM_NAME)));
executeForResponse(addSubsystem, masterClient);
}
@AfterClass
public static void tearDownDomain() throws Exception {
ModelNode removeSubsystem = Util.createEmptyOperation(REMOVE, PathAddress.pathAddress(
PathElement.pathElement(PROFILE, "default"),
PathElement.pathElement(SUBSYSTEM, ErrorExtension.SUBSYSTEM_NAME)));
executeForResponse(removeSubsystem, masterClient);
ModelNode removeExtension = Util.createEmptyOperation(REMOVE, PathAddress.pathAddress(PathElement.pathElement(EXTENSION, ErrorExtension.MODULE_NAME)));
executeForResponse(removeExtension, masterClient);
testSupport = null;
masterClient = null;
DomainTestSuite.stopSupport();
}
@SuppressWarnings("ThrowableResultOfMethodCallIgnored")
@After
public void awaitCompletion() throws Exception {
// Start from the leaves of the domain process tree and work inward validating
// that all error ops are cleared locally. This ensures that a later test doesn't
// mistakenly cancel a completing op from an earlier test
validateNoActiveOperation(masterClient, "master", "main-one");
validateNoActiveOperation(masterClient, "slave", "main-three");
validateNoActiveOperation(masterClient, "slave", null);
MgmtOperationException moe = validateNoActiveOperation(masterClient, "master", null);
if (moe != null) {
throw moe;
}
}
@Test
public void testMasterServerStageModel() throws Exception {
// The server fails in pre-prepare so whether overall outcome=failed depends on the rollout plan
errorTest("master", "main-one", ErrorExtension.ErrorPoint.MODEL, false, true);
errorTest("master", "main-one", ErrorExtension.ErrorPoint.MODEL, true, false);
}
@Test
public void testSlaveServerStageModel() throws Exception {
// The server fails in pre-prepare so whether overall outcome=failed depends on the rollout plan
errorTest("slave", "main-three", ErrorExtension.ErrorPoint.MODEL, false, true);
errorTest("slave", "main-three", ErrorExtension.ErrorPoint.MODEL, true, false);
}
@Test
public void testMasterHCStageModel() throws Exception {
// A pre-prepare failure on an HC always means outcome=failed
errorTest("master", null, ErrorExtension.ErrorPoint.MODEL, false, true);
errorTest("master", null, ErrorExtension.ErrorPoint.MODEL, true, true);
}
@Test
public void testSlaveHCStageModel() throws Exception {
// A pre-prepare failure on an HC always means outcome=failed
errorTest("slave", null, ErrorExtension.ErrorPoint.MODEL, false, true);
errorTest("slave", null, ErrorExtension.ErrorPoint.MODEL, true, true);
}
@Test
public void testMasterServerStageRuntime() throws Exception {
// The server fails in pre-prepare so whether overall outcome=failed depends on the rollout plan
errorTest("master", "main-one", ErrorExtension.ErrorPoint.RUNTIME, false, true);
errorTest("master", "main-one", ErrorExtension.ErrorPoint.RUNTIME, true, false);
}
@Test
public void testSlaveServerStageRuntime() throws Exception {
// The server fails in pre-prepare so whether overall outcome=failed depends on the rollout plan
errorTest("slave", "main-three", ErrorExtension.ErrorPoint.RUNTIME, false, true);
errorTest("slave", "main-three", ErrorExtension.ErrorPoint.RUNTIME, true, false);
}
@Test
public void testMasterHCStageRuntime() throws Exception {
// A pre-prepare failure on an HC always means outcome=failed
errorTest("master", null, ErrorExtension.ErrorPoint.RUNTIME, false, true);
errorTest("master", null, ErrorExtension.ErrorPoint.RUNTIME, true, true);
}
@Test
public void testSlaveHCStageRuntime() throws Exception {
// A pre-prepare failure on an HC always means outcome=failed
errorTest("slave", null, ErrorExtension.ErrorPoint.RUNTIME, false, true);
errorTest("slave", null, ErrorExtension.ErrorPoint.RUNTIME, true, true);
}
@Test
public void testMasterServerServiceStart() throws Exception {
// The server fails in pre-prepare so whether overall outcome=failed depends on the rollout plan
errorTest("master", "main-one", ErrorExtension.ErrorPoint.SERVICE_START, false, true);
errorTest("master", "main-one", ErrorExtension.ErrorPoint.SERVICE_START, true, false);
}
@Test
public void testSlaveServerServiceStart() throws Exception {
// The server fails in pre-prepare so whether overall outcome=failed depends on the rollout plan
errorTest("slave", "main-three", ErrorExtension.ErrorPoint.SERVICE_START, false, true);
errorTest("slave", "main-three", ErrorExtension.ErrorPoint.SERVICE_START, true, false);
}
@Test
public void testMasterHCServiceStart() throws Exception {
// A pre-prepare failure on an HC always means outcome=failed
errorTest("master", null, ErrorExtension.ErrorPoint.SERVICE_START, false, true);
errorTest("master", null, ErrorExtension.ErrorPoint.SERVICE_START, true, true);
}
@Test
public void testSlaveHCServiceStart() throws Exception {
// A pre-prepare failure on an HC always means outcome=failed
errorTest("slave", null, ErrorExtension.ErrorPoint.SERVICE_START, false, true);
errorTest("slave", null, ErrorExtension.ErrorPoint.SERVICE_START, true, true);
}
@Test
public void testMasterServerServiceStop() throws Exception {
// MSC only logs a WARN for service.stop() errors and stops the service regardless.
// So the op should succeed.
errorTest("master", "main-one", ErrorExtension.ErrorPoint.SERVICE_STOP, false, false);
errorTest("master", "main-one", ErrorExtension.ErrorPoint.SERVICE_STOP, true, false);
}
@Test
public void testSlaveServerServiceStop() throws Exception {
// MSC only logs a WARN for service.stop() errors and stops the service regardless.
// So the op should succeed.
errorTest("slave", "main-three", ErrorExtension.ErrorPoint.SERVICE_STOP, false, false);
errorTest("slave", "main-three", ErrorExtension.ErrorPoint.SERVICE_STOP, true, false);
}
@Test
public void testMasterHCServiceStop() throws Exception {
// MSC only logs a WARN for service.stop() errors and stops the service regardless.
// So the op should succeed.
errorTest("master", null, ErrorExtension.ErrorPoint.SERVICE_STOP, false, false);
errorTest("master", null, ErrorExtension.ErrorPoint.SERVICE_STOP, true, false);
}
@Test
public void testSlaveHCServiceStop() throws Exception {
// MSC only logs a WARN for service.stop() errors and stops the service regardless.
// So the op should succeed.
errorTest("slave", null, ErrorExtension.ErrorPoint.SERVICE_STOP, false, false);
errorTest("slave", null, ErrorExtension.ErrorPoint.SERVICE_STOP, true, false);
}
@Test
public void testMasterServerStageVerify() throws Exception {
// The server fails in pre-prepare so whether overall outcome=failed depends on the rollout plan
errorTest("master", "main-one", ErrorExtension.ErrorPoint.VERIFY, false, true);
errorTest("master", "main-one", ErrorExtension.ErrorPoint.VERIFY, true, false);
}
@Test
public void testSlaveServerStageVerify() throws Exception {
// The server fails in pre-prepare so whether overall outcome=failed depends on the rollout plan
errorTest("slave", "main-three", ErrorExtension.ErrorPoint.VERIFY, false, true);
errorTest("slave", "main-three", ErrorExtension.ErrorPoint.VERIFY, true, false);
}
@Test
public void testMasterHCStageVerify() throws Exception {
// A pre-prepare failure on an HC always means outcome=failed
errorTest("master", null, ErrorExtension.ErrorPoint.VERIFY, false, true);
errorTest("master", null, ErrorExtension.ErrorPoint.VERIFY, true, true);
}
@Test
public void testSlaveHCStageVerify() throws Exception {
// A pre-prepare failure on an HC always means outcome=failed
errorTest("slave", null, ErrorExtension.ErrorPoint.VERIFY, false, true);
errorTest("slave", null, ErrorExtension.ErrorPoint.VERIFY, true, true);
}
@Test
public void testMasterServerStageCommit() throws Exception {
// Post commit failure does not result in outcome=failed
errorTest("master", "main-one", ErrorExtension.ErrorPoint.COMMIT, false, false);
errorTest("master", "main-one", ErrorExtension.ErrorPoint.COMMIT, true, false);
}
@Test
public void testSlaveServerStageCommit() throws Exception {
// Post commit failure does not result in outcome=failed
errorTest("slave", "main-three", ErrorExtension.ErrorPoint.COMMIT, false, false);
errorTest("slave", "main-three", ErrorExtension.ErrorPoint.COMMIT, true, false);
}
@Test
public void testMasterHCStageCommit() throws Exception {
// Here the failure blows away the normal domain response and results in reporting a failure
// TODO this isn't ideal as the DC's model, MSC and the rest of the domain are changed
errorTest("master", null, ErrorExtension.ErrorPoint.COMMIT, false, true);
errorTest("master", null, ErrorExtension.ErrorPoint.COMMIT, true, true);
}
@Test
public void testSlaveHCStageCommit() throws Exception {
// Post commit failure does not result in outcome=failed
errorTest("slave", null, ErrorExtension.ErrorPoint.COMMIT, false, false);
errorTest("slave", null, ErrorExtension.ErrorPoint.COMMIT, true, false);
}
@Test
public void testMasterServerStageRollback() throws Exception {
// The server fails in pre-prepare (to trigger the rollback that throws Error),
// so whether overall outcome=failed depends on the rollout plan
errorTest("master", "main-one", ErrorExtension.ErrorPoint.ROLLBACK, false, true);
errorTest("master", "main-one", ErrorExtension.ErrorPoint.ROLLBACK, true, false);
}
@Test
public void testSlaveServerStageRollback() throws Exception {
// The server fails in pre-prepare (to trigger the rollback that throws Error),
// so whether overall outcome=failed depends on the rollout plan
errorTest("slave", "main-three", ErrorExtension.ErrorPoint.ROLLBACK, false, true);
errorTest("slave", "main-three", ErrorExtension.ErrorPoint.ROLLBACK, true, false);
}
@Test
public void testMasterHCStageRollback() throws Exception {
// A pre-prepare failure (used to trigger the rollback that throws Error) on an HC always means outcome=failed
errorTest("master", null, ErrorExtension.ErrorPoint.ROLLBACK, false, true);
errorTest("master", null, ErrorExtension.ErrorPoint.ROLLBACK, true, true);
}
@Test
public void testSlaveHCStageRollback() throws Exception {
// A pre-prepare failure (used to trigger the rollback that throws Error) on an HC always means outcome=failed
errorTest("slave", null, ErrorExtension.ErrorPoint.ROLLBACK, false, true);
errorTest("slave", null, ErrorExtension.ErrorPoint.ROLLBACK, true, true);
}
private void errorTest(String host, String server, ErrorExtension.ErrorPoint errorPoint, boolean addRolloutPlan, boolean expectFailure) throws Exception {
ModelNode op = ERROR_OP.clone();
op.get(TARGET_HOST.getName()).set(host);
if (server != null) {
op.get(TARGET_SERVER.getName()).set(server);
}
op.get(ErrorExtension.ERROR_POINT.getName()).set(errorPoint.toString());
op.get(CALLER.getName()).set(getTestMethod());
if (addRolloutPlan) {
op.get(OPERATION_HEADERS).set(ROLLOUT_HEADER);
}
ModelNode response;
if (expectFailure) {
response = executeForFailure(op, masterClient);
} else {
response = executeForResponse(op, masterClient);
}
if (errorPoint != ErrorExtension.ErrorPoint.SERVICE_STOP) {
validateResponseDetails(response, host, server, errorPoint);
} // else it's a success and I'm too lazy to write the code to verify the response
}
private void validateResponseDetails(ModelNode response, String host, String server, ErrorExtension.ErrorPoint errorPoint) {
ModelNode unmodified = response.clone();
ModelNode fd;
if (server != null) {
ModelNode serverResp = response.get(SERVER_GROUPS, "main-server-group", HOST, host, server, RESPONSE);
assertEquals(unmodified.toString(), FAILED, serverResp.get(OUTCOME).asString());
fd = serverResp.get(FAILURE_DESCRIPTION);
} else if ("master".equals(host)) {
if (errorPoint == ErrorExtension.ErrorPoint.COMMIT || errorPoint == ErrorExtension.ErrorPoint.ROLLBACK) {
// In this case the late error destroys the normal response structure and we
// get a simple failure. This isn't ideal. I'm writing the test to specifically assert
// this simple failure structure mostly so it can assert the more complex
// structure in the other cases, not because I'm explicitly ruling out any change to this
// simple structure and want this test to enforce that. OTOH it shouldn't be changed lightly
// as it would be easy to mess up.
fd = response.get(FAILURE_DESCRIPTION);
} else {
fd = response.get(FAILURE_DESCRIPTION, DOMAIN_FAILURE_DESCRIPTION);
}
} else {
if (errorPoint == ErrorExtension.ErrorPoint.COMMIT) {
// post-commit failure currently does not produce a failure-description
assertFalse(unmodified.toString(), response.hasDefined(FAILURE_DESCRIPTION));
return;
}
fd = response.get(FAILURE_DESCRIPTION, HOST_FAILURE_DESCRIPTIONS, host);
}
ModelType errorType = errorPoint == ErrorExtension.ErrorPoint.SERVICE_START ? ModelType.OBJECT : ModelType.STRING;
assertEquals(unmodified.toString(), errorType, fd.getType());
assertTrue(unmodified.toString(), fd.asString().contains(ErrorExtension.ERROR_MESSAGE));
}
private MgmtOperationException validateNoActiveOperation(DomainClient client, String host, String server) throws Exception {
PathAddress baseAddress = getManagementControllerAddress(host, server);
// The op should clear w/in a few ms but we'll wait up to 5 secs just in case
// something strange is happening on the machine is overloaded
long timeout = System.currentTimeMillis() + TimeoutUtil.adjust(5000);
MgmtOperationException failure;
String id;
do {
id = findActiveOperation(client, baseAddress, "error");
if (id == null) {
return null;
}
failure = null;
PathAddress address = baseAddress.append(PathElement.pathElement(ACTIVE_OPERATION, id));
ModelNode op = Util.createEmptyOperation(READ_ATTRIBUTE_OPERATION, address);
op.get(NAME).set(OP);
try {
executeForFailure(op, client);
} catch (MgmtOperationException moe) {
failure = moe;
}
Thread.sleep(50);
} while (System.currentTimeMillis() < timeout);
if (failure != null) {
PathAddress address = baseAddress.append(PathElement.pathElement(ACTIVE_OPERATION, id));
if (!executeForResponse(Util.createEmptyOperation("cancel", address), client).get(RESULT).asBoolean()) {
return failure;
}
}
return null;
}
private String findActiveOperation(DomainClient client, PathAddress address, String opName) throws Exception {
ModelNode op = Util.createEmptyOperation(READ_CHILDREN_RESOURCES_OPERATION, address);
op.get(CHILD_TYPE).set(ACTIVE_OPERATION);
ModelNode result = executeForResponse(op, client).get(RESULT);
if (result.isDefined()) {
assertEquals(result.asString(), ModelType.OBJECT, result.getType());
for (Property prop : result.asPropertyList()) {
if (prop.getValue().get(OP).asString().equals(opName)) {
return prop.getName();
}
}
}
return null;
}
private static ModelNode execute(final ModelNode op, final ModelControllerClient modelControllerClient) throws Exception {
Future<ModelNode> future = modelControllerClient.executeAsync(op, OperationMessageHandler.DISCARD);
try {
return future.get(GET_TIMEOUT, TimeUnit.MILLISECONDS);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
future.cancel(true);
throw e;
} catch (Exception e) {
future.cancel(true);
throw e;
}
}
private static ModelNode executeForResponse(final ModelNode op, final ModelControllerClient modelControllerClient) throws Exception {
final ModelNode ret = execute(op, modelControllerClient);
if (! SUCCESS.equals(ret.get(OUTCOME).asString())) {
System.out.println("Failed operation:");
System.out.println(op);
System.out.println("Response:");
System.out.println(ret);
throw new MgmtOperationException("Management operation failed.", op, ret);
}
return ret;
}
private static ModelNode executeForFailure(final ModelNode op, final ModelControllerClient modelControllerClient) throws Exception {
final ModelNode ret = execute(op, modelControllerClient);
if (! FAILED.equals(ret.get(OUTCOME).asString())) {
System.out.println("Unexpectedly successful operation:");
System.out.println(op);
System.out.println("Response:");
System.out.println(ret);
throw new MgmtOperationException("Management operation succeeded.", op, ret);
}
return ret;
}
private static String getTestMethod() {
final StackTraceElement[] stack = Thread.currentThread().getStackTrace();
for (StackTraceElement ste : stack) {
String method = ste.getMethodName();
if (method.startsWith("test")) {
return method;
}
}
return "unknown";
}
private static PathAddress getManagementControllerAddress(String host, String server) {
PathAddress address = PathAddress.pathAddress(PathElement.pathElement(HOST, host));
if (server != null) {
address = address.append(PathElement.pathElement(SERVER, server));
}
address = address.append(MGMT_CONTROLLER);
return address;
}
}