/***************************************************************************
* Copyright (c) 2012-2015 VMware, Inc. All Rights Reserved.
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
***************************************************************************/
package com.vmware.aurora.vc.vcservice;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.RejectedExecutionException;
import org.apache.commons.lang.ArrayUtils;
import org.apache.log4j.Logger;
import com.vmware.aurora.exception.AuroraException;
import com.vmware.aurora.exception.VcException;
import com.vmware.aurora.global.Configuration;
import com.vmware.aurora.stats.StatsLogger;
import com.vmware.aurora.util.AuAssert;
import com.vmware.aurora.vc.MoUtil;
import com.vmware.aurora.vc.VcTask;
import com.vmware.aurora.vc.VcTaskMgr;
import com.vmware.aurora.vc.vcevent.VcEventListener;
import com.vmware.aurora.vc.vcservice.VcConnectionStatusChangeEvent.VcConnectionStatusChangeCallback;
import com.vmware.vim.binding.vim.AboutInfo;
import com.vmware.vim.binding.vim.fault.NotAuthenticated;
import com.vmware.vim.binding.vmodl.LocalizableMessage;
import com.vmware.vim.binding.vmodl.ManagedObjectReference;
import com.vmware.vim.binding.vmodl.RuntimeFault;
import com.vmware.vim.vmomi.core.exception.InternalException;
import com.vmware.vim.vmomi.core.exception.UnmarshallException;
import com.vmware.vim.vmomi.core.types.VmodlContext;
/**
* Here is a general discussion on VC interface.
*
* VC MANAGED AND DATA OBJECTS
*
* There are two types of objects in VC: managed objects and data objects.
*
* Managed objects provide interfaces for performing queries and run
* tasks on an VC object. These are API level interfaces that don't actually
* contain data.
*
* Data objects contain values of VC objects at a given instance. It is
* obtained through queries on managed objects. Data objects may nest
* and accessing child objects do not require new VC queries.
*
* All managed objects related to Aurora are mapped to subclasses of
* {@link VcObject}. Most VC operations are performed through methods of
* these classes. One exception is {@link VcFileManager}, which contains
* some services for file operations.
*
* {@link VcObject} contains snapshots of VC object data through
* its member variables that point to VC data objects.
* Once initialized, all data referenced by its member variables
* can be accessed without new queries to VC. We (aurora) can
* add access functions to get a sub-data-object or a field of it.
*
* Refer to comments in class {@link VcObject} for more info.
*
* VC SESSIONS
*
* In order to deal with problems with VC connection interruption
* and errors, we enclose a section of code that accesses VC with
* wrapper class so that the code can be retried if exceptions are
* thrown due to VC connection errors. The section of code wrapped
* in a {@link body()} function is refered to as VC session.
* An abstract class {@link VcSession} defines the abstraction of
* the session for execution in a VC context. Depending on different
* VC operations, one or two of the following methods may be used
* to program VC accesses.
*
* VC OPERATIONS
*
* In general, there are two types of VC operations.
*
* 1. Queries that only read VC states.
* These operations are stateless and can be repeated without
* side effects. We support having these queries running in
* a VcSession through {@link inVcSessionDo}. Such a session
* can be embedded in SagaSteps that run CMS transactions.
* Refer to {@link com.vmware.aurora.vc.test.CreateRbSaga.CreateRbStep}
* for code example.
* 2. Queries that modifies VC states or run as VC tasks.
* These operations need to run in a Saga Step that extends
* {@link VcTaskSagaStep}. This Saga step cannot do CMS transactions
* in the execution. Refer to {@link VcTaskSagaStep} for usage
* and {@link com.vmware.aurora.vc.test.VmOpSaga} for code examples.
*
*/
/**
* {@link VcContext} class manages VC service life cycle.
*
* All accesses to VC should run from VC context.
*/
public class VcContext {
private static Logger logger = Logger.getLogger(VcContext.class);
/* vc connectivity requirements. */
private final static int vcRequiredMajorVersion = 5; // Starting with VC 5.0.
private final static String vcApiName = "VirtualCenter";
private final static String esxApiName = "HostAgent";
/*
* The global VC service object representing a VC session, initialized
* when system is connected to VC.
*/
private static VcService vcService = null;
/* A global VC session dedicated to Event Listener. */
private static VcService eventService = null;
private static VcEventListener eventListener = null;
private static VcTaskMgr taskMgr = new VcTaskMgr();
/* A global VC session dedicated to long running VC calls. */
private static VcService longCallService = null;
private static VcLongCallHandler longCallHandler = null;
private static VcCleaner vcCleaner;
private static String vcVersion;
private static boolean initialized = false; // Configured & initialized.
private static boolean shutdown = false; // Shutdown request received.
private static boolean fatalInitError = false; // Retrying will not help.
private static int genCount = 0; // Init attempt genCount.
private static Map<VcConnectionStatusChangeEvent, VcConnectionStatusChangeCallback> statusChangeEventCallback =
new HashMap<VcConnectionStatusChangeEvent, VcConnectionStatusChangeCallback>();
private static int eventCount = 0;
/**
* A unique identifier of the VC server, initialized once when vcService
* is created.
*/
private static String serverGuid = null;
public static String getServerGuid() { return serverGuid; }
/**
* Return VC version if a connection is established.
* @return vc version
*/
public static String getVcVersion() {
return vcVersion;
}
/**
* Are we in the process of shutting down VcContext?
* @return true if shutdown was requested
*/
public static synchronized boolean isShuttingDown() {
return shutdown;
}
/**
* Report the connection state with vCenter.
* @return true if connected with vCenter
*/
public static boolean isServiceConnected() {
VcService svc = vcService;
return svc != null && svc.isConnected();
}
/**
* Returns global event listener.
* @return event listener
*/
public static VcEventListener getEventListener() {
return eventListener;
}
/**
* @return VC long call handler
*/
public static VcLongCallHandler getVcLongCallHandler() {
return longCallHandler;
}
/**
* This should only be called from {\link VcSession}.
* @return global task manager.
*/
public static VcTaskMgr getGlobalTaskMgr() {
return taskMgr;
}
/**
* A thread local variable to represent the current VC session.
*
* In Aurora's terminology, a VC session is a chunk of code that
* accesses VC in a single thread. We try to repeat a VC session
* if the underlying code generates recoverable exceptions,
* such as VC connection failures.
*
* For stateful VC operations that rely on vmodl's session,
* such as property collector, events collector, they shouldn't
* use VC session wrappers.
*/
private static ThreadLocal<VcService> tVcService = new ThreadLocal<VcService>() ;
private static ThreadLocal<VcSession<?>> tVcSession = new ThreadLocal<VcSession<?>>();
/**
* @return a TLS VC session object.
* null if not accessed from a VC session.
*/
public static VcSession<?> getSession() {
return tVcSession.get();
}
/**
* @return a TLS service object to access VC,
* null if not accessed from a VC session.
*/
public static VcService getService() {
return tVcService.get();
}
/**
* @return a TLS task manager object to issue tasks to VC,
* null if not accessed from a VC task session.
*/
public static VcTaskMgr getTaskMgr() {
return getSession().getTaskMgr();
}
/**
* @return VcCleaner a cleaning service for vc logout requests
*/
public static VcCleaner getVcCleaner() {
AuAssert.check(vcCleaner != null);
return vcCleaner;
}
/**
* @return true if the current thread in a VC session
*/
public static boolean isInSession() {
return tVcService.get() != null;
}
/**
* @return true if the current thread in a VC task session
*/
public static boolean isInTaskSession() {
return isInSession() && getSession().isTaskSession();
}
/**
* Returns the number of times the context was re-initialized.
* @return genCount
*/
public static int getGenCount() {
return genCount;
}
/**
* Triggers a VC connection status change event.
* @param type event type
* @param serviceName vc service name, maybe null
*/
public static void triggerEvent(VcConnectionStatusChangeEvent event, String serviceName) {
/*
* XXX Because vc sessions can be established from various loops, make sure we
* don't abuse the database too much. Delete once we have confidence.
*/
if (++eventCount > 10000) {
logger.error("Too many VC events");
return;
}
VcConnectionStatusChangeCallback callback = statusChangeEventCallback.get(event);
if (callback != null) {
callback.call(event, serviceName);
}
}
/**
* Called if initVc fails for any reason: cleans up any partially initialized
* state, logs event and throws an unchecked exception with given errorId.
* @param e exception to re-throw
* @param eventType event to log, if any
* @param fatalIniError true for fatal errors (retry will not help)
*/
private static void initFailure(VcException e, VcConnectionStatusChangeEvent eventType,
boolean fatalInitError) {
cleanup(eventType);
VcContext.fatalInitError = fatalInitError; // Dont't retry if true.
AuAssert.check(!initialized);
throw e;
}
private static void checkVcVersion(AboutInfo info) {
if (esxApiName.equals(info.getApiType())) {
initFailure(VcException.CONNECTING_TO_HOSTD(),
VcConnectionStatusChangeEvent.VC_VERSION_HOSTD, true);
} else if (!vcApiName.equals(info.getApiType())) {
initFailure(VcException.CONNECTING_TO_INVALID_PRODUCT(),
VcConnectionStatusChangeEvent.VC_VERSION_INVALID_PRODUCT, true);
}
String mVersionString;
Integer mVersion = 0;
try {
mVersionString = info.getApiVersion().split("\\.")[0];
mVersion = new Integer(mVersionString);
} catch (NumberFormatException e) {
initFailure(VcException.UNSUPPORTED_VERSION(e, info.getApiVersion()),
VcConnectionStatusChangeEvent.VC_VERSION_UNSUPPORTED, true);
}
if (mVersion < vcRequiredMajorVersion) {
initFailure(VcException.UNSUPPORTED_VERSION(null, info.getApiVersion()),
VcConnectionStatusChangeEvent.VC_VERSION_UNSUPPORTED, true);
}
}
/**
* Initialize three global vc sessions and save them in the context:
* - a general purpose vc session used by short lived requests
* - an event listener session, a long running service
* - an dedicated session for long running VC calls
* Also initialize all other auxiliary data structures and state.
* This function either completes the entire initialization successfully
* or rolls back partial initialization on any failure. In the latter case,
* we will attempt to re-initialize again at a later time on VC use.
*/
public static synchronized void initVcContext() {
if(Configuration.getBoolean("vc_service_profiler.log.enabled", false)){
StatsLogger.init(Configuration.getInt("vc_service_profiler.log.interval", 2));
}
AboutInfo info = null;
genCount++;
if (initialized) {
logger.info("VC already initialized");
return;
}
/*
* Don't attempt to relogin into VC on fatal errors (will fail again) or
* when we are in the middle of a shutdown operation (just logged out).
*/
if (fatalInitError) {
logger.warn("Fatal VC init error");
throw VcException.INIT_ERROR();
}
if (shutdown) {
logger.warn("initVcContext during shutdown");
throw VcException.SHUTDOWN_ERROR();
}
/* No partial initialization tolerated. */
AuAssert.check(vcCleaner == null &&
vcService == null &&
eventService == null);
vcCleaner = new VcCleaner();
try {
VmodlContext.getContext();
} catch(IllegalStateException ex) {
//Context has not been initialized. Continue to initialize
VmodlContext.initContext(new String[] { "com.vmware.vim.binding.vim" });
}
try {
vcService = new VcService("VcService");
info = vcService.getServiceInstanceContent().getAbout();
serverGuid = info.getInstanceUuid();
vcVersion = info.getVersion();
} catch (Exception e) {
logger.error("init vc context failed (vc session)");
initFailure(VcException.LOGIN_ERROR(e), null, false); // Retry later.
}
checkVcVersion(info);
try {
eventService = new VcService("VcEventService");
/*
* XXX Since Aurora is currently happy with resource pools from different
* datatcenters, we have to monitor the entire inventory starting with the
* root folder. Will need to find a way to reduce this to either a set
* of datacenters or, better, a single datacenter. Also see Bug 677287.
*/
VcContext.startEventListener(vcService.getServiceInstanceContent().getRootFolder());
} catch (Exception e) {
logger.error("init vc context failed (vc event listener)");
initFailure(VcException.LOGIN_ERROR(e), null, false); // Retry later.
}
try {
/*
* Set the HTTP timeout for long running VC calls to 5 minutes.
*/
longCallService = new VcService("VcLongCallService", false, 5 * 60 * 1000);
VcContext.startVcLongCallHandler();
} catch (Exception e) {
logger.error("init vc context failed (vc long call handler)");
initFailure(VcException.LOGIN_ERROR(e), null, false); // Retry later.
}
logger.info("init vc context succeeded");
initialized = true;
triggerEvent(VcConnectionStatusChangeEvent.VC_CONNECTED, null);
}
public static void initVcContext(Map<VcConnectionStatusChangeEvent, VcConnectionStatusChangeCallback> callback) {
VcContext.statusChangeEventCallback.putAll(callback);
initVcContext();
}
/**
* Shut down all VC services on system shutdown.
*/
public static synchronized void shutdown() {
logger.info("Shutting down VcContext");
shutdown = true;
cleanup(null);
logger.info("VcContext shut down");
}
private static synchronized void cleanup(VcConnectionStatusChangeEvent eventType) {
VcService svc = vcService;
VcService eventSvc = eventService;
VcService longTaskSvc = longCallService;
if (eventType != null) {
triggerEvent(eventType, null);
}
stopEventListener();
stopVcLongCallHandler();
eventService = null;
longCallService = null;
vcService = null;
vcVersion = null;
serverGuid = null;
if (svc != null) {
svc.logout(null);
}
if (eventSvc != null) {
eventSvc.logout(null);
}
if (longTaskSvc != null) {
longTaskSvc.logout(null);
}
if (vcCleaner != null) {
vcCleaner.shutDown(); // Will wait for all logout completions.
vcCleaner = null;
}
initialized = false;
}
// public for testing only
public static synchronized void startSession(VcSession<?> vcSession) {
if (shutdown) {
throw VcException.SHUTDOWN_ERROR();
}
AuAssert.check(initialized && vcService != null);
tVcSession.set(vcSession);
tVcService.set(vcService);
}
// public for testing only
public static void endSession() {
tVcService.set(null);
tVcSession.set(null);
}
/**
* A special long running session used by Event Listener thread.
* Does not need to be protected against shutdowns like the general
* session because this gets executed only once, initiated by the
* main thread that holds VcContext monitor.
*/
public static void startEventSession() {
AuAssert.check(eventService != null && !shutdown);
tVcService.set(eventService);
tVcSession.set(null);
}
/**
* A special long running session used by VcLongCallHandler thread.
* Does not need to be protected against shutdowns like the general
* session because this gets executed only once, initiated by the
* main thread that holds VcContext monitor.
*/
public static void startLongCallSession() {
AuAssert.check(longCallService != null && !shutdown);
tVcService.set(longCallService);
tVcSession.set(null);
}
private static void wrapAndThrow(Throwable e) throws AuroraException {
if (e instanceof AuroraException) {
throw (AuroraException)e;
}
throw VcException.GENERAL_ERROR(e);
}
/**
* Wrapper function for a VC session.
*
* Repeat the VC session in a loop if there is a VC connection
* runtime fault during execution.
*
* Only repeatable VC operations can be executed in this loop.
* For VC operations that have side effects, use VcSagaStep.
*
* @param <T> type of the return value of the VC session
* @param session defines the VC session in {@link session.body()}
* @return result of the VC session
*/
public static <T> T inVcSessionDo(VcSession<T> session) throws AuroraException {
if (isInSession()) {
AuAssert.check(initialized);
// Make sure that we don't reuse a non-task session for a task session
AuAssert.check(!session.isTaskSession() || isInTaskSession());
// Already in VC session, simply return.
try {
return session.body();
} catch (Throwable e) {
wrapAndThrow(e);
}
}
/*
* If failed to initialize on start-up, try again, but without retrying.
*/
if (!initialized) {
initVcContext();
AuAssert.check(initialized);
}
boolean needsRetry = false;
int retries = 0;
long waitTime = VcSession.VC_INIT_WAIT_TIME;
T result = null;
long genCount = -1L;
startSession(session);
// The try block should follow immediately, so that endSession will
// always be called if startSession succeeded.
do {
try {
AuAssert.check(isInSession());
// Login VC if we are logged out of VC service.
genCount = getService().login();
if (needsRetry) {
// undo previous invocation of body()
session.undo();
}
result = session.body();
needsRetry = false;
} catch (Throwable e) {
VcService svc = getService();
Long curGenCount = svc.getServiceGenCount();
endSession();
logger.debug(e);
if(e instanceof RuntimeFault) {
RuntimeFault rtFault = (RuntimeFault)e;
if(!ArrayUtils.isEmpty(rtFault.getFaultMessage())) {
logger.error("vCenter Fault Message(s)>>>>");
for(LocalizableMessage message : rtFault.getFaultMessage()) {
logger.error(message.getMessage());
}
logger.error("<<<< End of Fault Message(s)");
}
}
if (e instanceof RejectedExecutionException ||
e instanceof IllegalStateException) {
/*
* It's likely that the current session was terminated by another thread,
* thus we receive this exception from java Executor. We should retry.
*/
if (curGenCount != null && genCount == svc.getServiceGenCount()) {
/* Log error if we got HTTP binding or thread executor exceptions
* in the same session.
*/
logger.error("genCount=" + genCount, e);
svc.dumpServiceStatus();
}
} else if (!(e instanceof NotAuthenticated ||
e instanceof InternalException ||
e instanceof UnmarshallException)) {
/*
* For a normal exception, we simply throw it.
*
* XXX need to verify that we caught the right exceptions
* The following code is by reading MethodInvocationHandlerImpl.java
* in vlsi code. We potentially didn't catch other interesting
* RuntimeExceptions.
*/
wrapAndThrow(e);
}
if (curGenCount != null && genCount != curGenCount) {
/* If another thread has updated the VcService instance,
* reset the retry counter for using the new service instance.
* This would slow down the VcSession failure rate
* if connection to VC experiences frequent hiccups.
*/
retries = 0;
genCount = curGenCount;
logger.info("reset retries with new genCounter=" + genCount);
} else if (++retries > session.retries) {
logger.error("failed to reconnect to VC connection");
// Failed N retries. Log out of VC and terminate with exception.
vcService.logout(genCount);
if ((e instanceof InternalException &&
MoUtil.isNetworkException(e.getCause())) ||
(e instanceof RejectedExecutionException) ||
(e instanceof IllegalStateException)) {
/* Either a network problem or VC is down. */
throw VcException.UNAVAILABLE_ERROR(e);
}
wrapAndThrow(e);
}
needsRetry = true;
/* This is likely caused by session timeout or has
* exceeded the threshold for forcing reconnection.
* Let's try to clean up, login again and retry.
* XXX needs to investigate what exceptions should do login.
*/
if (e instanceof NotAuthenticated ||
retries > session.reconnectThreshold) {
logger.info("try to reconnect for " + e);
genCount = vcService.reconnect(genCount);
/*
* If we managed to login and get VC connection,
* do not need to wait.
*/
} else {
try {
Thread.sleep(waitTime);
} catch (InterruptedException e1) {
logger.error("reconnect to VC connection interrupted");
wrapAndThrow(e);
}
// double wait time for next the round
waitTime = waitTime << 1;
}
startSession(session);
}
} while (needsRetry);
endSession();
return result;
}
/**
* A callback from a vc service executed on service reset. If this vc
* session was associated with Event Listener, need to re-establish
* Listener's server side state in the context of a new vc session.
*
* @param svc vc service that was (re)initialized
* @throws Exception
*/
public static void serviceReset(VcService svc) throws Exception {
if (svc == eventService && eventListener != null) {
eventListener.reset();
}
}
/**
* Start listening to events rooted in the specified target cluster
* or datacenter moRef in the context of vc event listener session.
* Cannot wait on VcTask completion until EventListener is started.
*
* @param targetMoRef either clutser or datacenter
* @throws Exception
*/
private static void startEventListener(ManagedObjectReference targetMoRef)
throws Exception {
eventListener = new VcEventListener(targetMoRef);
eventListener.setDaemon(true);
eventListener.start();
eventListener.waitUntilStarted();
/*
* If something went wrong during initialization of VcEventListener thread,
* rethrow exception in the parent thread context.
*/
if (eventListener.getInitException() != null) {
throw new Exception("VcEventListener failed to start",
eventListener.getInitException());
}
}
/**
* Safely stop Event Listener's thread.
*/
private static void stopEventListener() {
if (eventListener != null) {
eventListener.shutDown();
eventListener = null;
}
}
/**
* Start a thread polling for VC long calls.
*/
private static void startVcLongCallHandler() throws Exception {
longCallHandler = new VcLongCallHandler();
longCallHandler.setDaemon(true);
longCallHandler.start();
longCallHandler.waitUntilStarted();
}
/**
* Safely stop VC long call handler thread.
*/
private static void stopVcLongCallHandler() {
if (longCallHandler != null) {
longCallHandler.shutDown();
longCallHandler = null;
}
}
/**
* "Bad" tasks don't trigger recentTask[] property updates and we must poll
* for their completions. This function must be called by the poller to make
* sure that the registered TaskFinishedEvent handlers are called for "bad"
* tasks just like they are called for good ones. Synchronized prevents
* eventListener.shutDown().
* @param task
*/
public static synchronized void badTaskCompleted(VcTask task) throws Exception {
if (eventListener != null) {
eventListener.fireBadTaskFinishedEventHandlers(task);
}
}
}