DefaultConfigService.java

/*******************************************************************************
 * Copyright (c) 2011 The Board of Trustees of the Leland Stanford Junior University
 * as Operator of the SLAC National Accelerator Laboratory.
 * Copyright (c) 2011 Brookhaven National Laboratory.
 * EPICS archiver appliance is distributed subject to a Software License Agreement found
 * in file LICENSE that is included with this distribution.
 *******************************************************************************/
package org.epics.archiverappliance.config;

import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.lang.management.ManagementFactory;
import java.lang.management.PlatformLoggingMXBean;
import java.net.InetAddress;
import java.net.URL;
import java.net.UnknownHostException;
import java.nio.charset.Charset;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentSkipListSet;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import java.util.regex.Pattern;

import javax.servlet.ServletContext;

import org.apache.commons.validator.routines.InetAddressValidator;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
import org.epics.archiverappliance.StoragePlugin;
import org.epics.archiverappliance.common.ProcessMetrics;
import org.epics.archiverappliance.common.TimeUtils;
import org.epics.archiverappliance.config.PVTypeInfoEvent.ChangeType;
import org.epics.archiverappliance.config.exception.AlreadyRegisteredException;
import org.epics.archiverappliance.config.exception.ConfigException;
import org.epics.archiverappliance.config.persistence.MySQLPersistence;
import org.epics.archiverappliance.config.pubsub.PubSubEvent;
import org.epics.archiverappliance.engine.ArchiveEngine;
import org.epics.archiverappliance.engine.pv.EngineContext;
import org.epics.archiverappliance.etl.common.PBThreeTierETLPVLookup;
import org.epics.archiverappliance.mgmt.MgmtPostStartup;
import org.epics.archiverappliance.mgmt.MgmtRuntimeState;
import org.epics.archiverappliance.mgmt.NonMgmtPostStartup;
import org.epics.archiverappliance.mgmt.bpl.cahdlers.NamesHandler;
import org.epics.archiverappliance.mgmt.policy.ExecutePolicy;
import org.epics.archiverappliance.mgmt.policy.PolicyConfig;
import org.epics.archiverappliance.mgmt.policy.PolicyConfig.SamplingMethod;
import org.epics.archiverappliance.retrieval.RetrievalState;
import org.epics.archiverappliance.retrieval.channelarchiver.XMLRPCClient;
import org.epics.archiverappliance.utils.ui.GetUrlContent;
import org.epics.archiverappliance.utils.ui.JSONDecoder;
import org.json.simple.JSONArray;
import org.json.simple.JSONObject;
import org.xml.sax.SAXException;

import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.google.common.cache.RemovalListener;
import com.google.common.cache.RemovalNotification;
import com.google.common.eventbus.AsyncEventBus;
import com.google.common.eventbus.EventBus;
import com.google.common.eventbus.Subscribe;
import com.hazelcast.client.HazelcastClient;
import com.hazelcast.client.config.ClientConfig;
import com.hazelcast.config.Config;
import com.hazelcast.config.XmlConfigBuilder;
import com.hazelcast.core.Cluster;
import com.hazelcast.core.EntryEvent;
import com.hazelcast.core.Hazelcast;
import com.hazelcast.core.HazelcastInstance;
import com.hazelcast.core.IMap;
import com.hazelcast.core.ITopic;
import com.hazelcast.core.Member;
import com.hazelcast.core.MemberAttributeEvent;
import com.hazelcast.core.MembershipEvent;
import com.hazelcast.core.MembershipListener;
import com.hazelcast.core.Message;
import com.hazelcast.core.MessageListener;
import com.hazelcast.map.listener.EntryAddedListener;
import com.hazelcast.map.listener.EntryRemovedListener;
import com.hazelcast.map.listener.EntryUpdatedListener;

import edu.stanford.slac.archiverappliance.PB.data.PBTypeSystem;



/**
 * This is the default config service for the archiver appliance.
 * There is a subclass that is used in the junit tests.
 * 
 * @author mshankar
 *
 */
public class DefaultConfigService implements ConfigService {
	private static Logger logger = Logger.getLogger(DefaultConfigService.class.getName());
	private static Logger configlogger = Logger.getLogger("config." + DefaultConfigService.class.getName());
	private static Logger clusterLogger = Logger.getLogger("cluster." + DefaultConfigService.class.getName());
	
	public static final String SITE_FOR_UNIT_TESTS_NAME = "org.epics.archiverappliance.config.site";
	public static final String SITE_FOR_UNIT_TESTS_VALUE = "tests";
	
	/**
	 * Add a property in archappl.properties under this key to identify the class that implements the PVNameToKeyMapping for this installation.
	 */
	public static final String ARCHAPPL_PVNAME_TO_KEY_MAPPING_CLASSNAME = "org.epics.archiverappliance.config.DefaultConfigService.PVName2KeyMappingClassName";

	
	// Configuration state begins here.
	protected String myIdentity;
	protected ApplianceInfo myApplianceInfo = null;
	protected Map<String, ApplianceInfo> appliances = null;
	// Persisted state begins here.
	protected Map<String, PVTypeInfo> typeInfos = null;
	protected Map<String, UserSpecifiedSamplingParams> archivePVRequests = null;
	protected Map<String, String> channelArchiverDataServers = null;
	protected Map<String, String> aliasNamesToRealNames = null;
	// These are not persisted but derived from other info
	protected Map<String, ApplianceInfo> pv2appliancemapping = null;
	protected Map<String, String> clusterInet2ApplianceIdentity = null;
	protected Map<String, List<ChannelArchiverDataServerPVInfo>> pv2ChannelArchiverDataServer = null;
	protected ITopic<PubSubEvent> pubSub = null;
	protected Map<String, Boolean> namedFlags = null;
	// Configuration state ends here.
	
	// Runtime state begins here 
	protected LinkedList<Runnable> shutdownHooks = new LinkedList<Runnable>();
	protected PBThreeTierETLPVLookup etlPVLookup = null;
	protected RetrievalState retrievalState = null;
	protected MgmtRuntimeState mgmtRuntime = null;;
	protected EngineContext engineContext = null;
	protected ConcurrentSkipListSet<String> appliancesInCluster = new ConcurrentSkipListSet<String>();
	// Runtime state ends here 

	// This is an optimization; we cache a copy of PVs that are registered for this appliance.
	protected ConcurrentSkipListSet<String> pvsForThisAppliance = null;
	// Maintain a TRIE index for the pvNames in this appliance.
	protected ConcurrentHashMap<String, ConcurrentSkipListSet<String>> parts2PVNamesForThisAppliance = new ConcurrentHashMap<String, ConcurrentSkipListSet<String>>(); 
	protected ConcurrentSkipListSet<String> pausedPVsForThisAppliance = null;
	protected ApplianceAggregateInfo applianceAggregateInfo = new ApplianceAggregateInfo();
	protected EventBus eventBus = new AsyncEventBus(Executors.newSingleThreadExecutor(new ThreadFactory() { @Override public Thread newThread(Runnable r) { return new Thread(r, "Event bus");}}));
	protected Properties archapplproperties = new Properties();
	protected PVNameToKeyMapping pvName2KeyConverter = null;
	protected ConfigPersistence persistanceLayer;

	// State local to DefaultConfigService.
	protected WAR_FILE warFile = WAR_FILE.MGMT;
	protected STARTUP_SEQUENCE startupState = STARTUP_SEQUENCE.ZEROTH_STATE;
	protected ScheduledExecutorService startupExecutor = null;
	protected ProcessMetrics processMetrics = new ProcessMetrics();
	private HashSet<String> runTimeFields = new HashSet<String>();
	// Use a Guava cache to store one and only one ExecutePolicy object that expires after some inactivity.
	// The side effect is that it may take this many minutes to update the policy that is cached.
	private LoadingCache<String, ExecutePolicy> theExecutionPolicy = CacheBuilder.newBuilder()
			.expireAfterWrite(1, TimeUnit.MINUTES)
			.removalListener(new RemovalListener<String, ExecutePolicy>() {
				@Override
				public void onRemoval(RemovalNotification<String, ExecutePolicy> arg) {
					arg.getValue().close();
				}
			})
			.build(new CacheLoader<String, ExecutePolicy>() {
				public ExecutePolicy load(String key) throws IOException {
					logger.info("Updating the cached execute policy");
					return new ExecutePolicy(DefaultConfigService.this);
				}
			});



	private ServletContext servletContext;

	protected DefaultConfigService() {
		// Only the unit tests config service uses this constructor.
	}

	@Override
	public void initialize(ServletContext sce) throws ConfigException {
		this.servletContext = sce;
		String contextPath = sce.getContextPath();
		logger.info("DefaultConfigService was created with a servlet context " + contextPath);
		
		try { 
			String pathToVersionTxt = sce.getRealPath("ui/comm/version.txt");
			logger.debug("The full path to the version.txt is " + pathToVersionTxt);
			List<String> lines = Files.readAllLines(Paths.get(pathToVersionTxt), Charset.forName("UTF-8"));
			for(String line : lines) { 
				configlogger.info(line);
			}
		} catch(Throwable t) {
			logger.fatal("Unable to determine appliance version", t);
		}
		
		try {
			// We first try Java system properties for this appliance's identity
			// If a property is not defined, then we check the environment.
			// This gives us the ability to cater to unit tests as well as running using buildAndDeploy scripts without touching the server.xml file.
			// Probably not the most standard way but suited to this need.
			// Finally, we use the local machine's hostname as the myidentity.
			myIdentity = System.getProperty(ARCHAPPL_MYIDENTITY);
			if(myIdentity == null) {
				myIdentity = System.getenv(ARCHAPPL_MYIDENTITY);
				if(myIdentity != null) { 
					logger.info("Obtained my identity from environment variable " + myIdentity);
				} else { 
					logger.info("Using the local machine's hostname " + myIdentity + " as my identity");
					myIdentity = InetAddress.getLocalHost().getCanonicalHostName();
				}
				if(myIdentity == null) {
					throw new ConfigException("Unable to determine identity of this appliance");
				}
			} else {
				logger.info("Obtained my identity from Java system properties " + myIdentity);
			}

			logger.info("My identity is " + myIdentity);
		} catch(Exception ex) {
			String msg = "Cannot determine this appliance's identity using either the environment variable " + ARCHAPPL_MYIDENTITY + " or the java system property " + ARCHAPPL_MYIDENTITY;
			configlogger.fatal(msg); 
			throw new ConfigException(msg, ex);
		}
		// Appliances should be local and come straight from persistence.
		try {
			appliances = AppliancesList.loadAppliancesXML(servletContext);
		} catch(Exception ex) {
			throw new ConfigException("Exception loading appliances.xml", ex);
		}
		
		myApplianceInfo = appliances.get(myIdentity);
		if(myApplianceInfo == null) throw new ConfigException("Unable to determine applianceinfo using identity " + myIdentity);
		configlogger.info("My identity is " + myApplianceInfo.getIdentity() + " and my mgmt URL is " + myApplianceInfo.getMgmtURL());
		
		// To make sure we are not starting multiple appliance with the same identity, we make sure that the hostnames match
		try { 
			String machineHostName = InetAddress.getLocalHost().getCanonicalHostName();
			String[] myAddrParts = myApplianceInfo.getClusterInetPort().split(":");
			String myHostNameFromInfo = myAddrParts[0];
			if(myHostNameFromInfo.equals("localhost")) { 
				logger.debug("Using localhost for the cluster inet port. If you are indeed running a cluster, the cluster members will not join the cluster.");
			} else if (myHostNameFromInfo.equals(machineHostName)) { 
				logger.debug("Hostname from config and hostname from InetAddress match exactly; we are correctly configured " + machineHostName);				
			} else if(InetAddressValidator.getInstance().isValid(myHostNameFromInfo)) {
				logger.debug("Using ipAddress for cluster config " + myHostNameFromInfo);				
			} else { 
				String msg = "The hostname from appliances.xml is " + myHostNameFromInfo + " and from a call to InetAddress.getLocalHost().getCanonicalHostName() (typially FQDN) is " + machineHostName 
						+ ". These are not identical. They are probably equivalent but to prevent multiple appliances binding to the same identity we enforce this equality.";
				configlogger.fatal(msg);
				throw new ConfigException(msg);
			}
		} catch(UnknownHostException ex) { 
			configlogger.error("Got an UnknownHostException when trying to determine the hostname. This happens when DNS is not set correctly on this machine (for example, when using VM's. See the documentation for InetAddress.getLocalHost().getCanonicalHostName()");
		}

		try { 
			String archApplPropertiesFileName = System.getProperty(ARCHAPPL_PROPERTIES_FILENAME);
			if(archApplPropertiesFileName == null) {
				archApplPropertiesFileName = System.getenv(ARCHAPPL_PROPERTIES_FILENAME);
			} 
			if(archApplPropertiesFileName == null) {
				archApplPropertiesFileName = new URL(this.getClass().getClassLoader().getResource(DEFAULT_ARCHAPPL_PROPERTIES_FILENAME).toString()).getPath();
				configlogger.info("Loading archappl.properties from the webapp classpath " + archApplPropertiesFileName);
			} else { 
				configlogger.info("Loading archappl.properties using the environment/JVM property from " + archApplPropertiesFileName);
			}
			try(InputStream is = new FileInputStream(new File(archApplPropertiesFileName))) {
				archapplproperties.load(is);
				configlogger.info("Done loading installation specific properties file from " + archApplPropertiesFileName);
			} catch(Exception ex) {
				throw new ConfigException("Exception loading installation specific properties file " + archApplPropertiesFileName, ex);
			}
		} catch(ConfigException cex) { 
			throw cex;
		} catch(Exception ex) { 
			configlogger.fatal("Exception loading the appliance properties file", ex);
		}
		
		switch(contextPath) {
		case "/mgmt":
			warFile = WAR_FILE.MGMT;
			this.mgmtRuntime = new MgmtRuntimeState(this);
			break;
		case "/engine":
			warFile = WAR_FILE.ENGINE;
			this.engineContext=new EngineContext(this);
			break;
		case "/retrieval":
			warFile = WAR_FILE.RETRIEVAL;
			this.retrievalState = new RetrievalState(this);
			break;
		case "/etl":
			this.etlPVLookup = new PBThreeTierETLPVLookup(this);
			warFile = WAR_FILE.ETL;
			break;
		default:
			logger.error("We seem to have introduced a new component into the system " + contextPath);
		}
		
		

		String pvName2KeyMappingClass = this.getInstallationProperties().getProperty(ARCHAPPL_PVNAME_TO_KEY_MAPPING_CLASSNAME);
		if(pvName2KeyMappingClass == null || pvName2KeyMappingClass.equals("") || pvName2KeyMappingClass.length() < 1) {
			logger.info("Using the default key mapping class");
			pvName2KeyConverter = new ConvertPVNameToKey();
			pvName2KeyConverter.initialize(this);
		} else { 
			try { 
				logger.info("Using " + pvName2KeyMappingClass + " as the name to key mapping class");
				pvName2KeyConverter = (PVNameToKeyMapping) Class.forName(pvName2KeyMappingClass).newInstance();
				pvName2KeyConverter.initialize(this);
			} catch(Exception ex) { 
				logger.fatal("Cannot initialize pv name to key mapping class " + pvName2KeyMappingClass, ex);
				throw new ConfigException("Cannot initialize pv name to key mapping class " + pvName2KeyMappingClass, ex);			
			}
		}

		String runtimeFieldsListStr = this.getInstallationProperties().getProperty("org.epics.archiverappliance.config.RuntimeKeys");
		if(runtimeFieldsListStr != null && !runtimeFieldsListStr.isEmpty()) { 
			logger.debug("Got runtime fields from the properties file " + runtimeFieldsListStr);
			String[] runTimeFieldsArr = runtimeFieldsListStr.split(",");
			for(String rf : runTimeFieldsArr) {
				this.runTimeFields.add(rf.trim());
			}
		}
				
		
		startupExecutor = Executors.newScheduledThreadPool(1, new ThreadFactory() {
			@Override
			public Thread newThread(Runnable r) {
				Thread t = new Thread(r);
				t.setName("Startup executor");
				return t;
			}
		});
		
		this.addShutdownHook(new Runnable() {
			@Override
			public void run() {
				logger.info("Shutting down startup scheduled executor...");
				startupExecutor.shutdown();
			}
		});
		
		this.startupState = STARTUP_SEQUENCE.READY_TO_JOIN_APPLIANCE;
		if(this.warFile == WAR_FILE.MGMT) {
			logger.info("Scheduling webappReady's for the mgmt webapp ");
			MgmtPostStartup mgmtPostStartup = new MgmtPostStartup(this);
			ScheduledFuture<?> postStartupFuture = startupExecutor.scheduleAtFixedRate(mgmtPostStartup, 10, 20, TimeUnit.SECONDS);
			mgmtPostStartup.setCancellingFuture(postStartupFuture);
		} else {
			logger.info("Scheduling webappReady's for the non-mgmt webapp " + this.warFile.toString());
			NonMgmtPostStartup nonMgmtPostStartup = new NonMgmtPostStartup(this, this.warFile.toString());
			ScheduledFuture<?> postStartupFuture = startupExecutor.scheduleAtFixedRate(nonMgmtPostStartup, 10, 20, TimeUnit.SECONDS);
			nonMgmtPostStartup.setCancellingFuture(postStartupFuture);
		}
		
		// Measure some JMX metrics once a minute
		startupExecutor.scheduleAtFixedRate(new Runnable() {
			@Override
			public void run() {
				processMetrics.takeMeasurement();
			}
		}, 60, 60, TimeUnit.SECONDS);
	}
	
	
	/* (non-Javadoc)
	 * @see org.epics.archiverappliance.config.ConfigService#postStartup()
	 */
	@Override
	public void postStartup() throws ConfigException {
		if(this.startupState != STARTUP_SEQUENCE.READY_TO_JOIN_APPLIANCE) {
			configlogger.info("Webapp is not in correct state for postStartup " + this.getWarFile().toString() + ". It is in " + this.startupState.toString());
			return;
		}
		
		this.startupState = STARTUP_SEQUENCE.POST_STARTUP_RUNNING;
		configlogger.info("Post startup for " + this.getWarFile().toString());

		// Inherit logging from log4j configuration.
		try { 
			PlatformLoggingMXBean logging = ManagementFactory.getPlatformMXBean(PlatformLoggingMXBean.class);
			if(logging != null) {
				java.util.logging.Logger.getLogger("com.hazelcast");
				if(clusterLogger.isDebugEnabled()) { 
					logging.setLoggerLevel("com.hazelcast", java.util.logging.Level.FINE.toString());
				} else if(clusterLogger.isInfoEnabled()) { 
					logging.setLoggerLevel("com.hazelcast", java.util.logging.Level.INFO.toString());
				} else { 
					logger.info("Setting clustering logging based on log levels for cluster." + getClass().getName());
					logging.setLoggerLevel("com.hazelcast", java.util.logging.Level.SEVERE.toString());
				}
			}
			
			Logger hzMain = Logger.getLogger("com.hazelcast");
			if(clusterLogger.isDebugEnabled()) { 
				hzMain.setLevel(Level.DEBUG);
			} else if(clusterLogger.isInfoEnabled()) { 
				hzMain.setLevel(Level.INFO);
			} else { 
				logger.info("Setting clustering logging based on log levels for cluster." + getClass().getName());
				hzMain.setLevel(Level.FATAL);
			}			
		} catch(Exception ex) { 
			logger.error("Exception setting logging JVM levels ", ex);
		}
		
		// Add this to the system props before doing anything with Hz
		System.getProperties().put("hazelcast.logging.type", "log4j");

		HazelcastInstance hzinstance = null;
		
		// Set the thread count to control how may threads this library spawns.
		Properties hzThreadCounts = new Properties();
		if(System.getenv().containsKey("ARCHAPPL_ALL_APPS_ON_ONE_JVM")) { 
			logger.info("Reducing the generic clustering thread counts.");
			hzThreadCounts.put("hazelcast.clientengine.thread.count", "2");
			hzThreadCounts.put("hazelcast.operation.generic.thread.count", "2");
			hzThreadCounts.put("hazelcast.operation.thread.count", "2");
		}
		
		if(this.warFile == WAR_FILE.MGMT) {
			// The management webapps are the head honchos in the cluster. We set them up differently

			configlogger.debug("Initializing the MGMT webapp's clustering");
			// If we have a hazelcast.xml in the servlet classpath, the XmlConfigBuilder picks that up.
			// If not we use the default config found in hazelcast.jar
			// We then alter this config to suit our purposes.
			Config config = new XmlConfigBuilder().build();
			try {
				if(this.getClass().getResource("hazelcast.xml") == null) {
					logger.info("We override the default cluster config by disabling multicast discovery etc.");
					// We do not use multicast as it is not supported on all networks.
					config.getNetworkConfig().getJoin().getMulticastConfig().setEnabled(false);
					// We use TCPIP to discover the members in the cluster.
					// This is part of the config that comes from appliance.xml
					config.getNetworkConfig().getJoin().getTcpIpConfig().setEnabled(true);
					// Clear any tcpip config that comes from the default config
					// This gets rid of the localhost in the default that prevents clusters from forming..
					// If we need localhost, we'll add it back later.
					config.getNetworkConfig().getJoin().getTcpIpConfig().clear();
					// Enable interfaces; we seem to need this after 2.4 for clients to work correctly in a multi-homed environment.
					// We'll add the actual interface later below
					config.getNetworkConfig().getInterfaces().setEnabled(true);
					config.getNetworkConfig().getInterfaces().clear();
					
					// We don't really use the authentication provided by the tool; however, we set it to some default
					config.getGroupConfig().setName("archappl");
					config.getGroupConfig().setPassword("archappl");

					// Backup count is 1 by default; we set it explicitly however...
					config.getMapConfig("default").setBackupCount(1);
					
					config.setProperty("hazelcast.logging.type", "log4j");
				} else {
					logger.debug("There is a hazelcast.xml in the classpath; skipping default configuration in the code.");
				}
			} catch(Exception ex) {
				throw new ConfigException("Exception configuring cluster", ex);
			}

			config.setInstanceName(myIdentity);
			
			if(!hzThreadCounts.isEmpty()) {
				logger.info("Reducing the generic clustering thread counts.");
				config.getProperties().putAll(hzThreadCounts);
			}
			
			try {
				String[] myAddrParts = myApplianceInfo.getClusterInetPort().split(":");
				String myHostName = myAddrParts[0];
				InetAddress myInetAddr = InetAddress.getByName(myHostName);
				if(!myHostName.equals("localhost") && myInetAddr.isLoopbackAddress()) { 
					logger.info("Address for this appliance -- " + myInetAddr.toString() + " is a loopback address. Changing this to 127.0.0.1 to clustering happy");
					myInetAddr = InetAddress.getByName("127.0.0.1");
				}
				int myClusterPort = Integer.parseInt(myAddrParts[1]);
				
				logger.debug("We do not let the port auto increment for the MGMT webap");
				config.getNetworkConfig().setPortAutoIncrement(false);				

				config.getNetworkConfig().setPort(myClusterPort);
				config.getNetworkConfig().getInterfaces().addInterface(myInetAddr.getHostAddress());
				configlogger.info("Setting my cluster port base to " + myClusterPort + " and using interface " + myInetAddr.getHostAddress());
				
				for(ApplianceInfo applInfo : appliances.values()) {
					if(applInfo.getIdentity().equals(myIdentity) && this.warFile == WAR_FILE.MGMT) { 
						logger.debug("Not adding myself to the discovery process when I am the mgmt webapp");
					} else { 
						String[] addressparts = applInfo.getClusterInetPort().split(":");
						String inetaddrpart = addressparts[0];
						try { 
							InetAddress inetaddr = InetAddress.getByName(inetaddrpart);
							if(!inetaddrpart.equals("localhost") && inetaddr.isLoopbackAddress()) { 
								logger.info("Address for appliance " + applInfo.getIdentity() + " -  "+ inetaddr.toString() + " is a loopback address. Changing this to 127.0.0.1 to clustering happy");
								inetaddr = InetAddress.getByName("127.0.0.1");
							}
							int clusterPort = Integer.parseInt(addressparts[1]);
							logger.info("Adding " + applInfo.getIdentity() + " from appliances.xml to the cluster discovery using cluster inetport " + inetaddr.toString() + ":" + clusterPort);
							config.getNetworkConfig().getJoin().getTcpIpConfig().addMember(inetaddr.getHostAddress() + ":" + clusterPort);
						} catch(UnknownHostException ex) { 
							configlogger.info("Cannnot resolve the IP address for appliance " + inetaddrpart + ". Skipping adding this appliance to the cliuster.");
						}
					}
				}
				hzinstance = Hazelcast.newHazelcastInstance(config);
			} catch(Exception ex) {
				throw new ConfigException("Exception adding member to cluster", ex);
			}
		} else {
			// All other webapps are "native" clients.
			try { 
				configlogger.debug("Initializing a non-mgmt webapp's clustering");
				ClientConfig clientConfig = new ClientConfig();
				clientConfig.getGroupConfig().setName("archappl");
				clientConfig.getGroupConfig().setPassword("archappl");
				clientConfig.setExecutorPoolSize(4);
				// Non mgmt client can only connect to their MGMT webapp.
				String[] myAddrParts = myApplianceInfo.getClusterInetPort().split(":");
				String myHostName = myAddrParts[0];
				InetAddress myInetAddr = InetAddress.getByName(myHostName);
				if(!myHostName.equals("localhost") && myInetAddr.isLoopbackAddress()) { 
					logger.info("Address for this appliance -- " + myInetAddr.toString() + " is a loopback address. Changing this to 127.0.0.1 to clustering happy");
					myInetAddr = InetAddress.getByName("127.0.0.1");
				}
				int myClusterPort = Integer.parseInt(myAddrParts[1]);
	
				configlogger.debug(this.warFile + " connecting as a native client to " + myInetAddr.getHostAddress() + ":" + myClusterPort);
				clientConfig.getNetworkConfig().addAddress(myInetAddr.getHostAddress() + ":" + myClusterPort);
				clientConfig.setProperty("hazelcast.logging.type", "log4j");
				
				if(!hzThreadCounts.isEmpty()) {
					logger.info("Reducing the generic clustering thread counts.");
					clientConfig.getProperties().putAll(hzThreadCounts);
				}
				
				if(!clusterLogger.isDebugEnabled()) {
					// The client code logs some SEVERE exceptions on shutdown when deploying on the same Tomcat container.
					// These exceptions are confusing; ideally, we would not have to set the log levels like so.
					Logger.getLogger("com.hazelcast.client.spi.impl.ClusterListenerThread").setLevel(Level.OFF);
					Logger.getLogger("com.hazelcast.client.spi.ClientPartitionService").setLevel(Level.OFF);
				}
				hzinstance = HazelcastClient.newHazelcastClient(clientConfig);
			} catch(Exception ex) {
				throw new ConfigException("Exception adding client to cluster", ex);
			}
		}
		
		
		
		pv2appliancemapping = hzinstance.getMap("pv2appliancemapping");
		namedFlags = hzinstance.getMap("namedflags");
		typeInfos = hzinstance.getMap("typeinfo");
		archivePVRequests = hzinstance.getMap("archivePVRequests");
		channelArchiverDataServers = hzinstance.getMap("channelArchiverDataServers");
		clusterInet2ApplianceIdentity = hzinstance.getMap("clusterInet2ApplianceIdentity");
		aliasNamesToRealNames = hzinstance.getMap("aliasNamesToRealNames");
		pv2ChannelArchiverDataServer = hzinstance.getMap("pv2ChannelArchiverDataServer");
		pubSub = hzinstance.getTopic("pubSub");
		
		final HazelcastInstance shutdownHzInstance = hzinstance;
		shutdownHooks.add(0, new Runnable() {
			@Override
			public void run() {
				logger.debug("Shutting down clustering instance in webapp " + warFile.toString());
				shutdownHzInstance.shutdown();
			}
		});

		if(this.warFile == WAR_FILE.MGMT) {
			Cluster cluster = hzinstance.getCluster();
			String localInetPort = getMemberKey(cluster.getLocalMember());
			clusterInet2ApplianceIdentity.put(localInetPort, myIdentity);
			logger.debug("Adding myself " + myIdentity + " as having inetport " + localInetPort);
			hzinstance.getMap("clusterInet2ApplianceIdentity").addEntryListener(new EntryAddedListener<Object, Object>() {
				@Override
				public void entryAdded(EntryEvent<Object, Object> event) {
					String appliden = (String) event.getValue();
					appliancesInCluster.add(appliden);
					logger.info("Adding appliance " + appliden + " to the list of active appliances as inetport " + ((String) event.getKey()));
				}
			}, true);
			hzinstance.getMap("clusterInet2ApplianceIdentity").addEntryListener(new EntryRemovedListener<Object, Object>() {
				@Override
				public void entryRemoved(EntryEvent<Object, Object> event) {
					String appliden = (String) event.getValue();
					appliancesInCluster.remove(appliden);
					logger.info("Removing appliance " + appliden + " from the list of active appliancesas inetport " + ((String) event.getKey()));
				}				
			}, true);
			
			logger.debug("Establishing a cluster membership listener to detect when appliances drop off the cluster");
			cluster.addMembershipListener(new MembershipListener(){
				public void memberAdded(MembershipEvent membersipEvent) {
					Member member = membersipEvent.getMember();
					String inetPort = getMemberKey(member);
					if(clusterInet2ApplianceIdentity.containsKey(inetPort)) {
						String appliden = clusterInet2ApplianceIdentity.get(inetPort);
						appliancesInCluster.add(appliden);
						configlogger.info("Adding newly started appliance " + appliden + " to the list of active appliances for inetport " + inetPort);
					} else {
						logger.debug("Skipping adding appliance using inetport " + inetPort + " to the list of active instances as we do not have a mapping to its identity");
					}
				}
				public void memberRemoved(MembershipEvent membersipEvent) {
					Member member = membersipEvent.getMember();
					String inetPort = getMemberKey(member);
					if(clusterInet2ApplianceIdentity.containsKey(inetPort)) {
						String appliden = clusterInet2ApplianceIdentity.get(inetPort);
						appliancesInCluster.remove(appliden);
						configlogger.info("Removing appliance " + appliden + " from the list of active appliances");
					} else {
						configlogger.debug("Received member removed event for " + inetPort);
					}
				}
				@Override
				public void memberAttributeChanged(MemberAttributeEvent membersipEvent) {
					Member member = membersipEvent.getMember();
					String inetPort = getMemberKey(member);
					configlogger.debug("Received membership attribute changed event for " + inetPort);
				}
			});

			logger.debug("Adding the current members in the cluster after establishing the cluster membership listener");
			for (Member member : cluster.getMembers()) {
				String mbrInetPort = getMemberKey(member);
				logger.debug("Found member " + mbrInetPort);
				if(clusterInet2ApplianceIdentity.containsKey(mbrInetPort)) {
					String appliden = clusterInet2ApplianceIdentity.get(mbrInetPort);
					appliancesInCluster.add(appliden);
					logger.info("Adding appliance " + appliden + " to the list of active appliances for inetport " + mbrInetPort);
				} else {
					logger.debug("Skipping adding appliance using inetport " + mbrInetPort + " to the list of active instances as we do not have a mapping to its identity");
				}
			}
			logger.info("Established subscription(s) for appliance availability");
			
			if(this.getInstallationProperties().containsKey(ARCHAPPL_NAMEDFLAGS_PROPERTIES_FILE_PROPERTY)) {
				String namedFlagsFileName = (String) this.getInstallationProperties().get(ARCHAPPL_NAMEDFLAGS_PROPERTIES_FILE_PROPERTY);
				configlogger.info("Loading named flags from file " + namedFlagsFileName);
				File namedFlagsFile = new File(namedFlagsFileName);
				if(!namedFlagsFile.exists()) { 
					configlogger.error("File containing named flags " + namedFlagsFileName + " specified but not present");
				} else { 
					Properties namedFlagsFromFile = new Properties();
					try(FileInputStream is = new FileInputStream(namedFlagsFile)) { 
						namedFlagsFromFile.load(is);
						for(Object namedFlagFromFile : namedFlagsFromFile.keySet()) {
							try { 
								String namedFlagFromFileStr = (String) namedFlagFromFile;
								Boolean namedFlagFromFileValue = Boolean.parseBoolean((String)namedFlagsFromFile.get(namedFlagFromFileStr));
								logger.debug("Setting named flag " + namedFlagFromFileStr + " to " + namedFlagFromFileValue);
								this.namedFlags.put(namedFlagFromFileStr, namedFlagFromFileValue);
							} catch(Exception ex) { 
								logger.error("Exception loading named flag from file" + namedFlagsFileName, ex);
							}
						}
					} catch(Exception ex) { 
						configlogger.error("Exception loading named flags from " + namedFlagsFileName, ex);
					}
				}
			}
		}
		
		if(this.warFile == WAR_FILE.ENGINE) {
			// It can take a while for the engine to start up.
			// We probably want to do this in the background so that the appliance as a whole starts up quickly and we get retrieval up and running quickly.
			this.startupExecutor.schedule(new Runnable() {
				@Override
				public void run() {
					try { 
						logger.debug("Starting up the engine's channels on startup.");
						archivePVSonStartup();
						logger.debug("Done starting up the engine's channels in startup.");
					} catch(Throwable t) { 
						configlogger.fatal("Exception starting up the engine channels on startup", t);
					}
				}
			}, 1, TimeUnit.SECONDS); 
		} else if(this.warFile == WAR_FILE.ETL) {
			this.etlPVLookup.postStartup();
		} else if(this.warFile == WAR_FILE.MGMT) {
			pvsForThisAppliance = new ConcurrentSkipListSet<String>();
			pausedPVsForThisAppliance = new ConcurrentSkipListSet<String>();
			
			initializePersistenceLayer();

			loadTypeInfosFromPersistence();
			
			loadAliasesFromPersistence();

			loadArchiveRequestsFromPersistence();
			
			loadExternalServersFromPersistence();
			
			registerForNewExternalServers(hzinstance.getMap("channelArchiverDataServers"));

			// Cache the aggregate of all the PVs that are registered to this appliance.
			logger.debug("Building a local aggregate of PV infos that are registered to this appliance");
			for(String pvName : getPVsForThisAppliance()) {
				if(!pvsForThisAppliance.contains(pvName)) {
					applianceAggregateInfo.addInfoForPV(pvName, this.getTypeInfoForPV(pvName), this);
				}
			}
		}
		
		// Register for changes to the typeinfo map.
		logger.info("Registering for changes to typeinfos");
		hzinstance.getMap("typeinfo").addEntryListener(new EntryAddedListener<Object, Object>() {
			@Override
			public void entryAdded(EntryEvent<Object, Object> entryEvent) {
				logger.debug("Received entryAdded for pvTypeInfo");
				PVTypeInfo typeInfo = (PVTypeInfo) entryEvent.getValue();
				String pvName = typeInfo.getPvName();
				eventBus.post(new PVTypeInfoEvent(pvName, typeInfo, ChangeType.TYPEINFO_ADDED));
				if(persistanceLayer != null) { 
					try { 
						persistanceLayer.putTypeInfo(pvName, typeInfo);
					} catch(Exception ex) { 
						logger.error("Exception persisting pvTypeInfo for pv " + pvName, ex);
					}
				}
			}
		}, true);
		hzinstance.getMap("typeinfo").addEntryListener(new EntryRemovedListener<Object, Object>() {
			@Override
			public void entryRemoved(EntryEvent<Object, Object> entryEvent) {
				PVTypeInfo typeInfo =(PVTypeInfo) entryEvent.getOldValue();
				String pvName = typeInfo.getPvName();
				logger.info("Received entryRemoved for pvTypeInfo " + pvName);
				eventBus.post(new PVTypeInfoEvent(pvName, typeInfo, ChangeType.TYPEINFO_DELETED));
				if(persistanceLayer != null) { 
					try { 
						persistanceLayer.deleteTypeInfo(pvName);
					} catch(Exception ex) { 
						logger.error("Exception deleting pvTypeInfo for pv " + pvName, ex);
					}
				}
			}
		}, true);
		hzinstance.getMap("typeinfo").addEntryListener(new EntryUpdatedListener<Object, Object>() {
			@Override
			public void entryUpdated(EntryEvent<Object, Object> entryEvent) {
				PVTypeInfo typeInfo =(PVTypeInfo) entryEvent.getValue();
				String pvName = typeInfo.getPvName();
				eventBus.post(new PVTypeInfoEvent(pvName, typeInfo, ChangeType.TYPEINFO_MODIFIED));
				logger.debug("Received entryUpdated for pvTypeInfo");
				if(persistanceLayer != null) { 
					try { 
						persistanceLayer.putTypeInfo(pvName, typeInfo);
					} catch(Exception ex) { 
						logger.error("Exception persisting pvTypeInfo for pv " + pvName, ex);
					}
				}
			}
		}, true);
		
		eventBus.register(this);
		
		pubSub.addMessageListener(new MessageListener<PubSubEvent>() {
			@Override
			public void onMessage(Message<PubSubEvent> pubSubEventMsg) {
				PubSubEvent pubSubEvent = pubSubEventMsg.getMessageObject();
				if(pubSubEvent.getDestination() != null) {
					if(pubSubEvent.getDestination().equals("ALL") 
							|| (pubSubEvent.getDestination().startsWith(myIdentity) && pubSubEvent.getDestination().endsWith(DefaultConfigService.this.warFile.toString()))
							) {
						// We publish messages from hazelcast into this VM only if the intened WAR file is us.
						logger.debug("Publishing event into this JVM " + pubSubEvent.generateEventDescription());
						// In this case, we set the source as being the cluster to prevent republishing back into the cluster.
						pubSubEvent.markSourceAsCluster();
						eventBus.post(pubSubEvent);
					} else { 
						logger.debug("Skipping publishing event into this JVM " + pubSubEvent.generateEventDescription() + " as destination is not me " + DefaultConfigService.this.warFile.toString());
					}
				} else {
					logger.debug("Skipping publishing event with null destination");
				}
			}
		});
		
		logger.info("Done registering for changes to typeinfos");

		
		this.startupState = STARTUP_SEQUENCE.STARTUP_COMPLETE;
		configlogger.info("Start complete for webapp " + this.warFile);
	}

	@Override
	public STARTUP_SEQUENCE getStartupState() {
		return this.startupState;
	}

	@Subscribe public void updatePVSForThisAppliance(PVTypeInfoEvent event) {
		if(logger.isDebugEnabled()) logger.debug("Received pvTypeInfo change event for pv " + event.getPvName());
		PVTypeInfo typeInfo = event.getTypeInfo();
		String pvName = typeInfo.getPvName();
		if(typeInfo.getApplianceIdentity().equals(myApplianceInfo.getIdentity())) {
			if(event.getChangeType() == ChangeType.TYPEINFO_DELETED) {
				if(pvsForThisAppliance != null) {
					if(pvsForThisAppliance.contains(pvName)) {
						logger.debug("Removing pv " + pvName + " from the locally cached copy of pvs for this appliance");
						pvsForThisAppliance.remove(pvName);
						pausedPVsForThisAppliance.remove(pvName);
						// For now, we do not anticipate many PVs being deleted from the cache to worry about keeping applianceAggregateInfo upto date...
						// This may change later... 
						String[] parts = this.pvName2KeyConverter.breakIntoParts(pvName);
						for(String part : parts) { 
							parts2PVNamesForThisAppliance.get(part).remove(pvName);
						}
					}
				}
			} else {
				if(pvsForThisAppliance != null) {
					if(!pvsForThisAppliance.contains(pvName)) {
						logger.debug("Adding pv " + pvName + " to the locally cached copy of pvs for this appliance");
						pvsForThisAppliance.add(pvName);
						if(typeInfo.isPaused()) { 
							pausedPVsForThisAppliance.add(typeInfo.getPvName());
						}
						String[] parts = this.pvName2KeyConverter.breakIntoParts(pvName);
						for(String part : parts) { 
							if(!parts2PVNamesForThisAppliance.containsKey(part)) { 
								parts2PVNamesForThisAppliance.put(part, new ConcurrentSkipListSet<String>());
							}
							parts2PVNamesForThisAppliance.get(part).add(pvName);
						}
						applianceAggregateInfo.addInfoForPV(pvName, typeInfo, this);
					} else { 
						if(typeInfo.isPaused()) { 
							pausedPVsForThisAppliance.add(typeInfo.getPvName());
						} else { 
							pausedPVsForThisAppliance.remove(typeInfo.getPvName());
						}
					}
				}
			}
		}
	}
	
	@Subscribe public void publishEventIntoCluster(PubSubEvent pubSubEvent) {
		if(pubSubEvent.isSourceCluster()) { 
			logger.debug("Skipping publishing events from the cluster back into the cluster " + pubSubEvent.generateEventDescription());
			return;
		}
		
		if(pubSubEvent.getDestination().startsWith(myIdentity) && pubSubEvent.getDestination().endsWith(this.warFile.toString())) {
			logger.debug(this.warFile + " - Skipping publishing event " + pubSubEvent.generateEventDescription() + " meant for myself " + this.warFile.toString());
		} else {
			pubSubEvent.setSource(myIdentity);
			logger.debug(this.warFile + " - Publishing event from local event bus onto cluster " + pubSubEvent.generateEventDescription());
			pubSub.publish(pubSubEvent);
		}
	}

	/**
	 * Get the PVs that belong to this appliance and start archiving them
	 * Needless to day, this gets done only in the engine.
	 */
	private void archivePVSonStartup() {
		configlogger.debug("Start archiving PVs from persistence.");
		int secondsToBuffer = PVTypeInfo.getSecondsToBuffer(this);
		// To prevent broadcast storms, we pause for pausePerGroup seconds for every pausePerGroup PVs
		int currentPVCount = 0;
		int pausePerGroupPVCount = 2000;
		int pausePerGroupPauseTimeInSeconds = 2;
		for(String pvName : this.getPVsForThisAppliance()) {
			try {
				PVTypeInfo typeInfo = typeInfos.get(pvName);
				if(typeInfo == null) {
					logger.error("On restart, cannot find typeinfo for pv " + pvName + ". Not archiving");
					continue;
				}

				if(typeInfo.isPaused()) { 
					logger.debug("Skipping archiving paused PV " + pvName + " on startup");
					continue;
				}
				
				ArchDBRTypes dbrType = typeInfo.getDBRType();
				float samplingPeriod = typeInfo.getSamplingPeriod();
				SamplingMethod samplingMethod = typeInfo.getSamplingMethod();
				StoragePlugin firstDest = StoragePluginURLParser.parseStoragePlugin(typeInfo.getDataStores()[0], this);

				Timestamp lastKnownTimestamp = typeInfo.determineLastKnownEventFromStores(this);
				if(logger.isDebugEnabled()) logger.debug("Last known timestamp from ETL stores is for pv " + pvName + " is "+ TimeUtils.convertToHumanReadableString(lastKnownTimestamp));

				ArchiveEngine.archivePV(pvName, samplingPeriod, samplingMethod, secondsToBuffer, firstDest, this, dbrType,lastKnownTimestamp, typeInfo.getControllingPV(), typeInfo.getArchiveFields(), typeInfo.getHostName(), typeInfo.isUsePVAccess(), typeInfo.isUseDBEProperties()); 
				currentPVCount++;
				if(currentPVCount % pausePerGroupPVCount == 0) {
					logger.debug("Sleeping for " + pausePerGroupPauseTimeInSeconds + " to prevent CA search storms");
					Thread.sleep(pausePerGroupPauseTimeInSeconds*1000);
				}
			} catch(Throwable t) {
				logger.error("Exception starting up archiving of PV " + pvName + ". Moving on to the next pv.", t);
			}
		}
		configlogger.debug("Started " + currentPVCount + " PVs from persistence.");
	}

	@Override
	public boolean isStartupComplete() {
		return startupState == STARTUP_SEQUENCE.STARTUP_COMPLETE;
	}
	

	@Override
	public Properties getInstallationProperties() {
		return archapplproperties;
	}

	@Override
	public Collection<ApplianceInfo> getAppliancesInCluster() {
		ArrayList<ApplianceInfo> sortedAppliances = new ArrayList<ApplianceInfo>();
		for(ApplianceInfo info : appliances.values()) {
			if(appliancesInCluster.contains(info.getIdentity())) {
				sortedAppliances.add(info);
			} else {
				logger.debug("Skipping appliance that is in the persistence but not in the cluster" + info.getIdentity());
			}
		}
		
		Collections.sort(sortedAppliances, new Comparator<ApplianceInfo>() {
			@Override
			public int compare(ApplianceInfo o1, ApplianceInfo o2) {
				return o1.getIdentity().compareTo(o2.getIdentity());
			}
		});
		return sortedAppliances;
	}

	@Override
	public ApplianceInfo getMyApplianceInfo() {
		return myApplianceInfo;
	}

	@Override
	public ApplianceInfo getAppliance(String identity) {
		return appliances.get(identity);
	}

	
	@Override
	public Collection<String> getAllPVs() {
		List<PVApplianceCombo> sortedCombos = getSortedPVApplianceCombo();
		ArrayList<String> allPVs = new ArrayList<String>();
		for(PVApplianceCombo combo : sortedCombos) {
			allPVs.add(combo.pvName);
		}
		return allPVs;
	}

	@Override
	public ApplianceInfo getApplianceForPV(String pvName) {
		ApplianceInfo applianceInfo = pv2appliancemapping.get(pvName);
		if(applianceInfo == null && this.persistanceLayer != null) { 
			try { 
				PVTypeInfo typeInfo = this.persistanceLayer.getTypeInfo(pvName);
				if(typeInfo != null) { 
					applianceInfo = this.getAppliance(typeInfo.getApplianceIdentity());
				}
			} catch(IOException ex) { 
				logger.error("Exception lookin up appliance for pv in persistence", ex);
			}
		}
		return applianceInfo;
	}
	
	@Override
	public Iterable<String> getPVsForAppliance(ApplianceInfo info) {
		String identity = info.getIdentity();
		List<PVApplianceCombo> sortedCombos = getSortedPVApplianceCombo();
		ArrayList<String> pvsForAppliance = new ArrayList<String>();
		for(PVApplianceCombo combo : sortedCombos) {
			if(combo.applianceIdentity.equals(identity)) {
				pvsForAppliance.add(combo.pvName);
			}
		}
		return pvsForAppliance;
	}
	
	@Override
	public Iterable<String> getPVsForThisAppliance() {
		if(pvsForThisAppliance != null) {
			logger.debug("Returning the locally cached copy of the pvs for this appliance");
			return pvsForThisAppliance;
		} else {
			logger.debug("Fetching the list of PVs for this appliance from the mgmt app");
			JSONArray pvs = GetUrlContent.getURLContentAsJSONArray(myApplianceInfo.getMgmtURL() + "/getPVsForThisAppliance");
			LinkedList<String> retval = new LinkedList<String>();
			for(Object pv : pvs) {
				retval.add((String) pv);
			}
			return retval;
		}
	}

	@Override
	public boolean isBeingArchivedOnThisAppliance(String pvName) {
		boolean isField = PVNames.isField(pvName);
		String plainPVName = PVNames.stripFieldNameFromPVName(pvName);
		if(this.pvsForThisAppliance.contains(plainPVName)) {  
			if(isField) {
				PVTypeInfo typeInfo = this.getTypeInfoForPV(plainPVName);
				if(typeInfo != null && Arrays.asList(typeInfo.getArchiveFields()).contains(PVNames.getFieldName(pvName))) { 
					return true;
				}
			} else { 
				return true;
			}
		}
		
		if(this.aliasNamesToRealNames.containsKey(plainPVName)) { 
			plainPVName = this.aliasNamesToRealNames.get(plainPVName);

			if(this.pvsForThisAppliance.contains(plainPVName)) {  
				if(isField) {
					PVTypeInfo typeInfo = this.getTypeInfoForPV(plainPVName);
					if(typeInfo != null && Arrays.asList(typeInfo.getArchiveFields()).contains(PVNames.getFieldName(pvName))) { 
						return true;
					}
				} else { 
					return true;
				}
			}
		}

		
		return false;
	}
	
	@Override
	public Set<String> getPVsForApplianceMatchingRegex(String nameToMatch) {
		logger.debug("Finding matching names for " + nameToMatch);
		LinkedList<String> fixedStringParts = new LinkedList<String>();
		String[] parts = this.pvName2KeyConverter.breakIntoParts(nameToMatch);
		Pattern fixedStringParttern = Pattern.compile("[a-zA-Z_0-9-]+");
		for(String part : parts) { 
			if(fixedStringParttern.matcher(part).matches()) { 
				logger.debug("Fixed string part " +  part);
				fixedStringParts.add(part);
			} else { 
				logger.debug("Regex string part " + part);
			}
		}
		
		if(fixedStringParts.size() > 0) {
			HashSet<String> ret = new HashSet<String>();
			HashSet<String> namesSubset = new HashSet<String>();
			// This reverse is probably specific to SLAC's namespace rules but it does make a big difference. 
			// Perhaps we can use a more intelligent way of choosing the specific path thru the trie.
			Collections.reverse(fixedStringParts);
			for(String fixedStringPart : fixedStringParts) { 
				ConcurrentSkipListSet<String> pvNamesForPart = parts2PVNamesForThisAppliance.get(fixedStringPart);
				if(pvNamesForPart != null) { 
					if(namesSubset.isEmpty()) { 
						namesSubset.addAll(pvNamesForPart);
					} else { 
						namesSubset.retainAll(pvNamesForPart);
					}
				}
			}
			logger.debug("Using fixed string path matching against names " + namesSubset.size());
			Pattern pattern = Pattern.compile(nameToMatch);
			for(String pvName : namesSubset) { 
				if(pattern.matcher(pvName).matches()) { 
					ret.add(pvName);
				}
			}
			return ret;
		} else { 
			// The use pattern did not have any fixed elements at all. 
			// In this case we do brute force matching; should take longer.
			// This is also not optimal but probably don't want yet another list of PV's
			Pattern pattern = Pattern.compile(nameToMatch);
			HashSet<String> allNames = new HashSet<String>();
			HashSet<String> ret = new HashSet<String>();
			logger.debug("Using brute force pattern matching against names");
			for(ConcurrentSkipListSet<String> pvNamesForPart : parts2PVNamesForThisAppliance.values()) { 
				allNames.addAll(pvNamesForPart);
			}
			for(String pvName : allNames) { 
				if(pattern.matcher(pvName).matches()) { 
					ret.add(pvName);
				}
			}
			return ret;
		}
	}	


	@Override
	public ApplianceAggregateInfo getAggregatedApplianceInfo(ApplianceInfo applianceInfo) throws IOException {
		if(applianceInfo.getIdentity().equals(myApplianceInfo.getIdentity()) && this.warFile == WAR_FILE.MGMT) {
			logger.debug("Returning local copy of appliance info for " + applianceInfo.getIdentity());
			return applianceAggregateInfo;
		} else {
			try {
				JSONObject aggregateInfo = GetUrlContent.getURLContentAsJSONObject(applianceInfo.getMgmtURL() + "/getAggregatedApplianceInfo", false);
				JSONDecoder<ApplianceAggregateInfo> jsonDecoder = JSONDecoder.getDecoder(ApplianceAggregateInfo.class);
				ApplianceAggregateInfo retval = new ApplianceAggregateInfo();
				jsonDecoder.decode(aggregateInfo, retval);
				return retval;
			} catch(Exception ex) {
				throw new IOException(ex);
			}
		}
	}


	@Override
	public void registerPVToAppliance(String pvName, ApplianceInfo applianceInfo) throws AlreadyRegisteredException {
		ApplianceInfo info = pv2appliancemapping.get(pvName);
		if(info != null) throw new AlreadyRegisteredException(info);
		pv2appliancemapping.put(pvName, applianceInfo);
	}
	
	
	@Override
	public PVTypeInfo getTypeInfoForPV(String pvName) {
		if(typeInfos.containsKey(pvName))  {
			logger.debug("Retrieving typeinfo from cache for pv " + pvName);
			return typeInfos.get(pvName);
		}
		
		return null;
	}
	
	
	@Override
	public void updateTypeInfoForPV(String pvName, PVTypeInfo typeInfo) {
		logger.debug("Updating typeinfo for " + pvName);
		if(!typeInfo.keyAlreadyGenerated()) { 
			// This call should also typically set the chunk key in the type info.
			this.pvName2KeyConverter.convertPVNameToKey(pvName);
		}
		
		typeInfos.put(pvName, typeInfo);
	}

	@Override
	public void removePVFromCluster(String pvName) {
		logger.info("Removing PV from cluster.." + pvName);
		pv2appliancemapping.remove(pvName);
		pvsForThisAppliance.remove(pvName);		
		typeInfos.remove(pvName);
		pausedPVsForThisAppliance.remove(pvName);
		String[] parts = this.pvName2KeyConverter.breakIntoParts(pvName);
		for(String part : parts) { 
			ConcurrentSkipListSet<String> pvNamesForPart = parts2PVNamesForThisAppliance.get(part);
			if(pvNamesForPart != null) { 
				pvNamesForPart.remove(pvName);
			}
		}
	}

	private class PVApplianceCombo implements Comparable<PVApplianceCombo> {
		String applianceIdentity;
		String pvName;
		public PVApplianceCombo(String applianceIdentity, String pvName) {
			this.applianceIdentity = applianceIdentity;
			this.pvName = pvName;
		}
		@Override
		public int compareTo(PVApplianceCombo other) {
			if(this.applianceIdentity.equals(other.applianceIdentity)) {
				return this.pvName.compareTo(other.pvName);
			} else {
				return this.applianceIdentity.compareTo(other.applianceIdentity);
			}
		}
	}
	
	private List<PVApplianceCombo> getSortedPVApplianceCombo() {
		ArrayList<PVApplianceCombo> sortedCombos = new ArrayList<PVApplianceCombo>();
		for(Map.Entry<String, ApplianceInfo> entry : pv2appliancemapping.entrySet()) {
			sortedCombos.add(new PVApplianceCombo(entry.getValue().getIdentity(), entry.getKey()));
		}
		Collections.sort(sortedCombos);
		return sortedCombos;
	}
	
	
	@Override
	public void addToArchiveRequests(String pvName, UserSpecifiedSamplingParams userSpecifiedSamplingParams) {
		archivePVRequests.put(pvName, userSpecifiedSamplingParams);
		try { 
			persistanceLayer.putArchivePVRequest(pvName, userSpecifiedSamplingParams);
		} catch(IOException ex) {
			logger.error("Exception adding request to persistence", ex);
		}
	}

	@Override
	public void updateArchiveRequest(String pvName, UserSpecifiedSamplingParams userSpecifiedSamplingParams) { 
		try { 
			if(persistanceLayer.getArchivePVRequest(pvName) != null) { 
				archivePVRequests.put(pvName, userSpecifiedSamplingParams);
				persistanceLayer.putArchivePVRequest(pvName, userSpecifiedSamplingParams);
			} else { 
				logger.error("Do not have user specified params for pv " + pvName + " in this appliance. Not updating.");
			}
		} catch(IOException ex) {
			logger.error("Exception updating request in persistence", ex);
		}
	}

	

	@Override
	public Set<String> getArchiveRequestsCurrentlyInWorkflow() {
		return new HashSet<String>(archivePVRequests.keySet());
	}
	
	@Override
	public boolean doesPVHaveArchiveRequestInWorkflow(String pvname) {
		return archivePVRequests.containsKey(pvname);
	}
	
	@Override
	public UserSpecifiedSamplingParams getUserSpecifiedSamplingParams(String pvName) {
		return archivePVRequests.get(pvName);
	}

	@Override
	public void archiveRequestWorkflowCompleted(String pvName) {
		archivePVRequests.remove(pvName);
		try { 
			persistanceLayer.removeArchivePVRequest(pvName);
		} catch(IOException ex) {
			logger.error("Exception removing request from persistence", ex);
		}
	}

	@Override
	public int getInitialDelayBeforeStartingArchiveRequestWorkflow() {
		int appliancesInCluster = 0;
		for(@SuppressWarnings("unused") ApplianceInfo info : this.getAppliancesInCluster()) { 
			appliancesInCluster++;
		}
		
		int initialDelayInSeconds = 10;
		if(appliancesInCluster > 1) { 
			// We use a longer initial delay here to get all the appliances in the cluster a chance to restart
			initialDelayInSeconds = 30*60;
		}
		
		return initialDelayInSeconds;
	}
	
	
	@Override
	public void addAlias(String aliasName, String realName) {
		aliasNamesToRealNames.put(aliasName, realName);
		try { 
			persistanceLayer.putAliasNamesToRealName(aliasName, realName);
		} catch(IOException ex) {
			logger.error("Exception adding alias name to persistence " + aliasName, ex);
		}
		
		// Add aliases into the trie
		String[] parts = this.pvName2KeyConverter.breakIntoParts(aliasName);
		for(String part : parts) { 
			if(!parts2PVNamesForThisAppliance.containsKey(part)) { 
				parts2PVNamesForThisAppliance.put(part, new ConcurrentSkipListSet<String>());
			}
			parts2PVNamesForThisAppliance.get(part).add(aliasName);
		}
	}
	
	
	@Override
	public void removeAlias(String aliasName, String realName) {
		aliasNamesToRealNames.remove(aliasName);
		try { 
			persistanceLayer.removeAliasName(aliasName, realName);
		} catch(IOException ex) {
			logger.error("Exception removing alias name from persistence " + aliasName, ex);
		}
		
		// Remove the aliasname from the trie
		String[] parts = this.pvName2KeyConverter.breakIntoParts(aliasName);
		for(String part : parts) { 
			parts2PVNamesForThisAppliance.get(part).remove(aliasName);
		}
	}

	

	@Override
	public String getRealNameForAlias(String aliasName) {
		return aliasNamesToRealNames.get(aliasName);
	}
	
	@Override
	public List<String> getAllAliases() {
		return new ArrayList<String>(aliasNamesToRealNames.keySet());
	}



	private final String[] extraFields = new String[] {"MDEL","ADEL", "SCAN","RTYP"};
	@Override
	public String[] getExtraFields() {
		return extraFields;
	}
	
	
	@Override
	public Set<String> getRuntimeFields() {
		return runTimeFields;
	}


	
	@Override
	public PBThreeTierETLPVLookup getETLLookup() {
		return etlPVLookup;
	}
	
	
	@Override
	public RetrievalState getRetrievalRuntimeState() {
		return retrievalState;
	}

	
	@Override
	public boolean isShuttingDown() {
		return startupExecutor.isShutdown();
	}
	
	@Override
	public void addShutdownHook(Runnable runnable) {
		shutdownHooks.add(runnable);
		
	}
	
	private void runShutDownHooksAndCleanup() { 
		LinkedList<Runnable> shutDnHooks = new LinkedList<Runnable>(this.shutdownHooks);
		Collections.reverse(shutDnHooks);
		logger.debug("Running shutdown hooks in webapp " + this.warFile);
		for(Runnable shutdownHook : shutDnHooks) {
			try {
				shutdownHook.run();
			} catch(Throwable t) {
				logger.warn("Exception shutting down service using shutdown hook " + shutdownHook.toString(), t);
			}
		}
		logger.debug("Done running shutdown hooks in webapp " + this.warFile);
	}

	@Override
	public void shutdownNow() {
		this.runShutDownHooksAndCleanup();
	}

	

	
	
	@Override
	public Map<String, String> getExternalArchiverDataServers() {
		return channelArchiverDataServers;
	}

	@Override
	public void addExternalArchiverDataServer(String serverURL, String archivesCSV) throws IOException {
		String[] archives = archivesCSV.split(",");
		boolean loadCAPVs = false;
		if(!this.getExternalArchiverDataServers().containsKey(serverURL)) { 
			this.getExternalArchiverDataServers().put(serverURL, archivesCSV);
			loadCAPVs = true;
		} else { 
			logger.info(serverURL + " already exists in the map. So, skipping loading PVs from the external server.");
		}
		
		
		// We always add to persistence; whether this is from the UI or from the other appliances in the cluster.
		if(this.persistanceLayer != null) { 
			persistanceLayer.putExternalDataServer(serverURL, archivesCSV);
		}
		
		try {
			// We load PVs from the external server only if this is the first server starting up...
			if(loadCAPVs) { 
				for(int i = 0; i < archives.length; i++) {
					String archive = archives[i];
					loadExternalArchiverPVs(serverURL, archive);
				}
			}
		} catch(Exception ex) {
			logger.error("Exception adding Channel Archiver archives " + serverURL + " - " + archivesCSV, ex);
			throw new IOException(ex);
		}
	}
	
	
	@Override
	public void removeExternalArchiverDataServer(String serverURL, String archivesCSV) throws IOException {
		this.getExternalArchiverDataServers().remove(serverURL);
		// We always add to persistence; whether this is from the UI or from the other appliances in the cluster.
		if(this.persistanceLayer != null) { 
			logger.info("Removing the channel archiver server " + serverURL + " from the persistent store.");
			persistanceLayer.removeExternalDataServer(serverURL, archivesCSV);
		}

	}


	/**
	 * Given a external Archiver data server URL and an archive; 
	 * If this is a ChannelArchiver (archives != pbraw); 
	 * this adds the PVs in the Channel Archiver so that they can be proxied.
	 * @param serverURL
	 * @param archive
	 * @throws IOException
	 * @throws SAXException
	 */
	private void loadExternalArchiverPVs(String serverURL, String archive) throws IOException, SAXException {
		if(archive.equals("pbraw")) {
			logger.debug("We do not load PV names from external EPICS archiver appliances. These can number in the multiple millions and the respone on retrieval is fast enough anyways");
			return;
		}
		
		ChannelArchiverDataServerInfo serverInfo = new ChannelArchiverDataServerInfo(serverURL, archive);
		NamesHandler handler = new NamesHandler();
		logger.debug("Getting list of PV's from Channel Archiver Server at " + serverURL + " using index " + archive);
		XMLRPCClient.archiverNames(serverURL, archive, handler);
		HashMap<String, List<ChannelArchiverDataServerPVInfo>> tempPVNames = new HashMap<String, List<ChannelArchiverDataServerPVInfo>>();
		long totalPVsProxied = 0;
		for(NamesHandler.ChannelDescription pvChannelDesc : handler.getChannels()) {
			String pvName = PVNames.normalizePVName(pvChannelDesc.getName());
			if(this.pv2ChannelArchiverDataServer.containsKey(pvName)) { 
				List<ChannelArchiverDataServerPVInfo> alreadyExistingServers = this.pv2ChannelArchiverDataServer.get(pvName);
				logger.debug("Adding new server to already existing ChannelArchiver server for " + pvName);
				addExternalCAServerToExistingList(alreadyExistingServers, serverInfo, pvChannelDesc);
				tempPVNames.put(pvName, alreadyExistingServers);
			} else if(tempPVNames.containsKey(pvName)) { 
				List<ChannelArchiverDataServerPVInfo> alreadyExistingServers = tempPVNames.get(pvName);
				logger.debug("Adding new server to already existing ChannelArchiver server (in tempspace) for " + pvName);
				addExternalCAServerToExistingList(alreadyExistingServers, serverInfo, pvChannelDesc);
				tempPVNames.put(pvName, alreadyExistingServers);
			} else { 
				List<ChannelArchiverDataServerPVInfo> caServersForPV = new ArrayList<ChannelArchiverDataServerPVInfo>();
				caServersForPV.add(new ChannelArchiverDataServerPVInfo(serverInfo, pvChannelDesc.getStartSec(), pvChannelDesc.getEndSec()));
				tempPVNames.put(pvName, caServersForPV);
			}
			
			
			if(tempPVNames.size() > 1000) { 
				this.pv2ChannelArchiverDataServer.putAll(tempPVNames);
				totalPVsProxied += tempPVNames.size();
				tempPVNames.clear();
			}
		}
		if(!tempPVNames.isEmpty()) { 
			this.pv2ChannelArchiverDataServer.putAll(tempPVNames);
			totalPVsProxied += tempPVNames.size();
			tempPVNames.clear();
		}
		if(logger.isDebugEnabled()) logger.debug("Proxied a total of " + totalPVsProxied + " from server " + serverURL + " using archive " + archive);
		
	}
	
	private static void addExternalCAServerToExistingList(List<ChannelArchiverDataServerPVInfo> alreadyExistingServers, ChannelArchiverDataServerInfo serverInfo, NamesHandler.ChannelDescription pvChannelDesc) {
		List<ChannelArchiverDataServerPVInfo> copyOfAlreadyExistingServers = new LinkedList<ChannelArchiverDataServerPVInfo>();
		for(ChannelArchiverDataServerPVInfo alreadyExistingServer : alreadyExistingServers) { 
			if(alreadyExistingServer.getServerInfo().equals(serverInfo)) { 
				logger.debug("Removing a channel archiver server that already exists " + alreadyExistingServer.toString());
			} else { 
				copyOfAlreadyExistingServers.add(alreadyExistingServer);
			}
		}
		
		int beforeCount = alreadyExistingServers.size();
		alreadyExistingServers.clear();
		alreadyExistingServers.addAll(copyOfAlreadyExistingServers);
		
		// Readd the CA server - this should take into account any updated start times, end times and so on.
		alreadyExistingServers.add(new ChannelArchiverDataServerPVInfo(serverInfo, pvChannelDesc.getStartSec(), pvChannelDesc.getEndSec()));
		
		int afterCount = alreadyExistingServers.size();
		logger.debug("We had " + beforeCount + " and now we have " + afterCount + " when adding external ChannelArchiver server");

		// Sort the servers by ascending time stamps before adding it back.
		ChannelArchiverDataServerPVInfo.sortServersBasedOnStartAndEndSecs(alreadyExistingServers);
	}
	
	
	@Override
	public List<ChannelArchiverDataServerPVInfo> getChannelArchiverDataServers(String pvName) {
		String normalizedPVName = PVNames.normalizePVName(pvName);
		logger.debug("Looking for CA sever for pv " + normalizedPVName);
		return pv2ChannelArchiverDataServer.get(normalizedPVName);
	}

	@Override
	public PolicyConfig computePolicyForPV(String pvName, MetaInfo metaInfo, UserSpecifiedSamplingParams userSpecParams) throws IOException {
		try(InputStream is = this.getPolicyText()) {
			logger.debug("Computing policy for pvName");
			HashMap<String, Object> pvInfo = new HashMap<String, Object>();
			pvInfo.put("dbrtype", metaInfo.getArchDBRTypes().toString());
			pvInfo.put("elementCount", metaInfo.getCount());
			pvInfo.put("eventRate", metaInfo.getEventRate());
			pvInfo.put("storageRate", metaInfo.getStorageRate());
			pvInfo.put("aliasName", metaInfo.getAliasName());
			if(userSpecParams != null && userSpecParams.getPolicyName() != null) {
				logger.debug("Passing user override of policy " + userSpecParams.getPolicyName() + " as the dict entry policyName");
				pvInfo.put("policyName", userSpecParams.getPolicyName());
			}
			if(userSpecParams.getControllingPV() != null) { 
				pvInfo.put("controlPV", userSpecParams.getControllingPV());
			}

			
			HashMap<String,String> otherMetaInfo = metaInfo.getOtherMetaInfo();
			for(String otherMetaInfoKey : this.getExtraFields()) {
				if(otherMetaInfo.containsKey(otherMetaInfoKey)) {
					if(otherMetaInfoKey.equals("ADEL") || otherMetaInfoKey.equals("MDEL")) {
						try { 
							pvInfo.put(otherMetaInfoKey, Double.parseDouble(otherMetaInfo.get(otherMetaInfoKey)));
						} catch(Exception ex) { 
							logger.error("Exception adding MDEL and ADEL to the info", ex);
						}
					} else { 
						pvInfo.put(otherMetaInfoKey,otherMetaInfo.get(otherMetaInfoKey));
					}
				}
			}

			if(logger.isDebugEnabled()) {
				StringBuilder buf = new StringBuilder();
				buf.append("Before computing policy for");
				buf.append(pvName);
				buf.append(" pvInfo is \n");
				for(String key : pvInfo.keySet()) {
					buf.append(key);
					buf.append("=");
					buf.append(pvInfo.get(key));
					buf.append("\n");
				}
				logger.debug(buf.toString());
			}
			
			try {
				// We only have one policy in the cache...
				ExecutePolicy executePolicy = theExecutionPolicy.get("ThePolicy");
				PolicyConfig policyConfig = executePolicy.computePolicyForPV(pvName, pvInfo);
				return policyConfig;
			} catch (ExecutionException e) {
				Throwable cause = e.getCause();
				logger.error("Exception executing policy for pv " + pvName, cause);
				if(cause instanceof IOException) { 
					throw (IOException) cause;
				} else { 
					throw new IOException(cause);
				}
			}
		}
	}
	
	
	
	@Override
	public HashMap<String, String> getPoliciesInInstallation() throws IOException {
		try(ExecutePolicy executePolicy = new ExecutePolicy(this)) { 
			return executePolicy.getPolicyList();
		}
	}
	
	
	@Override
	public List<String> getFieldsArchivedAsPartOfStream() throws IOException {
		try(ExecutePolicy executePolicy = new ExecutePolicy(this)) { 
			return executePolicy.getFieldsArchivedAsPartOfStream();
		}
	}

	@Override
	public TypeSystem getArchiverTypeSystem() {
		return new PBTypeSystem();
	}


	private boolean finishedLoggingPolicyLocation = false;
	@Override
	public InputStream getPolicyText() throws IOException {
		String policiesPyFile = System.getProperty(ARCHAPPL_POLICIES);
		if(policiesPyFile == null) {
			policiesPyFile = System.getenv(ARCHAPPL_POLICIES);
			if(policiesPyFile != null) { 
				if(!finishedLoggingPolicyLocation) { configlogger.info("Obtained policies location from environment " + policiesPyFile); finishedLoggingPolicyLocation = true; } 
				return new FileInputStream(new File(policiesPyFile));
			} else { 
				logger.info("Looking for /WEB-INF/classes/policies.py in classpath");
				if(servletContext != null) {
					if(!finishedLoggingPolicyLocation) { configlogger.info("Using policies file /WEB-INF/classes/policies.py found in classpath"); finishedLoggingPolicyLocation = true; } 
					return servletContext.getResourceAsStream("/WEB-INF/classes/policies.py");
				} else {
					throw new IOException("Cannot determine location of policies file as both servlet context and webInfClassesFolder are null");
				}
			}
		} else { 
			if(!finishedLoggingPolicyLocation) { configlogger.info("Obtained policies location from system property " + policiesPyFile); finishedLoggingPolicyLocation = true; } 
			return new FileInputStream(new File(policiesPyFile));
		}
	}
	
	

	@Override
	public EngineContext getEngineContext() {
		return engineContext;
	}

	@Override
	public MgmtRuntimeState getMgmtRuntimeState() {
		return mgmtRuntime;
	}

	@Override
	public WAR_FILE getWarFile() {
		return warFile;
	}
		
	/**
	 * Return a string representation of the member.
	 * @param member
	 * @return
	 */
	private String getMemberKey(Member member) {
		// We use deprecated versions of the methods as the non-deprecated versions do not work as of 2.0x?
		return member.getSocketAddress().toString();
	}

	@Override
	public EventBus getEventBus() {
		return eventBus;
	}

	@Override
	public PVNameToKeyMapping getPVNameToKeyConverter() {
		return pvName2KeyConverter;
	}
	
	/**
	 * Load typeInfos into the cluster hashmaps from the persistence layer on startup.
	 * To avoid overwhelming the cluster, we batch the loads 
	 */
	private void loadTypeInfosFromPersistence() {
		try { 
			configlogger.info("Loading PVTypeInfo from persistence");
			List<String> upgradedPVs = new LinkedList<String>();
			List<String> pvNamesFromPersistence = persistanceLayer.getTypeInfoKeys();
			HashMap<String, PVTypeInfo> newTypeInfos = new HashMap<String, PVTypeInfo>();
			HashMap<String, ApplianceInfo> newPVMappings = new HashMap<String, ApplianceInfo>();
			int objectCount = 0;
			int batch = 0;
			int clusterPVCount = 0;
			for(String pvNameFromPersistence : pvNamesFromPersistence) {
				PVTypeInfo typeInfo = persistanceLayer.getTypeInfo(pvNameFromPersistence);
				if(typeInfo.getApplianceIdentity().equals(myIdentity)) {
					// Here's where we put schema update logic
					upgradeTypeInfo(typeInfo, upgradedPVs);
					
					String pvName = typeInfo.getPvName();
					newTypeInfos.put(pvName, typeInfo);
					newPVMappings.put(pvName, appliances.get(typeInfo.getApplianceIdentity()));
					pvsForThisAppliance.add(pvName);
					if(typeInfo.isPaused()) { 
						pausedPVsForThisAppliance.add(pvName);
					}
					String[] parts = this.pvName2KeyConverter.breakIntoParts(pvName);
					for(String part : parts) { 
						if(!parts2PVNamesForThisAppliance.containsKey(part)) { 
							parts2PVNamesForThisAppliance.put(part, new ConcurrentSkipListSet<String>());
						}
						parts2PVNamesForThisAppliance.get(part).add(pvName);
					}

					objectCount++;
				}
				// Add in batch sizes of 1000 or so...
				if(objectCount > 1000) {
					this.typeInfos.putAll(newTypeInfos);
					this.pv2appliancemapping.putAll(newPVMappings);
					for(String pvName : newTypeInfos.keySet()) {
						applianceAggregateInfo.addInfoForPV(pvName, newTypeInfos.get(pvName), this);
					}
					clusterPVCount += newTypeInfos.size();
					newTypeInfos = new HashMap<String, PVTypeInfo>();
					newPVMappings = new HashMap<String, ApplianceInfo>();
					objectCount = 0;
					logger.debug("Adding next batch of PVs " + batch++);
				}
			}

			if(newTypeInfos.size() > 0) {
				logger.debug("Adding final batch of PVs from persistence");
				this.typeInfos.putAll(newTypeInfos);
				this.pv2appliancemapping.putAll(newPVMappings);
				for(String pvName : newTypeInfos.keySet()) {
					applianceAggregateInfo.addInfoForPV(pvName, newTypeInfos.get(pvName), this);
				}
				clusterPVCount += newTypeInfos.size();
			}

			configlogger.info("Done loading " + + clusterPVCount + " PVs from persistence into cluster");
			
			for(String upgradedPVName : upgradedPVs) { 
				logger.debug("PV " + upgradedPVName + "'s schema was upgraded");
				persistanceLayer.putTypeInfo(upgradedPVName, getTypeInfoForPV(upgradedPVName));
				logger.debug("Done persisting upgraded PV's " + upgradedPVName + "'s typeInfo");				
			}
		} catch(Exception ex) {
			configlogger.error("Exception loading PVs from persistence", ex);
		}
	}

	/**
	 * The occasional upgrade to PVTypeInfo schema is handed here.
	 * @param typeInfo - Typeinfo to be upgraded
	 * @param upgradedPVs - Add the pvName here if we actually did an upgrade to the typeInfo.
	 */
	private void upgradeTypeInfo(PVTypeInfo typeInfo, List<String> upgradedPVs) {
		// We added the chunkKey to typeInfo to permanently remember the key mapping to accomodate slowly changing key mappings. 
		// This could be a possibility after talking to SPEAR folks.
		if(!typeInfo.keyAlreadyGenerated()) { 
			typeInfo.setChunkKey(this.pvName2KeyConverter.convertPVNameToKey(typeInfo.getPvName()));
			upgradedPVs.add(typeInfo.getPvName());
		}
	}
	
	/**
	 * Load alias mappings from persistence on startup in batches
	 */
	private void loadAliasesFromPersistence() {
		try { 
			configlogger.info("Loading aliases from persistence");
			List<String> pvNamesFromPersistence = persistanceLayer.getAliasNamesToRealNamesKeys();
			HashMap<String, String> newAliases = new HashMap<String, String>();
			int objectCount = 0;
			int batch = 0;
			int clusterPVCount = 0;
			for(String pvNameFromPersistence : pvNamesFromPersistence) {
				String realName = persistanceLayer.getAliasNamesToRealName(pvNameFromPersistence);
				if(this.pvsForThisAppliance.contains(realName)) {
					newAliases.put(pvNameFromPersistence, realName);
					// Add the alias into the trie
					String[] parts = this.pvName2KeyConverter.breakIntoParts(pvNameFromPersistence);
					for(String part : parts) { 
						if(!parts2PVNamesForThisAppliance.containsKey(part)) { 
							parts2PVNamesForThisAppliance.put(part, new ConcurrentSkipListSet<String>());
						}
						parts2PVNamesForThisAppliance.get(part).add(pvNameFromPersistence);
					}					
				}
				// Add in batch sizes of 1000 or so...
				if(objectCount > 1000) {
					this.aliasNamesToRealNames.putAll(newAliases);
					clusterPVCount += newAliases.size();
					newAliases = new HashMap<String, String>();
					objectCount = 0;
					logger.debug("Adding next batch of aliases " + batch++);
				}
			}

			if(newAliases.size() > 0) {
				logger.debug("Adding final batch of aliases from persistence");
				this.aliasNamesToRealNames.putAll(newAliases);
				clusterPVCount += newAliases.size();
			}

			configlogger.info("Done loading " + clusterPVCount + " aliases from persistence into cluster ");
		} catch(Exception ex) {
			configlogger.error("Exception loading aliases from persistence", ex);
		}
	}
	
	
	/**
	 * Load any pending archive requests that have not been fulfilled yet on startup
	 * Also, start their workflows..
	 * @throws ConfigException
	 */
	private void loadArchiveRequestsFromPersistence() throws ConfigException {
		try { 
			configlogger.info("Loading archive requests from persistence");
			List<String> pvNamesFromPersistence = persistanceLayer.getArchivePVRequestsKeys();
			HashMap<String, UserSpecifiedSamplingParams> newArchiveRequests = new HashMap<String, UserSpecifiedSamplingParams>();
			int objectCount = 0;
			int batch = 0;
			int clusterPVCount = 0;
			for(String pvNameFromPersistence : pvNamesFromPersistence) {
				UserSpecifiedSamplingParams userSpecifiedParams = persistanceLayer.getArchivePVRequest(pvNameFromPersistence);
				// We should not need to add an appliance check here.. However, if after production deployment, we determine we need to do so; this is the right place.
				newArchiveRequests.put(pvNameFromPersistence, userSpecifiedParams);
				// Add in batch sizes of 1000 or so...
				if(objectCount > 1000) {
					this.archivePVRequests.putAll(newArchiveRequests);
					clusterPVCount += newArchiveRequests.size();
					newArchiveRequests = new HashMap<String, UserSpecifiedSamplingParams>();
					objectCount = 0;
					logger.debug("Adding next batch of archive pv requests " + batch++);
				}
			}

			if(newArchiveRequests.size() > 0) {
				logger.debug("Adding final batch of archive pv requests from persistence");
				this.archivePVRequests.putAll(newArchiveRequests);
				clusterPVCount += newArchiveRequests.size();
			}

			configlogger.info("Done loading " + clusterPVCount + " archive pv requests from persistence into cluster ");
		} catch(Exception ex) {
			configlogger.error("Exception loading archive pv requests from persistence", ex);
		}
	}


	/**
	 * Initialize the persistenceLayer using environment/system property.
	 * By default, initialize the MySQLPersistence
	 * @throws ConfigException
	 */
	private void initializePersistenceLayer() throws ConfigException {
		String persistenceFromEnv = System.getenv(ARCHAPPL_PERSISTENCE_LAYER);
		if(persistenceFromEnv == null || persistenceFromEnv.equals("")) {
			persistenceFromEnv = System.getProperty(ARCHAPPL_PERSISTENCE_LAYER);
		}
		if(persistenceFromEnv == null || persistenceFromEnv.equals("")) {
			logger.info("Using MYSQL for persistence; we expect to find a JNDI connection pool called jdbc/archappl");
			persistanceLayer = new MySQLPersistence();
		} else {
			try { 
				logger.info("Using persistence provided by class " + persistenceFromEnv);
				persistanceLayer = (ConfigPersistence) getClass().getClassLoader().loadClass(persistenceFromEnv).newInstance();
			} catch(Exception ex) {
				throw new ConfigException("Exception initializing persistence layer using " + persistenceFromEnv, ex);
			}
		}
	}

	public ProcessMetrics getProcessMetrics() {
		return processMetrics;
	}
	
	
	@Override
	public String getWebInfFolder() {
		return servletContext.getRealPath("WEB-INF/");
	}
	
	
	private void loadExternalServersFromPersistence() throws ConfigException {
		try { 
			configlogger.info("Loading external servers from persistence");
			List<String> externalServerKeys = persistanceLayer.getExternalDataServersKeys();
			for(String serverUrl : externalServerKeys) {
				String archivesCSV = persistanceLayer.getExternalDataServer(serverUrl);
				if(this.getExternalArchiverDataServers().containsKey(serverUrl)) { 
					configlogger.info("Skipping adding " + serverUrl + " on this appliance as another appliance has already added it");
				} else { 
					this.getExternalArchiverDataServers().put(serverUrl, archivesCSV);
					String[] archives = archivesCSV.split(",");

					this.startupExecutor.schedule(new Runnable() {
						@Override
						public void run() {
							try {
								for(int i = 0; i < archives.length; i++) {
									String archive = archives[i];
									loadExternalArchiverPVs(serverUrl, archive);
								}
							} catch(Exception ex) {
								logger.error("Exception adding Channel Archiver archives " + serverUrl + " - " + archivesCSV, ex);
							}
						}
					}, 15, TimeUnit.SECONDS); 
				}
			}
			configlogger.info("Done loading external servers from persistence ");
		} catch(Exception ex) {
			configlogger.error("Exception loading external servers from persistence", ex);
		}
	}
	
	
	private void registerForNewExternalServers(IMap<Object, Object> dataServerMap) throws ConfigException {
		dataServerMap.addEntryListener(new EntryAddedListener<Object, Object>() {
			@Override
			public void entryAdded(EntryEvent<Object, Object> arg0) {
				String url = (String) arg0.getKey();
				String archivesCSV = (String) arg0.getValue();
				try { 
					addExternalArchiverDataServer(url, archivesCSV);
				} catch(Exception ex) { 
					logger.error("Exception syncing external data server " + url + archivesCSV, ex);
				}
			}
		}, true);
		dataServerMap.addEntryListener(new EntryRemovedListener<Object, Object>() {
			@Override
			public void entryRemoved(EntryEvent<Object, Object> arg0) {
				String url = (String) arg0.getKey();
				String archivesCSV = (String) arg0.getValue();
				try { 
					removeExternalArchiverDataServer(url, archivesCSV);
				} catch(Exception ex) { 
					logger.error("Exception syncing external data server " + url + archivesCSV, ex);
				}
			}
		}, true);
	}

	@Override
	public Set<String> getPausedPVsInThisAppliance() {
		if(pausedPVsForThisAppliance != null) { 
			logger.debug("Returning the locally cached copy of the paused pvs for this appliance");
			return pausedPVsForThisAppliance;
		} else {
			logger.debug("Fetching the list of paused PVs for this appliance from the mgmt app");
			JSONArray pvs = GetUrlContent.getURLContentAsJSONArray(myApplianceInfo.getMgmtURL() + "/getPausedPVsForThisAppliance");
			HashSet<String> retval = new HashSet<String>();
			for(Object pv : pvs) {
				retval.add((String) pv);
			}
			return retval;
		}
	}

	@Override
	public void refreshPVDataFromChannelArchiverDataServers() {
		Map<String, String> existingCAServers = this.getExternalArchiverDataServers();
		for(String serverURL : existingCAServers.keySet()) { 
			String archivesCSV = existingCAServers.get(serverURL);
			String[] archives = archivesCSV.split(",");

			try {
				for(int i = 0; i < archives.length; i++) {
					String archive = archives[i];
					loadExternalArchiverPVs(serverURL, archive);
				}
			} catch(Throwable ex) {
				logger.error("Exception adding Channel Archiver archives " + serverURL + " - " + archivesCSV, ex);
			}
		}
	}

	@Override
	public boolean getNamedFlag(String name) {
		if(name == null) { 
			return false;
		}
		if(name.equalsIgnoreCase("false")) { 
			return false;
		}
		if(name.equalsIgnoreCase("true")) { 
			return true;
		}
		if(namedFlags.containsKey(name)) { 
			return namedFlags.get(name);
		}
		// If we don't know about this named flag, we return false;
		return false;
	}

	@Override
	public void setNamedFlag(String name, boolean value) {
		namedFlags.put(name, value);
	}

	@Override
	public Set<String> getNamedFlagNames() {
		return namedFlags.keySet();
	}
}