SearchIndexBuilderWorkerImpl.java

/**********************************************************************************
 * $URL: https://source.sakaiproject.org/svn/search/trunk/search-impl/impl/src/java/org/sakaiproject/search/component/service/impl/SearchIndexBuilderWorkerImpl.java $
 * $Id: SearchIndexBuilderWorkerImpl.java 111643 2012-08-20 13:41:59Z david.horwitz@uct.ac.za $
 ***********************************************************************************
 *
 * Copyright (c) 2003, 2004, 2005, 2006, 2007, 2008 The Sakai Foundation
 *
 * Licensed under the Educational Community License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *       http://www.opensource.org/licenses/ECL-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 **********************************************************************************/

package org.sakaiproject.search.component.service.impl;

import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Observable;
import java.util.Observer;
import java.util.UUID;

import javax.sql.DataSource;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.hibernate.HibernateException;
import org.sakaiproject.component.api.ServerConfigurationService;
import org.sakaiproject.entity.api.EntityManager;
import org.sakaiproject.event.api.EventTrackingService;
import org.sakaiproject.search.api.SearchIndexBuilderWorker;
import org.sakaiproject.search.api.SearchService;
import org.sakaiproject.search.dao.SearchIndexBuilderWorkerDao;
import org.sakaiproject.search.indexer.api.IndexQueueListener;
import org.sakaiproject.search.model.SearchBuilderItem;
import org.sakaiproject.search.model.SearchWriterLock;
import org.sakaiproject.search.model.impl.SearchWriterLockImpl;
import org.sakaiproject.thread_local.cover.ThreadLocalManager;
import org.sakaiproject.tool.api.SessionManager;
import org.sakaiproject.user.api.User;
import org.sakaiproject.user.api.UserDirectoryService;

public class SearchIndexBuilderWorkerImpl implements Runnable, SearchIndexBuilderWorker
{

	private static Log log = LogFactory.getLog(SearchIndexBuilderWorkerImpl.class);
	/**
	 * The lock we use to ensure single search index writer
	 */
	public static final String LOCKKEY = "searchlockkey";

	protected static final Object GLOBAL_CONTEXT = null;

	private static final String NO_NODE = "none";

	private static final String NODE_LOCK = "nodelockkey";


	private int numThreads = 2;

	/**
	 * The maximum sleep time for the wait/notify semaphore
	 */
	public long sleepTime = 5L * 60000L;

	/**
	 * A load factor 1 is full load, 100 is normal The load factor controlls the
	 * backoff of the indexer threads. If the load Factor is high, the search
	 * threads back off more.
	 */
	private long loadFactor = 1000L;

	/**
	 * The currently running index Builder thread
	 */
	private Thread indexBuilderThread[] = null;

	/**
	 * sync object
	 */
	private Object threadStartLock = new Object();

	/**
	 * dependency: the search index builder that is accepting new items
	 */
	private SearchIndexBuilderImpl searchIndexBuilder = null;

	/**
	 * dependency: the current search service, used to get the location of the
	 * index
	 */
	private SearchService searchService = null;
	/**
	 * dependency 
	 */
	private ServerConfigurationService serverConfigurationService;

	private DataSource dataSource = null;

	/**
	 * Semaphore
	 */
	private Object sem = new Object();

	/**
	 * The number of items to process in a batch, default = 100
	 */

	private boolean enabled = false;

	private SessionManager sessionManager;

	private UserDirectoryService userDirectoryService;

	private EntityManager entityManager;

	private EventTrackingService eventTrackingService;

	private boolean runThreads = false;

	private ThreadLocal<String> nodeIDHolder = new ThreadLocal<String>();

	private SearchIndexBuilderWorkerDao searchIndexBuilderWorkerDao = null;

	private long lastLock = System.currentTimeMillis();

	private long lastEvent = System.currentTimeMillis();

	private long lastIndex;

	private long startDocIndex;

	private String nowIndexing;

	private String lastIndexing;

	private boolean soakTest = false;

	private boolean started = false;

	private boolean indexExists = false;
	

	private static HashMap<String, String> nodeIDList = new HashMap<String, String>();;

	private static String lockedTo = null;

	private static String SELECT_LOCK_SQL = "select id, nodename, "
			+ "lockkey, expires from searchwriterlock where lockkey = ?";

	private static String UPDATE_LOCK_SQL = "update searchwriterlock set "
			+ "nodename = ?, expires = ? where id = ? "
			+ "and nodename = ? and lockkey = ? ";

	private static String INSERT_LOCK_SQL = "insert into searchwriterlock "
			+ "( id,nodename,lockkey, expires ) values ( ?, ?, ?, ? )";

	private static String COUNT_WORK_SQL = " select count(*) "
			+ "from searchbuilderitem where searchstate = ? ";

	private static String CLEAR_LOCK_SQL = "update searchwriterlock "
			+ "set nodename = ?, expires = ? where nodename = ? and lockkey = ? ";

	private static String SELECT_NODE_LOCK_SQL = "select id, nodename, "
			+ "lockkey, expires from searchwriterlock where lockkey like '" + NODE_LOCK
			+ "%'";

	private static String UPDATE_NODE_LOCK_SQL = "update searchwriterlock set "
			+ "expires = ? where nodename = ? and lockkey = ? ";

	private static final String SELECT_EXPIRED_NODES_SQL = "select id from searchwriterlock "
			+ "where lockkey like '" + NODE_LOCK + "%' and expires < ? ";

	private static final String DELETE_LOCKNODE_SQL = "delete from searchwriterlock "
			+ "where id = ? ";

	public void init()
	{
		if (started && !runThreads)
		{
			log.warn("JVM Shutdown in progress, will not startup");
			return;
		}
		if (org.sakaiproject.component.cover.ComponentManager.hasBeenClosed())
		{
			log.warn("Component manager Shutdown in progress, will not startup");
			return;
		}
		started = true;
		runThreads = true;
		
		
		
		

		enabled = "true".equals(serverConfigurationService.getString("search.enable",
				"false"));

		enabled = enabled && serverConfigurationService.getBoolean("search.indexbuild",true);
		

		try
		{
			if (searchIndexBuilder == null)
			{
				log.error("Search Index Worker needs SearchIndexBuilder ");
			}
			if (searchService == null)
			{
				log.error("Search Index Worker needs SearchService ");
			}
			if (searchIndexBuilderWorkerDao == null)
			{
				log.error("Search Index Worker needs SearchIndexBuilderWorkerDao ");
			}
			if (eventTrackingService == null)
			{
				log.error("Search Index Worker needs EventTrackingService ");
			}
			if (entityManager == null)
			{
				log.error("Search Index Worker needs EntityManager ");
			}
			if (userDirectoryService == null)
			{
				log.error("Search Index Worker needs UserDirectortyService ");
			}
			if (sessionManager == null)
			{
				log.error("Search Index Worker needs SessionManager ");
			}
			log.debug("init start");
			indexBuilderThread = new Thread[numThreads];
			for (int i = 0; i < indexBuilderThread.length; i++)
			{
				indexBuilderThread[i] = new Thread(this);
				indexBuilderThread[i].setName("SearchBuilder_"+String.valueOf(i));
				indexBuilderThread[i].start();
			}

			
			eventTrackingService.addLocalObserver(new Observer(){

				public void update(Observable arg0, Object arg1)
				{
					lastEvent = System.currentTimeMillis();
				}
				
			});
			
			searchIndexBuilder.addIndexQueueListener(new IndexQueueListener() {

				public void added(String name)
				{
					checkRunning();
				}
				
			});

			/*
			 * Capture shutdown
			 */
			Runtime.getRuntime().addShutdownHook(new Thread()
			{
				/*
				 * (non-Javadoc)
				 * 
				 * @see java.lang.Thread#run()
				 */
				@Override
				public void run()
				{
					runThreads = false;
				}
			});

		}
		catch (Throwable t)
		{
			log.error("Failed to init ", t);
		}
	}


	/**
	 * Main run target of the worker thread {@inheritDoc}
	 */
	public void run()
	{
		if (!enabled) return;
		
		
	
		int threadno = -1;
		Thread tt = Thread.currentThread();
		for ( int i = 0; i < indexBuilderThread.length; i++ ) {
			if ( indexBuilderThread[i] == tt ) {
				threadno = i;
			}
		}
		
		String nodeID = getNodeID();

		org.sakaiproject.component.cover.ComponentManager.waitTillConfigured();

		try
		{

			while (runThreads)
			{
				log.debug("Run Processing Thread");
				org.sakaiproject.tool.api.Session s = null;
				if (s == null)
				{
					s = sessionManager.startSession();
					User u = userDirectoryService.getUser("admin");
					s.setUserId(u.getId());
				}

				while (runThreads)
				{
					sessionManager.setCurrentSession(s);
					
					//SAK-17117 before we do this clear threadLocal
					//get the security advisor stack otherwise later calls will fail
					Object obj = ThreadLocalManager.get("SakaiSecurity.advisor.stack");
					Object sess = ThreadLocalManager.get("org.sakaiproject.api.kernel.session.current");
					Object toolsess = ThreadLocalManager.get("org.sakaiproject.api.kernel.session.current.tool");
					
					ThreadLocalManager.clear();
					ThreadLocalManager.set("SakaiSecurity.advisor.stack", obj);
					ThreadLocalManager.set("org.sakaiproject.api.kernel.session.current", sess);
					ThreadLocalManager.set("org.sakaiproject.api.kernel.session.current.tool", toolsess);
					
					
					try
					{
						int totalDocs = searchIndexBuilder.getPendingDocuments();
						long lastEvent = getLastEventTime();
						long now = System.currentTimeMillis();
						long interval = now - lastEvent;
						boolean process = false;
						boolean createIndex = false;
						if (!indexExists)
						{
							if (!searchIndexBuilderWorkerDao.indexExists())
							{
								process = true;
								createIndex = true;
								log
										.debug("No cluster Index exists, creating for the first time");
							} else {
								indexExists = true;
							}
						}
						else
						{

							// if activity == totalDocs and interval > 10
							if ( totalDocs > 200 ) {
								loadFactor = 10L;
							} else {
								loadFactor = 1000L;
							}
							if ( totalDocs == 0 ) {
								process = false;
							} else if (totalDocs < 20 && interval > (20 * loadFactor))
							{
								process = true;
							}
							else if (totalDocs >= 20 && totalDocs < 50
									&& interval > (10 * loadFactor))
							{
								process = true;
							}
							else if (totalDocs >= 50 && totalDocs < 90
									&& interval > (5 * loadFactor))
							{
								process = true;
							}
							else if (totalDocs > ((90 * loadFactor) / 1000))
							{
								process = true;
							}
						}

						// should this node consider taking the lock ?
						long lastLockInterval = (System.currentTimeMillis() - lastLock);
						long lastLockMetric = lastLockInterval * totalDocs;

						// if we have 1000 docs, then indexing should happen
						// after 10 seconds break
						// 1000*10000 10000000
						// 500 docs/ 20 seconds
						//

						// make certain that we are alive
						log.debug("Activity "+(lastLockMetric > (10000L * loadFactor))+" "+(lastLockInterval > (60L * loadFactor))+" "+createIndex);

						if (lastLockMetric > (10000L * loadFactor)
								|| lastLockInterval > (60L * loadFactor) || createIndex)
						{
							log.debug("===" + process + "=============PROCESSING ");
							if (process && getLockTransaction(2L * 60L * 1000L,createIndex))
							{

								log.debug("===" + nodeID + "=============PROCESSING ");
								if (lockedTo != null && lockedTo.equals(nodeID))
								{
									log
											.error("+++++++++++++++Local Lock Collision+++++++++++++");
								}
								lockedTo = nodeID;

								lastLock = System.currentTimeMillis();

								if (createIndex)
								{
									log
											.info("=======================Search Index being created for the first time");
									searchIndexBuilderWorkerDao
											.createIndexTransaction(this);
									indexExists = true;
									log
											.info("=======================Done creating Search Index for the first time");

								}
								else
								{
									int batchSize = 100;
									if ( totalDocs > 500 ) {
										batchSize = 200;
									} else if ( totalDocs > 1000 ) {
										batchSize = 500;
									} else if ( totalDocs > 10000 ) {
										batchSize = 1000;
									}
									searchIndexBuilderWorkerDao
											.processToDoListTransaction(this, batchSize);

								}

								lastLock = System.currentTimeMillis();

								if (lockedTo.equals(nodeID))
								{
									lockedTo = null;
								}
								else
								{
									log
											.error("+++++++++++++++++++++++++++Lost Local Lock+++++++++++");
								}
								log.debug("===" + nodeID + "=============COMPLETED ");

							}
							else
							{
								break;
							}
						}
						else
						{
							// make certain the node updates hearbeat
							updateNodeLock(2L * 60L * 1000L);
							log.debug("Not taking Lock, too much activity");
							break;
						}
					}
					finally
					{
						clearLockTransaction();
					}
				}
				// this is here force cluster members
				// this will not reload the index on this node as
				if (indexExists)
				{
					try
					{
						searchService.reload();
					}
					catch (Exception ex)
					{
						log
								.info("No Search Segment exists at present, this is Ok on first start :"
										+ ex.getMessage());
					}
				}
				if (!runThreads)
				{
					break;
				}
				try
				{
					log.debug("Sleeping Processing Thread");
					synchronized (sem)
					{
						log.debug("++++++WAITING " + nodeID);
						sem.wait(sleepTime);

						log.debug("+++++ALIVE " + nodeID);
					}
					log.debug("Wakey Wakey Processing Thread");

					if (org.sakaiproject.component.cover.ComponentManager.hasBeenClosed())
					{
						runThreads = false;
						break;
					}
					if (soakTest && (searchService.getPendingDocs() == 0))
					{
						log
								.error("SOAK TEST---SOAK TEST---SOAK TEST. Index Rebuild Started");
						searchService.rebuildInstance();
					}
				}
				catch (InterruptedException e)
				{
					log.debug(" Exit From sleep " + e.getMessage());
					break;
				}
			}
		}
		catch (Throwable t)
		{
			
			log.warn("Failed in IndexBuilder when indexing document: "+getNowIndexing(), t);
		}
		finally
		{

			log.debug("IndexBuilder run exit " + tt.getName());
			if ( threadno != -1 ) {
				indexBuilderThread[threadno] = null;
			}
		}
	}

	private String getNodeID()
	{
		String nodeID = (String) nodeIDHolder.get();
		if (nodeID == null)
		{
			nodeID = UUID.randomUUID().toString();
			nodeIDHolder.set(nodeID);
			if (nodeIDList.get(nodeID) == null)
			{
				nodeIDList.put(nodeID, nodeID);
			}
			else
			{
				log.error("============NODE ID " + nodeID
						+ " has already been issued, there must be a clash");
			}
		}
		return nodeID;
	}

	/*
	 * (non-Javadoc)
	 * 
	 * @see org.sakaiproject.search.component.service.impl.SearchIndexBuilderWorkerAPI#updateNodeLock(java.sql.Connection)
	 */
	public void updateNodeLock(long lifeLeft) throws SQLException
	{

		Connection connection = null;
		String nodeID = getNodeID();

		PreparedStatement updateNodeLock = null;
		PreparedStatement deleteExpiredNodeLock = null;
		PreparedStatement selectExpiredNodeLock = null;
		PreparedStatement insertLock = null;
		ResultSet resultSet = null;
		String threadID = Thread.currentThread().getName();
		boolean savedautocommit = false;
		Timestamp now = new Timestamp(System.currentTimeMillis());
		// a node can expire, after 2 minutes, to indicate to an admin that it
		// is dead
		// the admin can then force the
		Timestamp nodeExpired = new Timestamp(now.getTime() + lifeLeft);
		try
		{
			connection = dataSource.getConnection();
			connection.setAutoCommit(false);

			updateNodeLock = connection.prepareStatement(UPDATE_NODE_LOCK_SQL);
			deleteExpiredNodeLock = connection.prepareStatement(DELETE_LOCKNODE_SQL);
			selectExpiredNodeLock = connection.prepareStatement(SELECT_EXPIRED_NODES_SQL);
			insertLock = connection.prepareStatement(INSERT_LOCK_SQL);
			int retries = 5;
			boolean updated = false;
			while (!updated && retries > 0)
			{
				try
				{

					try
					{
						insertLock.clearParameters();
						insertLock.setString(1, "Node:" + nodeID); // id
						insertLock.setString(2, nodeID); // nodename
						insertLock.setString(3, NODE_LOCK + nodeID); // lockkey
						insertLock.setTimestamp(4, nodeExpired); // expires
						log.debug(threadID + " Doing " + INSERT_LOCK_SQL + ":{" + "Node:"
								+ nodeID + "}{" + nodeID + "}{" + NODE_LOCK + nodeID
								+ "}{" + nodeExpired + "}");
						insertLock.executeUpdate();
					}
					catch (SQLException ex)
					{
						updateNodeLock.clearParameters();
						updateNodeLock.setTimestamp(1, nodeExpired); // expires
						updateNodeLock.setString(2, nodeID); // nodename
						updateNodeLock.setString(3, NODE_LOCK + nodeID); // lockkey
						log.debug(threadID + " Doing " + UPDATE_NODE_LOCK_SQL + ":{"
								+ nodeExpired + "}{" + nodeID + "}{" + NODE_LOCK + nodeID
								+ "}");
						if (updateNodeLock.executeUpdate() != 1)
						{
							log.warn("Failed to update node heartbeat " + nodeID);
						}
					}
					log.debug(threadID + " Doing Commit ");
					connection.commit();
					updated = true;
				}
				catch (SQLException e)
				{
					log.warn("Retrying ", e);
					try
					{
						connection.rollback();
					}
					catch (Exception ex)
					{
						log.debug(ex);
					}
					retries--;
					try
					{
						Thread.sleep(100);
					}
					catch (InterruptedException ie)
					{
						log.debug(ie);
					}
				}
			}
			if (!updated)
			{
				log.error("Failed to update node lock, will try next time ");
			}
			else
			{
				log.debug("Updated Node Lock on " + nodeID + " to Expire at"
						+ nodeExpired);
			}

			retries = 5;
			updated = false;
			while (!updated && retries > 0)
			{
				try
				{
					selectExpiredNodeLock.clearParameters();
					selectExpiredNodeLock.setTimestamp(1, now);
					log.debug(threadID + " Doing " + SELECT_EXPIRED_NODES_SQL + ":{"
							+ now + "}");

					resultSet = selectExpiredNodeLock.executeQuery();
					while (resultSet.next())
					{
						String id = resultSet.getString(1);
						deleteExpiredNodeLock.clearParameters();
						deleteExpiredNodeLock.setString(1, id);
						deleteExpiredNodeLock.execute();
						connection.commit();
					}
					log.debug(threadID + " Doing Commit");
					connection.commit();
					resultSet.close();
					updated = true;
				}
				catch (SQLException e)
				{

					log.info("Retrying Delete Due to  " + e.getMessage());
					log.debug("Detailed Traceback  ", e);
					try
					{
						resultSet.close();
					}
					catch (Exception ex)
					{
						log.debug(ex);
					}
					try
					{
						connection.rollback();
					}
					catch (Exception ex)
					{
						log.debug(ex);
					}
					retries--;
					try
					{
						Thread.sleep(100);
					}
					catch (InterruptedException ie)
					{
						log.debug(ie);
					}
				}
			}
			if (!updated)
			{
				log.warn("Failed to clear old nodes, will try next time ");
			}

		}
		catch (Exception ex)
		{
			log.error("Failed to register node ", ex);
			if (connection != null) 
			{
				connection.rollback();
			}
		}
		finally
		{
			if (resultSet != null)
			{
				try
				{
					resultSet.close();
				}
				catch (SQLException e)
				{
					log.debug(e);
				}
			}
			if (insertLock != null)
			{
				try
				{
					insertLock.close();
				}
				catch (SQLException e)
				{
					log.debug(e);
				}
			}
			if (updateNodeLock != null)
			{
				try
				{
					updateNodeLock.close();
				}
				catch (SQLException e)
				{
					log.debug(e);
				}
			}
			if (selectExpiredNodeLock != null)
			{
				try
				{
					selectExpiredNodeLock.close();
				}
				catch (SQLException e)
				{
					log.debug(e);
				}
			}
			if (deleteExpiredNodeLock != null)
			{
				try
				{
					deleteExpiredNodeLock.close();
				}
				catch (SQLException e)
				{
					log.debug(e);
				}
			}
			if (connection != null)
			{
				try
				{
					connection.setAutoCommit(savedautocommit);
					connection.close();
				}
				catch (SQLException e)
				{
					log.debug(e);
				}
				connection = null;
			}

		}

	}

	public boolean getLockTransaction(long nodeLifetime)
	{
		return getLockTransaction(nodeLifetime, false);
	}

	/**
	 * @return
	 * @throws HibernateException
	 */
	public boolean getLockTransaction(long nodeLifetime, boolean forceLock)
	{
		if (searchIndexBuilderWorkerDao.isLockRequired())
		{
			return getHardLock(nodeLifetime, forceLock);
		}
		else
		{
			try
			{
				updateNodeLock(nodeLifetime);
			}
			catch (SQLException e)
			{
				log.warn("Failed to update node lock " + e.getClass().getName() + " :"
						+ e.getMessage());
			}
			return true;
		}
	}

	public boolean getHardLock(long nodeLifetime, boolean forceLock)
	{
		String nodeID = getNodeID();
		Connection connection = null;
		boolean locked = false;
		boolean autoCommit = false;
		PreparedStatement selectLock = null;
		PreparedStatement updateLock = null;
		PreparedStatement insertLock = null;
		PreparedStatement countWork = null;

		ResultSet resultSet = null;
		Timestamp now = new Timestamp(System.currentTimeMillis());
		Timestamp expiryDate = new Timestamp(now.getTime() + (10L * 60L * 1000L));

		try
		{

			// I need to go direct to JDBC since its just too awful to
			// try and do this in Hibernate.

			updateNodeLock(nodeLifetime);

			connection = dataSource.getConnection();
			autoCommit = connection.getAutoCommit();
			if (autoCommit)
			{
				connection.setAutoCommit(false);
			}

			selectLock = connection.prepareStatement(SELECT_LOCK_SQL);
			updateLock = connection.prepareStatement(UPDATE_LOCK_SQL);
			insertLock = connection.prepareStatement(INSERT_LOCK_SQL);
			countWork = connection.prepareStatement(COUNT_WORK_SQL);

			SearchWriterLock swl = null;
			selectLock.clearParameters();
			selectLock.setString(1, LOCKKEY);
			resultSet = selectLock.executeQuery();
			if (resultSet.next())
			{
				swl = new SearchWriterLockImpl();
				swl.setId(resultSet.getString(1));
				swl.setNodename(resultSet.getString(2));
				swl.setLockkey(resultSet.getString(3));
				swl.setExpires(resultSet.getTimestamp(4));
				log.debug("GOT Lock Record " + swl.getId() + "::" + swl.getNodename()
						+ "::" + swl.getExpires());

			}

			resultSet.close();
			resultSet = null;

			boolean takelock = false;
			if (swl == null)
			{
				log.debug("_-------------NO Lock Record");
				takelock = true;
			}
			else if ("none".equals(swl.getNodename()))
			{
				takelock = true;
				log.debug(nodeID + "_-------------no lock");
			}
			else if (nodeID.equals(swl.getNodename()))
			{
				takelock = true;
				log.debug(nodeID + "_------------matched threadid ");
			}
			else if (swl.getExpires() == null || swl.getExpires().before(now))
			{
				takelock = true;
				log.debug(nodeID + "_------------thread dead ");
			}

			if (takelock)
			{
				// any work ?
				int nitems = 0;
				if (!forceLock)
				{
					countWork.clearParameters();
					countWork.setInt(1, SearchBuilderItem.STATE_PENDING.intValue());
					resultSet = countWork.executeQuery();
					if (resultSet.next())
					{
						nitems = resultSet.getInt(1);
					}
					resultSet.close();
					resultSet = null;
				}
				if (nitems > 0 || forceLock)
				{
					try
					{
						if (swl == null)
						{
							insertLock.clearParameters();
							insertLock.setString(1, nodeID);
							insertLock.setString(2, nodeID);
							insertLock.setString(3, LOCKKEY);
							insertLock.setTimestamp(4, expiryDate);

							if (insertLock.executeUpdate() == 1)
							{
								log.debug("INSERT Lock Record " + nodeID + "::" + nodeID
										+ "::" + expiryDate);

								locked = true;
							}

						}
						else
						{
							updateLock.clearParameters();
							updateLock.setString(1, nodeID);
							updateLock.setTimestamp(2, expiryDate);
							updateLock.setString(3, swl.getId());
							updateLock.setString(4, swl.getNodename());
							updateLock.setString(5, swl.getLockkey());
							if (updateLock.executeUpdate() == 1)
							{
								log.debug("UPDATED Lock Record " + swl.getId() + "::"
										+ nodeID + "::" + expiryDate);
								locked = true;
							}

						}
					}
					catch (SQLException sqlex)
					{
						locked = false;
						log.debug("Failed to get lock, but this is Ok ", sqlex);
					}

				}

			}
			connection.commit();

		}
		catch (Exception ex)
		{
			if (connection != null)
			{
				try
				{
					connection.rollback();
				}
				catch (SQLException e)
				{
					log.debug(e);
				}
			}
			log.error("Failed to get lock " + ex.getMessage());
			locked = false;
		}
		finally
		{
			if (resultSet != null)
			{
				try
				{
					resultSet.close();
				}
				catch (SQLException e)
				{
					log.debug(e);
				}
			}
			if (selectLock != null)
			{
				try
				{
					selectLock.close();
				}
				catch (SQLException e)
				{
					log.debug(e);
				}
			}
			if (updateLock != null)
			{
				try
				{
					updateLock.close();
				}
				catch (SQLException e)
				{
					log.debug(e);
				}
			}
			if (insertLock != null)
			{
				try
				{
					insertLock.close();
				}
				catch (SQLException e)
				{
					log.debug(e);
				}
			}
			if (countWork != null)
			{
				try
				{
					countWork.close();
				}
				catch (SQLException e)
				{
					log.debug(e);
				}
			}

			if (connection != null)
			{
				try
				{
					connection.setAutoCommit(autoCommit);
				}
				catch (SQLException e)
				{
				}
				try
				{
					connection.close();
					log.debug("Connection Closed ");
				}
				catch (SQLException e)
				{
					log.error("Error Closing Connection ", e);
				}
				connection = null;
			}
		}
		return locked;

	}

	/**
	 * Count the number of pending documents waiting to be indexed on this
	 * cluster node. All nodes will potentially perform the index in a cluster,
	 * however only one must be doing the index, hence this method attampts to
	 * grab a lock on the writer. If sucessfull it then gets the real number of
	 * pending documents. There is a timeout, such that if the witer has not
	 * been seen for 10 minutes, it is assumed that something has gon wrong with
	 * it, and a new writer is elected on a first grab basis. Every time the
	 * elected writer comes back, it updates its record to say its still active.
	 * We could do some round robin timeout, or allow deployers to select a pool
	 * of index writers in Sakai properties. {@inheritDoc}
	 */

	private void clearLockTransaction()
	{
		if (searchIndexBuilderWorkerDao.isLockRequired())
		{
			clearHardLock();
		}
	}

	public void clearHardLock()
	{
		String nodeID = getNodeID();

		Connection connection = null;
		PreparedStatement clearLock = null;
		try
		{
			connection = dataSource.getConnection();

			clearLock = connection.prepareStatement(CLEAR_LOCK_SQL);
			clearLock.clearParameters();
			clearLock.setString(1, NO_NODE);
			clearLock.setTimestamp(2, new Timestamp(System.currentTimeMillis()));
			clearLock.setString(3, nodeID);
			clearLock.setString(4, LOCKKEY);
			if (clearLock.executeUpdate() == 1)
			{
				log.debug("UNLOCK - OK::" + nodeID + "::now");

			}
			else
			{
				log.debug("UNLOCK - no-lock::" + nodeID + "::now");
			}
			connection.commit();

		}
		catch (Exception ex)
		{
			try
			{
				if (connection != null) 
				{
					connection.rollback();
				}
			}
			catch (SQLException e)
			{
			}
			log.error("Failed to clear lock" + ex.getMessage());
		}
		finally
		{
			if (clearLock != null)
			{
				try
				{
					clearLock.close();
				}
				catch (SQLException e)
				{
					log.error("Error Closing Prepared Statement ", e);
				}
			}
			if (connection != null)
			{
				try
				{
					connection.close();
					log.debug("Connection Closed");
				}
				catch (SQLException e)
				{
					log.error("Error Closing Connection", e);
				}
			}
		}

	}

	public boolean isRunning()
	{
		if (org.sakaiproject.component.cover.ComponentManager.hasBeenClosed())
		{
			runThreads = false;
		}
		return runThreads;
	}

	/*
	 * (non-Javadoc)
	 * 
	 * @see org.sakaiproject.search.component.service.impl.SearchIndexBuilderWorkerAPI#checkRunning()
	 */
	public void checkRunning()
	{
		if (!enabled) return;
		runThreads = true;
		synchronized (threadStartLock)
		{
			for (int i = 0; i < indexBuilderThread.length; i++)
			{
				if (indexBuilderThread[i] == null)
				{
					indexBuilderThread[i] = new Thread(this);
					indexBuilderThread[i].setName(String.valueOf(i) + "::"
							+ this.getClass().getName());
					indexBuilderThread[i].start();
				}
			}
		}
		synchronized (sem)
		{
			log.debug("_________NOTIFY");
			sem.notify();
			log.debug("_________NOTIFY COMPLETE");
		}

	}

	public void destroy()
	{
		if (!enabled) return;

		log.debug("Destroy SearchIndexBuilderWorker ");
		runThreads = false;

		synchronized (sem)
		{
			sem.notifyAll();
		}
	}

	/**
	 * @return Returns the sleepTime.
	 */
	public long getSleepTime()
	{
		return sleepTime;
	}

	/**
	 * @param sleepTime
	 *        The sleepTime to set.
	 */
	public void setSleepTime(long sleepTime)
	{
		this.sleepTime = sleepTime;
	}

	/**
	 * @return Returns the dataSource.
	 */
	public DataSource getDataSource()
	{
		return dataSource;
	}

	/**
	 * @param dataSource
	 *        The dataSource to set.
	 */
	public void setDataSource(DataSource dataSource)
	{
		this.dataSource = dataSource;
	}

	/**
	 * @return Returns the searchIndexBuilderWorkerDao.
	 */
	public SearchIndexBuilderWorkerDao getSearchIndexBuilderWorkerDao()
	{
		return searchIndexBuilderWorkerDao;
	}

	/**
	 * @param searchIndexBuilderWorkerDao
	 *        The searchIndexBuilderWorkerDao to set.
	 */
	public void setSearchIndexBuilderWorkerDao(
			SearchIndexBuilderWorkerDao searchIndexBuilderWorkerDao)
	{
		this.searchIndexBuilderWorkerDao = searchIndexBuilderWorkerDao;
	}

	/*
	 * (non-Javadoc)
	 * 
	 * @see org.sakaiproject.search.component.service.impl.SearchIndexBuilderWorkerAPI#getCurrentLock()
	 */
	public SearchWriterLock getCurrentLock()
	{
		getNodeID();
		Connection connection = null;
		PreparedStatement selectLock = null;
		ResultSet resultSet = null;

		try
		{

			// I need to go direct to JDBC since its just too awful to
			// try and do this in Hibernate.

			connection = dataSource.getConnection();

			selectLock = connection.prepareStatement(SELECT_LOCK_SQL);

			SearchWriterLock swl = null;
			selectLock.clearParameters();
			selectLock.setString(1, LOCKKEY);
			resultSet = selectLock.executeQuery();
			if (resultSet.next())
			{
				swl = new SearchWriterLockImpl();
				swl.setId(resultSet.getString(1));
				swl.setNodename(resultSet.getString(2));
				swl.setLockkey(resultSet.getString(3));
				swl.setExpires(resultSet.getTimestamp(4));
				log.debug("GOT Lock Record " + swl.getId() + "::" + swl.getNodename()
						+ "::" + swl.getExpires());

			}

			resultSet.close();
			resultSet = null;
			if (swl == null)
			{
				swl = new SearchWriterLockImpl();
				swl.setNodename(NO_NODE);
				swl.setLockkey(LOCKKEY);
				swl.setExpires(new Timestamp(0));

			}
			return swl;

		}
		catch (Exception ex)
		{
			log.error("Failed to get lock " + ex.getMessage());
			SearchWriterLock swl = new SearchWriterLockImpl();
			swl.setNodename(NO_NODE);
			swl.setLockkey(LOCKKEY);
			swl.setExpires(new Timestamp(0));

			return swl;
		}
		finally
		{
			if (resultSet != null)
			{
				try
				{
					resultSet.close();
				}
				catch (SQLException e)
				{
				}
			}
			if (selectLock != null)
			{
				try
				{
					selectLock.close();
				}
				catch (SQLException e)
				{
				}
			}

			if (connection != null)
			{
				try
				{
					connection.close();
					log.debug("Connection Closed ");
				}
				catch (SQLException e)
				{
					log.error("Error Closing Connection ", e);
				}
				connection = null;
			}
		}

	}

	/*
	 * (non-Javadoc)
	 * 
	 * @see org.sakaiproject.search.component.service.impl.SearchIndexBuilderWorkerAPI#getNodeStatus()
	 */
	public List<SearchWriterLock> getNodeStatus()
	{
		getNodeID();
		Connection connection = null;
		PreparedStatement selectLock = null;
		ResultSet resultSet = null;
		ArrayList<SearchWriterLock>locks = new ArrayList<SearchWriterLock>();

		try
		{

			// I need to go direct to JDBC since its just too awful to
			// try and do this in Hibernate.

			connection = dataSource.getConnection();

			selectLock = connection.prepareStatement(SELECT_NODE_LOCK_SQL);

			selectLock.clearParameters();
			resultSet = selectLock.executeQuery();
			while (resultSet.next())
			{
				SearchWriterLock swl = new SearchWriterLockImpl();
				swl.setId(resultSet.getString(1));
				swl.setNodename(resultSet.getString(2));
				swl.setLockkey(resultSet.getString(3));
				swl.setExpires(resultSet.getTimestamp(4));
				log.debug("GOT Lock Record " + swl.getId() + "::" + swl.getNodename()
						+ "::" + swl.getExpires());
				locks.add(swl);
			}

			resultSet.close();
			resultSet = null;
			return locks;

		}
		catch (Exception ex)
		{
			log.error("Failed to load nodes ", ex);
			return locks;
		}
		finally
		{
			if (resultSet != null)
			{
				try
				{
					resultSet.close();
				}
				catch (SQLException e)
				{
				}
			}
			if (selectLock != null)
			{
				try
				{
					selectLock.close();
				}
				catch (SQLException e)
				{
				}
			}

			if (connection != null)
			{
				try
				{
					connection.close();
					log.debug("Connection Closed ");
				}
				catch (SQLException e)
				{
					log.error("Error Closing Connection ", e);
				}
				connection = null;
			}
		}

	}

	public boolean removeWorkerLock()
	{
		Connection connection = null;
		PreparedStatement selectLock = null;
		PreparedStatement selectNodeLock = null;
		PreparedStatement clearLock = null;
		ResultSet resultSet = null;
		

		try
		{

			// I need to go direct to JDBC since its just too awful to
			// try and do this in Hibernate.

			connection = dataSource.getConnection();

			selectNodeLock = connection.prepareStatement(SELECT_NODE_LOCK_SQL);
			selectLock = connection.prepareStatement(SELECT_LOCK_SQL);
			clearLock = connection.prepareStatement(CLEAR_LOCK_SQL);

			SearchWriterLock swl = null;
			selectLock.clearParameters();
			selectLock.setString(1, LOCKKEY);
			resultSet = selectLock.executeQuery();
			if (resultSet.next())
			{
				swl = new SearchWriterLockImpl();
				swl.setId(resultSet.getString(1));
				swl.setNodename(resultSet.getString(2));
				swl.setLockkey(resultSet.getString(3));
				swl.setExpires(resultSet.getTimestamp(4));
			}
			else
			{
				connection.rollback();
				return true;
			}

			resultSet.close();
			resultSet = null;

			selectNodeLock.clearParameters();
			resultSet = selectLock.executeQuery();

			while (resultSet.next())
			{
				SearchWriterLock node = new SearchWriterLockImpl();
				node.setId(resultSet.getString(1));
				node.setNodename(resultSet.getString(2));
				node.setLockkey(resultSet.getString(3));
				node.setExpires(resultSet.getTimestamp(4));
				if (swl.getNodename().equals(node.getNodename()))
				{
					log.info("Cant remove Lock to node " + node.getNodename()
							+ " node exists ");
					connection.rollback();
					return false;
				}
			}

			resultSet.close();
			resultSet = null;

			clearLock.clearParameters();
			clearLock.setString(1, NO_NODE);
			clearLock.setTimestamp(2, new Timestamp(System.currentTimeMillis()));
			clearLock.setString(3, swl.getNodename());
			clearLock.setString(4, LOCKKEY);
			if (clearLock.executeUpdate() == 1)
			{
				log.warn("NODE UNLOCKED BY USER " + swl.getNodename());
				connection.commit();
			}
			else
			{
				log.info("NODE NOT UNLOCKED BY USER " + swl.getNodename());
				connection.commit();
				return false;
			}

			return true;

		}
		catch (Exception ex)
		{
			log.error("Failed to unlock ", ex);
			return false;
		}
		finally
		{
			if (resultSet != null)
			{
				try
				{
					resultSet.close();
				}
				catch (SQLException e)
				{
				}
			}
			if (selectLock != null)
			{
				try
				{
					selectLock.close();
				}
				catch (SQLException e)
				{
				}
			}
			if (selectNodeLock != null)
			{
				try
				{
					selectNodeLock.close();
				}
				catch (SQLException e)
				{
				}
			}
			if (clearLock != null)
			{
				try
				{
					clearLock.close();
				}
				catch (SQLException e)
				{
				}
			}

			if (connection != null)
			{
				try
				{
					connection.close();
					log.debug("Connection Closed ");
				}
				catch (SQLException e)
				{
					log.error("Error Closing Connection ", e);
				}
				connection = null;
			}
		}
	}


	public long getLastEventTime()
	{
		return lastEvent;
	}



	public void setLastIndex(long l)
	{
		this.lastIndex = l;

	}

	public void setStartDocIndex(long startDocIndex)
	{
		this.startDocIndex = startDocIndex;
	}

	public void setNowIndexing(String reference)
	{
		this.lastIndexing = this.nowIndexing;
		this.nowIndexing = reference;
	}

	/**
	 * @return Returns the lastIndex.
	 */
	public long getLastIndex()
	{
		return lastIndex;
	}

	/**
	 * @return Returns the nowIndexing.
	 */
	public String getNowIndexing()
	{
		return nowIndexing;
	}

	/**
	 * @return Returns the startDocIndex.
	 */
	public long getStartDocIndex()
	{
		return startDocIndex;
	}

	public String getLastDocument()
	{
		return lastIndexing;
	}

	public String getLastElapsed()
	{
		long l = lastIndex;
		long h = l / 3600000L;
		l = l - (3600000L * h);
		long m = l / 600000L;;
		l = l - (60000L * m);
		long s = l / 1000;
		l = l - (1000L * s);
		return "" + h + "h" + m + "m" + s + "." + l + "s";
	}

	public String getCurrentDocument()
	{
		return nowIndexing;
	}

	public String getCurrentElapsed()
	{
		long l = System.currentTimeMillis() - startDocIndex;
		long h = l / 3600000L;
		l = l - (3600000L * h);
		long m = l / 60000L;
		l = l - (60000L * m);
		long s = l / 1000L;
		l = l - (1000L * s);
		return "" + h + "h" + m + "m" + s + "." + l + "s";
	}

	/**
	 * @return the loadFactor
	 */
	public long getLoadFactor()
	{
		return loadFactor;
	}

	/**
	 * @param loadFactor
	 *        the loadFactor to set
	 */
	public void setLoadFactor(long loadFactor)
	{
		this.loadFactor = loadFactor;
	}

	/**
	 * Is the lock on this node, but not this thread lockedTo == null, localloc ==
	 * false lockedTo == this node, locallock = false; lockedTo != this node,
	 * localLock = true
	 */
	public boolean isLocalLock()
	{
		if (lockedTo == null)
		{
			return false;
		}
		else if (getNodeID().equals(lockedTo))
		{
			return false;
		}
		return true;
	}

	/**
	 * @return the soakTest
	 */
	public boolean getSoakTest()
	{
		return soakTest;
	}

	/**
	 * Puts the index builder into a Soak test, when there are no pending items,
	 * it starts building again.
	 * 
	 * @param soakTest
	 *        the soakTest to set
	 */
	public void setSoakTest(boolean soakTest)
	{

		this.soakTest = soakTest;
		if (soakTest)
		{
			log.warn("SOAK TEST ACTIVE ======================DONT USE FOR PRODUCTION ");
		}
	}

	/**
	 * @return the entityManager
	 */
	public EntityManager getEntityManager()
	{
		return entityManager;
	}

	/**
	 * @param entityManager the entityManager to set
	 */
	public void setEntityManager(EntityManager entityManager)
	{
		this.entityManager = entityManager;
	}

	/**
	 * @return the eventTrackingService
	 */
	public EventTrackingService getEventTrackingService()
	{
		return eventTrackingService;
	}

	/**
	 * @param eventTrackingService the eventTrackingService to set
	 */
	public void setEventTrackingService(EventTrackingService eventTrackingService)
	{
		this.eventTrackingService = eventTrackingService;
	}

	/**
	 * @return the searchIndexBuilder
	 */
	public SearchIndexBuilderImpl getSearchIndexBuilder()
	{
		return searchIndexBuilder;
	}

	/**
	 * @param searchIndexBuilder the searchIndexBuilder to set
	 */
	public void setSearchIndexBuilder(SearchIndexBuilderImpl searchIndexBuilder)
	{
		this.searchIndexBuilder = searchIndexBuilder;
	}

	/**
	 * @return the searchService
	 */
	public SearchService getSearchService()
	{
		return searchService;
	}

	/**
	 * @param searchService the searchService to set
	 */
	public void setSearchService(SearchService searchService)
	{
		this.searchService = searchService;
	}

	/**
	 * @return the serverConfigurationService
	 */
	public ServerConfigurationService getServerConfigurationService()
	{
		return serverConfigurationService;
	}

	/**
	 * @param serverConfigurationService the serverConfigurationService to set
	 */
	public void setServerConfigurationService(
			ServerConfigurationService serverConfigurationService)
	{
		this.serverConfigurationService = serverConfigurationService;
	}

	/**
	 * @return the sessionManager
	 */
	public SessionManager getSessionManager()
	{
		return sessionManager;
	}

	/**
	 * @param sessionManager the sessionManager to set
	 */
	public void setSessionManager(SessionManager sessionManager)
	{
		this.sessionManager = sessionManager;
	}

	/**
	 * @return the userDirectoryService
	 */
	public UserDirectoryService getUserDirectoryService()
	{
		return userDirectoryService;
	}

	/**
	 * @param userDirectoryService the userDirectoryService to set
	 */
	public void setUserDirectoryService(UserDirectoryService userDirectoryService)
	{
		this.userDirectoryService = userDirectoryService;
	}


	public void setNumThreads(int numThreads) {
		this.numThreads = numThreads;
	}

	
}