StatsUpdateManagerImpl.java

/**
 * $URL: https://source.sakaiproject.org/svn/sitestats/trunk/sitestats-impl/src/java/org/sakaiproject/sitestats/impl/StatsUpdateManagerImpl.java $
 * $Id: StatsUpdateManagerImpl.java 116373 2012-11-14 18:40:48Z matthew.buckett@it.ox.ac.uk $
 *
 * Copyright (c) 2006-2009 The Sakai Foundation
 *
 * Licensed under the Educational Community License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *             http://www.opensource.org/licenses/ECL-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package org.sakaiproject.sitestats.impl;

import java.sql.SQLException;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Observable;
import java.util.Observer;
import java.util.Set;
import java.util.concurrent.locks.ReentrantLock;

import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.hibernate.Criteria;
import org.hibernate.HibernateException;
import org.hibernate.Query;
import org.hibernate.Session;
import org.hibernate.Transaction;
import org.hibernate.criterion.Expression;
import org.hibernate.criterion.Order;
import org.sakaiproject.alias.api.AliasService;
import org.sakaiproject.component.cover.ComponentManager;
import org.sakaiproject.entity.api.EntityManager;
import org.sakaiproject.event.api.Event;
import org.sakaiproject.event.api.EventTrackingService;
import org.sakaiproject.event.api.UsageSession;
import org.sakaiproject.event.api.UsageSessionService;
import org.sakaiproject.exception.IdUnusedException;
import org.sakaiproject.site.api.Site;
import org.sakaiproject.site.api.SiteService;
import org.sakaiproject.sitestats.api.EventStat;
import org.sakaiproject.sitestats.api.JobRun;
import org.sakaiproject.sitestats.api.ResourceStat;
import org.sakaiproject.sitestats.api.ServerStat;
import org.sakaiproject.sitestats.api.SiteActivity;
import org.sakaiproject.sitestats.api.SitePresence;
import org.sakaiproject.sitestats.api.SiteVisits;
import org.sakaiproject.sitestats.api.StatsManager;
import org.sakaiproject.sitestats.api.StatsUpdateManager;
import org.sakaiproject.sitestats.api.UserStat;
import org.sakaiproject.sitestats.api.Util;
import org.sakaiproject.sitestats.api.event.EventRegistryService;
import org.sakaiproject.sitestats.api.event.ToolInfo;
import org.sakaiproject.sitestats.api.parser.EventParserTip;
import org.springframework.orm.hibernate3.HibernateCallback;
import org.springframework.orm.hibernate3.support.HibernateDaoSupport;


/**
 * @author <a href="mailto:nuno@ufp.pt">Nuno Fernandes</a>
 */
public class StatsUpdateManagerImpl extends HibernateDaoSupport implements Runnable, StatsUpdateManager, Observer {
	private Log								LOG									= LogFactory.getLog(StatsUpdateManagerImpl.class);
	private final static String				PRESENCE_SUFFIX						= "-presence";
	private final static int				PRESENCE_SUFFIX_LENGTH				= PRESENCE_SUFFIX.length();

	/** Spring bean members */
	private boolean							collectThreadEnabled				= true;
	public long								collectThreadUpdateInterval			= 4000L;
	private boolean							collectAdminEvents					= false;
	private boolean							collectEventsForSiteWithToolOnly	= true;

	/** Sakai services */
	private StatsManager					M_sm;
	private EventRegistryService			M_ers;
	private SiteService						M_ss;
	private AliasService					M_as;
	private EntityManager					M_em;
	private UsageSessionService				M_uss;
	private EventTrackingService			M_ets;

	/** Collect Thread and Semaphore */
	private Thread							collectThread;
	private List<Event>						collectThreadQueue					= new ArrayList<Event>();
	private Object							collectThreadSemaphore				= new Object();
	private boolean							collectThreadRunning				= false;

	/** Collect thread queue maps */
	private Map<String, EventStat>			eventStatMap						= Collections.synchronizedMap(new HashMap<String, EventStat>());
	private Map<String, ResourceStat>		resourceStatMap						= Collections.synchronizedMap(new HashMap<String, ResourceStat>());
	private Map<String, SiteActivity>		activityMap							= Collections.synchronizedMap(new HashMap<String, SiteActivity>());
	private Map<String, SiteVisits>			visitsMap							= Collections.synchronizedMap(new HashMap<String, SiteVisits>());
	private Map<String, SitePresenceConsolidation>	presencesMap				= Collections.synchronizedMap(new HashMap<String, SitePresenceConsolidation>());
	private Map<UniqueVisitsKey, Integer>	uniqueVisitsMap						= Collections.synchronizedMap(new HashMap<UniqueVisitsKey, Integer>());
	private Map<String, ServerStat>			serverStatMap						= Collections.synchronizedMap(new HashMap<String, ServerStat>());
	private Map<String, UserStat>			userStatMap							= Collections.synchronizedMap(new HashMap<String, UserStat>());

	private boolean							initialized 						= false;
	
	private final ReentrantLock				lock								= new ReentrantLock();
	
	/** Metrics */
	private boolean							isIdle								= true;
	private long							totalEventsProcessed				= 0;
	private long							totalTimeInEventProcessing			= 0;
	private long							resetTime					= System.currentTimeMillis();

	
	// ################################################################
	// Spring related methods
	// ################################################################	
	public void setCollectThreadEnabled(boolean enabled) {
		this.collectThreadEnabled = enabled;
		if(initialized) {
			if(enabled && !collectThreadRunning) {
				// start update thread
				startUpdateThread();
				
				// add this as EventInfo observer
				M_ets.addLocalObserver(this);
			}else if(!enabled && collectThreadRunning){
				// remove this as EventInfo observer
				M_ets.deleteObserver(this);	
				
				// stop update thread
				stopUpdateThread();
			}
		}
	}
	
	public boolean isCollectThreadEnabled() {
		return collectThreadEnabled;
	}
	
	public void setCollectThreadUpdateInterval(long dbUpdateInterval){
		this.collectThreadUpdateInterval = dbUpdateInterval;
	}
	
	public long getCollectThreadUpdateInterval(){
		return collectThreadUpdateInterval;
	}	
	
	public void setCollectAdminEvents(boolean value){
		this.collectAdminEvents = value;
	}

	public boolean isCollectAdminEvents(){
		return collectAdminEvents;
	}

	public void setCollectEventsForSiteWithToolOnly(boolean value){
		this.collectEventsForSiteWithToolOnly = value;
	}
	
	public boolean isCollectEventsForSiteWithToolOnly(){
		return collectEventsForSiteWithToolOnly;
	}
	
	public void setStatsManager(StatsManager mng){
		this.M_sm = mng;
	}

	public void setEventRegistryService(EventRegistryService eventRegistryService) {
		this.M_ers = eventRegistryService;
	}
	
	public void setSiteService(SiteService ss){
		this.M_ss = ss;
	}
	
	public void setAliasService(AliasService as) {
		this.M_as = as;
	}
	
	public void setEntityManager(EntityManager em) {
		this.M_em = em;
	}
	
	public void setEventTrackingService(EventTrackingService ets){
		this.M_ets = ets;
	}
	
	public void setUsageSessionService(UsageSessionService uss){
		this.M_uss = uss;
	}
	
	public void init(){		
		StringBuilder buff = new StringBuilder();
		buff.append("init(): collect thread enabled: ");
		buff.append(collectThreadEnabled);
		if(collectThreadEnabled) {
			buff.append(", db update interval: ");
			buff.append(collectThreadUpdateInterval);
			buff.append(" ms");
		}
		buff.append(", collect administrator events: " + collectAdminEvents);
		buff.append(", collect events only for sites with SiteStats: " + collectEventsForSiteWithToolOnly);			
		logger.info(buff.toString());
		
		initialized = true;
		setCollectThreadEnabled(collectThreadEnabled);
	}
	
	public void destroy(){
		if(collectThreadEnabled) {
			// remove this as EventInfo observer
			M_ets.deleteObserver(this);	
			
			// stop update thread
			stopUpdateThread();
		}
	}

	
	// ################################################################
	// Public methods
	// ################################################################
	public Event buildEvent(Date date, String event, String ref, String sessionUser, String sessionId) {
		return new CustomEventImpl(date, event, ref, sessionUser, sessionId);
	}

	public Event buildEvent(Date date, String event, String ref, String context, String sessionUser, String sessionId) {
		return new CustomEventImpl(date, event, ref, context, sessionUser, sessionId);
	}
	
	/* (non-Javadoc)
	 * @see org.sakaiproject.sitestats.api.StatsUpdateManager#collectEvent(org.sakaiproject.event.api.Event)
	 */
	public boolean collectEvent(Event e) {
		if(e != null) {
			long startTime = System.currentTimeMillis();
			isIdle = false;
			preProcessEvent(e);
			//long endTime = System.currentTimeMillis();
			//LOG.debug("Time spent pre-processing 1 event: " + (endTime-startTime) + " ms");
			boolean success = doUpdateConsolidatedEvents();
			isIdle = true;
			totalTimeInEventProcessing += (System.currentTimeMillis() - startTime);
			return success;
		}
		return true;
	}

	/* (non-Javadoc)
	 * @see org.sakaiproject.sitestats.api.StatsUpdateManager#collectEvents(java.util.List)
	 */
	public boolean collectEvents(List<Event> events) {
		if(events != null) {
			int eventCount = events.size();
			if(eventCount > 0) {
				return collectEvents(events.toArray(new Event[eventCount]));
			}
		}
		return true;
	}
	
	/* (non-Javadoc)
	 * @see org.sakaiproject.sitestats.api.StatsUpdateManager#collectEvents(org.sakaiproject.event.api.Event[])
	 */
	public boolean collectEvents(Event[] events) {
		if(events != null) {
			int eventCount = events.length;
			if(eventCount > 0) {
				long startTime = System.currentTimeMillis();
				isIdle = false;
				for(int i=0; i<events.length; i++){
					if(events[i] != null) {
						preProcessEvent(events[i]);
					}
				}
				//long endTime = System.currentTimeMillis();
				//LOG.debug("Time spent pre-processing " + eventCount + " event(s): " + (endTime-startTime) + " ms");
				boolean success = doUpdateConsolidatedEvents();
				isIdle = true;
				totalTimeInEventProcessing += (System.currentTimeMillis() - startTime);
				return success;
			}
		}
		return true;
	}
	
	/* (non-Javadoc)
	 * @see org.sakaiproject.sitestats.api.StatsUpdateManager#collectPastSiteEvents(java.lang.String, java.util.Date, java.util.Date)
	 */
	public long collectPastSiteEvents(String siteId, Date initialDate, Date finalDate) {
		StatsAggregateJobImpl statsAggregateJob = (StatsAggregateJobImpl) ComponentManager.get("org.sakaiproject.sitestats.api.StatsAggregateJob");
		return statsAggregateJob.collectPastSiteEvents(siteId, initialDate, finalDate);
	}
	
	/* (non-Javadoc)
	 * @see org.sakaiproject.sitestats.api.StatsUpdateManager#saveJobRun(org.sakaiproject.sitestats.api.JobRun)
	 */
	public boolean saveJobRun(final JobRun jobRun){
		if(jobRun == null) {
			return false;
		}
		Object r = getHibernateTemplate().execute(new HibernateCallback() {			
			public Object doInHibernate(Session session) throws HibernateException, SQLException {
				Transaction tx = null;
				try{
					tx = session.beginTransaction();
					session.saveOrUpdate(jobRun);
					tx.commit();
				}catch(Exception e){
					if(tx != null) tx.rollback();
					LOG.warn("Unable to commit transaction: ", e);
					return Boolean.FALSE;
				}
				return Boolean.TRUE;
			}			
		});
		return ((Boolean) r).booleanValue();
	}
	
	/* (non-Javadoc)
	 * @see org.sakaiproject.sitestats.api.StatsUpdateManager#getLatestJobRun()
	 */
	public JobRun getLatestJobRun() throws Exception {
		Object r = getHibernateTemplate().execute(new HibernateCallback() {			
			public Object doInHibernate(Session session) throws HibernateException, SQLException {
				JobRun jobRun = null;
				Criteria c = session.createCriteria(JobRunImpl.class);
				c.setMaxResults(1);
				c.addOrder(Order.desc("id"));
				List jobs = c.list();
				if(jobs != null && jobs.size() > 0){
					jobRun = (JobRun) jobs.get(0);
				}
				return jobRun;
			}			
		});
		return (JobRun) r;
	}
	
	/* (non-Javadoc)
	 * @see org.sakaiproject.sitestats.api.StatsUpdateManager#getEventDateFromLatestJobRun()
	 */
	public Date getEventDateFromLatestJobRun() throws Exception {
		Object r = getHibernateTemplate().execute(new HibernateCallback() {			
			public Object doInHibernate(Session session) throws HibernateException, SQLException {
				Criteria c = session.createCriteria(JobRunImpl.class);
				c.add(Expression.isNotNull("lastEventDate"));
				c.setMaxResults(1);
				c.addOrder(Order.desc("id"));
				List jobs = c.list();
				if(jobs != null && jobs.size() > 0){
					JobRun jobRun = (JobRun) jobs.get(0);
					return jobRun.getLastEventDate();
				}
				return null;
			}			
		});
		return (Date) r;
	}
	
	
	// ################################################################
	// Metrics related methods
	// ################################################################	
	public int getQueueSize() {
		return collectThreadQueue.size();
	}
	
	public boolean isIdle() {
		return this.isIdle && getQueueSize() == 0;
	}
	
	public void resetMetrics() {
		totalEventsProcessed = 0;
		totalTimeInEventProcessing = 0;
		resetTime = System.currentTimeMillis();
	}
	
	public long getNumberOfEventsProcessed() {
		return totalEventsProcessed;
	}
	
	public long getTotalTimeInEventProcessing() {
		return totalTimeInEventProcessing;
	}
	
	public long getResetTime() {
		return resetTime;
	}
	
	public long getTotalTimeEllapsedSinceReset() {
		return System.currentTimeMillis() - resetTime;
	}
	
	public double getNumberOfEventsProcessedPerSec() {
		if(totalTimeInEventProcessing > 0) {
			return Util.round((double)totalEventsProcessed / ((double)totalTimeInEventProcessing/1000), 3);
		}else{
			return Util.round((double)totalEventsProcessed / 0.001, 3); // => will assume 1ms instead of 0ms
		}
	}
	
	public double getNumberOfEventsGeneratedPerSec() {
		double ellapsed = (double) getTotalTimeEllapsedSinceReset();
		if(ellapsed > 0) {
			return Util.round((double)totalEventsProcessed / (ellapsed/1000), 3);
		}else{
			return Util.round((double)totalEventsProcessed / 0.001, 3); // => will assume 1ms instead of 0ms
		}
	}
	
	public long getAverageTimeInEventProcessingPerEvent() {
		if(totalEventsProcessed > 0) {
			return totalTimeInEventProcessing / totalEventsProcessed;
		}else{
			return 0;
		}
	}
	
	public String getMetricsSummary(boolean compact) {
		StringBuilder sb = new StringBuilder();
		if(!compact) {
			sb.append("SiteStats Event aggregation metrics summary:\n");
			sb.append("\t\tNumber of total events processed: ").append(getNumberOfEventsProcessed()).append("\n");
			sb.append("\t\tReset/init time: ").append(new Date(getResetTime())).append("\n");
			sb.append("\t\tTotal time ellapsed since reset: ").append(getTotalTimeEllapsedSinceReset()).append(" ms\n");
			sb.append("\t\tTotal time spent processing events: ").append(getTotalTimeInEventProcessing()).append(" ms\n");
			sb.append("\t\tNumber of events processed per sec: ").append(getNumberOfEventsProcessedPerSec()).append("\n");
			sb.append("\t\tNumber of events genereated in Sakai per sec: ").append(getNumberOfEventsGeneratedPerSec()).append("\n");
			sb.append("\t\tAverage time spent in event processing per event: ").append(getAverageTimeInEventProcessingPerEvent()).append(" ms\n");
			sb.append("\t\tEvent queue size: ").append(getQueueSize()).append("\n");
			sb.append("\t\tIdle: ").append(isIdle());
		}else{
			sb.append("#Events processed: ").append(getNumberOfEventsProcessed()).append(", ");
			sb.append("Time ellapsed since reset: ").append(getTotalTimeEllapsedSinceReset()).append(" ms, ");
			sb.append("Time spent processing events: ").append(getTotalTimeInEventProcessing()).append(" ms, ");
			sb.append("#Events processed/sec: ").append(getNumberOfEventsProcessedPerSec()).append(", ");
			sb.append("Avg. Time/event: ").append(getAverageTimeInEventProcessingPerEvent()).append(" ms, ");
			sb.append("Event queue size: ").append(getQueueSize()).append(", ");
			sb.append("Idle: ").append(isIdle());
		}
		return sb.toString();
	}
	

	// ################################################################
	// Update thread related methods
	// ################################################################	
	/** Method called whenever an new event is generated from EventTrackingService: do not call this method! */
	public void update(Observable obs, Object o) {
		// At the moment this isn't threadsafe, but as sakai event handling is single threaded this shoudn't be a problem,
		// but it's not a formal contract.
		if(o instanceof Event){
			Event e = (Event) o;
			Event eventWithPreciseDate = buildEvent(getToday(), e.getEvent(), e.getResource(), e.getContext(), e.getUserId(), e.getSessionId());
			collectThreadQueue.add(eventWithPreciseDate);
		}
	}
	
	/** Update thread: do not call this method! */
	public void run(){
		try{
			LOG.debug("Started statistics update thread");
			while(collectThreadRunning){
				// do update job
				isIdle = false;
				long startTime = System.currentTimeMillis();
				int eventCount = collectThreadQueue.size();
				if(eventCount > 0) {
					//long startTime2 = System.currentTimeMillis();
					while(collectThreadQueue.size() > 0){
						preProcessEvent(collectThreadQueue.remove(0));
					}
					//long endTime2 = System.currentTimeMillis();
					//LOG.debug("Time spent pre-processing " + eventCount + " event(s): " + (endTime2-startTime2) + " ms");
				}
				doUpdateConsolidatedEvents();
				isIdle = true;
				totalTimeInEventProcessing += (System.currentTimeMillis() - startTime);
				
				// sleep if no work to do
				if(!collectThreadRunning) break;
				try{
					synchronized (collectThreadSemaphore){
						collectThreadSemaphore.wait(collectThreadUpdateInterval);
					}
				}catch(InterruptedException e){
					LOG.warn("Failed to sleep statistics update thread",e);
				}
			}
		}catch(Throwable t){
			LOG.debug("Failed to execute statistics update thread",t);
		}finally{
			if(collectThreadRunning){
				// thread was stopped by an unknown error: restart
				LOG.debug("Statistics update thread was stoped by an unknown error: restarting...");
				startUpdateThread();
			}else
				LOG.debug("Finished statistics update thread");
		}
	}

	/** Start the update thread */
	private void startUpdateThread(){
		collectThreadRunning = true;
		collectThread = null;
		collectThread = new Thread(this, "org.sakaiproject.sitestats.impl.StatsUpdateManagerImpl");
		collectThread.start();
	}
	
	/** Stop the update thread */
	private void stopUpdateThread(){
		collectThreadRunning = false;
		synchronized (collectThreadSemaphore){
			collectThreadSemaphore.notifyAll();
		}
	}
	

	// ################################################################
	// Event process methods
	// ################################################################	
	private void preProcessEvent(Event e) {
		totalEventsProcessed++;
		String userId = e.getUserId();
		e = fixMalFormedEvents(e);
		if(getRegisteredEvents().contains(e.getEvent()) && isValidEvent(e)){
			
			// site check
			String siteId = parseSiteId(e);
			if(siteId == null || M_ss.isUserSite(siteId) || M_ss.isSpecialSite(siteId)){
				return;
			}
			Site site = getSite(siteId);
			if(site == null) {
				return;
			}
			if(isCollectEventsForSiteWithToolOnly() && site.getToolForCommonId(StatsManager.SITESTATS_TOOLID) == null) {
				return;
			}
			
			// user check
			if(userId == null) {
				UsageSession session = M_uss.getSession(e.getSessionId());
				if(session != null) {
					userId = session.getUserId();
				}
			}
			if(!isCollectAdminEvents() && ("admin").equals(userId)){
				return;
			}if(!M_sm.isShowAnonymousAccessEvents() && ("?").equals(userId)){
				return;
			}
			
			// consolidate event
			Date date = null;
			if(e instanceof CustomEventImpl){
				date = ((CustomEventImpl) e).getDate();
			}else{
				date = getToday();
			}
			String eventId = e.getEvent();
			String resourceRef = e.getResource();
			
			if(userId == null || eventId == null || resourceRef == null)
				return;
			consolidateEvent(date, eventId, resourceRef, userId, siteId);
		} else if(getServerEvents().contains(e.getEvent())){
			
			//it's a server event
			if(LOG.isDebugEnabled()) {
				LOG.debug("Server event: "+e.toString());
			}
			
			String eventId = e.getEvent();
			
			if(eventId == null) {
				return;
			}
			Date date = new Date();
			
			consolidateServerEvent(date, eventId);
		} 
		
		//we do this separately as we want individual login stats as well as totals from the server stats section
		if (isUserLoginEvent(e)) {
			
			//it's a user event
			if(LOG.isDebugEnabled()) {
				LOG.debug("User event: "+e.toString());
			}
			
			// user check
			if(userId == null) {
				UsageSession session = M_uss.getSession(e.getSessionId());
				if(session != null) {
					userId = session.getUserId();
				}
			}
			
			if(userId == null) {
				return;
			}
			
			Date date = new Date();
			consolidateUserEvent(date, userId);
		}
		
		
		
		//else LOG.debug("EventInfo ignored:  '"+e.toString()+"' ("+e.toString()+") USER_ID: "+userId);
	}
	
	private void consolidateEvent(Date dateTime, String eventId, String resourceRef, String userId, String siteId) {
		if(eventId == null)
			return;
		Date date = getTruncatedDate(dateTime);
		// update		
		if(getRegisteredEvents().contains(eventId) && !StatsManager.SITEVISITEND_EVENTID.equals(eventId)){
			// add to eventStatMap
			String key = userId+siteId+eventId+date;
			synchronized(eventStatMap){
				EventStat e1 = eventStatMap.get(key);
				if(e1 == null){
					e1 = new EventStatImpl();
					e1.setUserId(userId);
					e1.setSiteId(siteId);
					e1.setEventId(eventId);
					e1.setDate(date);
				}
				e1.setCount(e1.getCount() + 1);
				eventStatMap.put(key, e1);
			}
			
			if(!StatsManager.SITEVISIT_EVENTID.equals(eventId)){
				// add to activityMap
				String key2 = siteId+date+eventId;
				synchronized(activityMap){
					SiteActivity e2 = activityMap.get(key2);
					if(e2 == null){
						e2 = new SiteActivityImpl();
						e2.setSiteId(siteId);
						e2.setDate(date);
						e2.setEventId(eventId);
					}
					e2.setCount(e2.getCount() + 1);
					activityMap.put(key2, e2);
				}
			}
		}	
		
		if(eventId.startsWith(StatsManager.RESOURCE_EVENTID_PREFIX)){
			// add to resourceStatMap
			String resourceAction = null;
			try{
				resourceAction = eventId.split("\\.")[1];
			}catch(ArrayIndexOutOfBoundsException ex){
				resourceAction = eventId;
			}
			String key = userId+siteId+resourceRef+resourceAction+date;
			synchronized(resourceStatMap){
				ResourceStat e1 = resourceStatMap.get(key);
				if(e1 == null){
					e1 = new ResourceStatImpl();
					e1.setUserId(userId);
					e1.setSiteId(siteId);
					e1.setResourceRef(resourceRef);
					e1.setResourceAction(resourceAction);
					e1.setDate(date);
				}
				e1.setCount(e1.getCount() + 1);
				resourceStatMap.put(key, e1);
			}
			
		}else if(StatsManager.SITEVISIT_EVENTID.equals(eventId)){
			// add to visitsMap
			String key = siteId+date;
			lock.lock();
			try{
				SiteVisits e1 = visitsMap.get(key);
				if(e1 == null){
					e1 = new SiteVisitsImpl();
					e1.setSiteId(siteId);
					e1.setDate(date);
				}
				e1.setTotalVisits(e1.getTotalVisits() + 1);
				// unique visits are determined when updating to db:
				//   --> e1.setTotalUnique(totalUnique);
				visitsMap.put(key, e1);
				// place entry on map so we can update unique visits later
				UniqueVisitsKey keyUniqueVisits = new UniqueVisitsKey(siteId, date);
				uniqueVisitsMap.put(keyUniqueVisits, Integer.valueOf(1));
				
				// site presence started
				if(M_sm.isEnableSitePresences()) {
					String pKey = siteId+userId+date;
					SitePresenceConsolidation spc = presencesMap.get(pKey);
					if(spc == null) {
						SitePresence sp = new SitePresenceImpl();
						sp.setSiteId(siteId);
						sp.setUserId(userId);
						sp.setDate(date);
						spc = new SitePresenceConsolidation(sp);
					}
					spc.sitePresence.setLastVisitStartTime(dateTime);
					presencesMap.put(pKey, spc);					
				}
			}finally{
				lock.unlock();
			}
			
		}else if(StatsManager.SITEVISITEND_EVENTID.equals(eventId)){
			// site presence ended
			if(M_sm.isEnableSitePresences()) {
				String pKey = siteId+userId+date;
				lock.lock();
				try{
					SitePresenceConsolidation spc = presencesMap.get(pKey);
					if(spc == null) {
						SitePresence sp = new SitePresenceImpl();
						sp.setSiteId(siteId);
						sp.setUserId(userId);
						sp.setDate(date);
						sp.setLastVisitStartTime(null);
						spc = new SitePresenceConsolidation(sp, dateTime);
					}
					if(spc.sitePresence.getLastVisitStartTime() != null) {
						long existingDuration = spc.sitePresence.getDuration();
						long start = spc.sitePresence.getLastVisitStartTime().getTime();
						long thisEventTime = dateTime.getTime();
						long additionalDuration = thisEventTime - start;
						if(additionalDuration > 4*60*60*1000) {
							LOG.warn("A site presence is longer than 4h!: duration="+(additionalDuration/1000/60)+" min (SITE:"+siteId+", USER:"+userId+", DATE:"+date+")");
						}
						spc.sitePresence.setDuration(existingDuration + additionalDuration);						
						spc.sitePresence.setLastVisitStartTime(null);	
					}
					presencesMap.put(pKey, spc);
				}finally{
					lock.unlock();
				}
			}
		} 
		
	}
	
	//STAT-299 consolidate a server event
	private void consolidateServerEvent(Date dateTime, String eventId) {
		
		Date date = getTruncatedDate(dateTime);
				
		// add to serverStatMap
		String key = eventId+date;
		synchronized(serverStatMap){
			ServerStat s = serverStatMap.get(key);
			if(s == null){
				s = new ServerStatImpl();
				s.setEventId(eventId);
				s.setDate(date);
			}
			s.setCount(s.getCount() + 1);
			serverStatMap.put(key, s);
		}
		
	}
	
	//STAT-299 consolidate a user event
	private void consolidateUserEvent(Date dateTime, String userId) {
		
		Date date = getTruncatedDate(dateTime);
				
		// add to userStatMap
		String key = userId+date;
		synchronized(userStatMap){
			UserStat s = userStatMap.get(key);
			if(s == null){
				s = new UserStatImpl();
				s.setUserId(userId);
				s.setDate(date);
			}
			s.setCount(s.getCount() + 1);
			userStatMap.put(key, s);
		}
		
	}
	

	// ################################################################
	// Db update methods
	// ################################################################	
	@SuppressWarnings("unchecked")
	private synchronized boolean doUpdateConsolidatedEvents() {
		long startTime = System.currentTimeMillis();
		if(eventStatMap.size() > 0 || resourceStatMap.size() > 0
				|| activityMap.size() > 0 || uniqueVisitsMap.size() > 0 
				|| visitsMap.size() > 0 || presencesMap.size() > 0
				|| serverStatMap.size() > 0 || userStatMap.size() > 0) {
			Object r = getHibernateTemplate().execute(new HibernateCallback() {			
				public Object doInHibernate(Session session) throws HibernateException, SQLException {
					Transaction tx = null;
					try{
						tx = session.beginTransaction();
						// do: EventStat
						if(eventStatMap.size() > 0) {
							Collection<EventStat> tmp1 = null;
							synchronized(eventStatMap){
								tmp1 = eventStatMap.values();
								eventStatMap = Collections.synchronizedMap(new HashMap<String, EventStat>());
							}
							doUpdateEventStatObjects(session, tmp1);
						}
						
						// do: ResourceStat
						if(resourceStatMap.size() > 0) {
							Collection<ResourceStat> tmp2 = null;
							synchronized(resourceStatMap){
								tmp2 = resourceStatMap.values();
								resourceStatMap = Collections.synchronizedMap(new HashMap<String, ResourceStat>());
							}
							doUpdateResourceStatObjects(session, tmp2);
						}
						
						// do: SiteActivity
						if(activityMap.size() > 0) {
							Collection<SiteActivity> tmp3 = null;
							synchronized(activityMap){
								tmp3 = activityMap.values();
								activityMap = Collections.synchronizedMap(new HashMap<String, SiteActivity>());
							}
							doUpdateSiteActivityObjects(session, tmp3);
						}
	
						// do: SiteVisits
						if(uniqueVisitsMap.size() > 0 || visitsMap.size() > 0) {	
							// determine unique visits for event related sites
							Map<UniqueVisitsKey, Integer> tmp4;
							synchronized(uniqueVisitsMap){
								tmp4 = uniqueVisitsMap;
								uniqueVisitsMap = Collections.synchronizedMap(new HashMap<UniqueVisitsKey, Integer>());
							}
							tmp4 = doGetSiteUniqueVisits(session, tmp4);
						
							// do: SiteVisits
							if(visitsMap.size() > 0) {
								Collection<SiteVisits> tmp5 = null;
								synchronized(visitsMap){
									tmp5 = visitsMap.values();
									visitsMap = Collections.synchronizedMap(new HashMap<String, SiteVisits>());
								}
								doUpdateSiteVisitsObjects(session, tmp5, tmp4);
							}
						}
						
						// do: SitePresences
						if(presencesMap.size() > 0) {
							Collection<SitePresenceConsolidation> tmp6 = null;
							synchronized(presencesMap){
								tmp6 = presencesMap.values();
								presencesMap = Collections.synchronizedMap(new HashMap<String, SitePresenceConsolidation>());
							}
							doUpdateSitePresencesObjects(session, tmp6);
						}
						
						// do: ServerStats
						if(serverStatMap.size() > 0) {
							Collection<ServerStat> tmp7 = null;
							synchronized(serverStatMap){
								tmp7 = serverStatMap.values();
								serverStatMap = Collections.synchronizedMap(new HashMap<String, ServerStat>());
							}
							doUpdateServerStatObjects(session, tmp7);
						}
						
						// do: UserStats
						if(userStatMap.size() > 0) {
							Collection<UserStat> tmp8 = null;
							synchronized(userStatMap){
								tmp8 = userStatMap.values();
								userStatMap = Collections.synchronizedMap(new HashMap<String, UserStat>());
							}
							doUpdateUserStatObjects(session, tmp8);
						}
	
						// commit ALL
						tx.commit();
					}catch(Exception e){
						if(tx != null) tx.rollback();
						LOG.warn("Unable to commit transaction: ", e);
						return Boolean.FALSE;
					}
					return Boolean.TRUE;
				}			
			});
			long endTime = System.currentTimeMillis();
			LOG.debug("Time spent in doUpdateConsolidatedEvents(): " + (endTime-startTime) + " ms");
			return ((Boolean) r).booleanValue();
		}else{
			return true;
		}
	}
	
	private void doUpdateEventStatObjects(Session session, Collection<EventStat> objects) {
		if(objects == null) return;
		Iterator<EventStat> i = objects.iterator();
		while(i.hasNext()){
			EventStat eUpdate = i.next();
			String eExistingSiteId = null;
			EventStat eExisting = null;
			try{
				Criteria c = session.createCriteria(EventStatImpl.class);
				c.add(Expression.eq("siteId", eUpdate.getSiteId()));
				c.add(Expression.eq("eventId", eUpdate.getEventId()));
				c.add(Expression.eq("userId", eUpdate.getUserId()));
				c.add(Expression.eq("date", eUpdate.getDate()));
				try{
					eExisting = (EventStat) c.uniqueResult();
				}catch(HibernateException ex){
					try{
						List events = c.list();
						if ((events!=null) && (events.size()>0)){
							LOG.debug("More than 1 result when unique result expected.", ex);
							eExisting = (EventStat) c.list().get(0);
						}else{
							LOG.debug("No result found", ex);
							eExisting = null;
						}
					}catch(Exception ex3){
						eExisting = null;
					}
				}catch(Exception ex2){
					LOG.debug("Probably ddbb error when loading data at java object", ex2);
					System.out.println("Probably ddbb error when loading data at java object!!!!!!!!");
					
				}
				if(eExisting == null) 
					eExisting = eUpdate;
				else
					eExisting.setCount(eExisting.getCount() + eUpdate.getCount());
	
				eExistingSiteId = eExisting.getSiteId();
			}catch(Exception ex){
				//If something happens, skip the event processing
				ex.printStackTrace();
			}
			if ((eExistingSiteId!=null) && (eExistingSiteId.trim().length()>0))
					session.saveOrUpdate(eExisting);
		}
	}

	private void doUpdateResourceStatObjects(Session session, Collection<ResourceStat> objects) {
		if(objects == null) return;
		Iterator<ResourceStat> i = objects.iterator();
		while(i.hasNext()){
			ResourceStat eUpdate = i.next();
			ResourceStat eExisting = null;
			String eExistingSiteId = null;
			try{
				Criteria c = session.createCriteria(ResourceStatImpl.class);
				c.add(Expression.eq("siteId", eUpdate.getSiteId()));
				c.add(Expression.eq("resourceRef", eUpdate.getResourceRef()));
				c.add(Expression.eq("resourceAction", eUpdate.getResourceAction()));
				c.add(Expression.eq("userId", eUpdate.getUserId()));
				c.add(Expression.eq("date", eUpdate.getDate()));
				try{
					eExisting = (ResourceStat) c.uniqueResult();
				}catch(HibernateException ex){
					try{
						List events = c.list();
						if ((events!=null) && (events.size()>0)){
							LOG.debug("More than 1 result when unique result expected.", ex);
							eExisting = (ResourceStat) c.list().get(0);
						}else{
							LOG.debug("No result found", ex);
							eExisting = null;
						}
					}catch(Exception ex3){
						eExisting = null;
					}
				}catch(Exception ex2){
					LOG.debug("Probably ddbb error when loading data at java object", ex2);
					System.out.println("Probably ddbb error when loading data at java object!!!!!!!!");
					
				}
				if(eExisting == null) 
					eExisting = eUpdate;
				else
					eExisting.setCount(eExisting.getCount() + eUpdate.getCount());
				
				eExistingSiteId = eExisting.getSiteId();
			}catch(Exception e){
				e.printStackTrace();
			}
			if ((eExistingSiteId!=null) && (eExistingSiteId.trim().length()>0))
					session.saveOrUpdate(eExisting);
		}
	}
	
	private void doUpdateSiteActivityObjects(Session session, Collection<SiteActivity> objects) {
		if(objects == null) return;
		Iterator<SiteActivity> i = objects.iterator();
		while(i.hasNext()){
			SiteActivity eUpdate = i.next();
			SiteActivity eExisting = null;
			String eExistingSiteId = null;
			try{
				Criteria c = session.createCriteria(SiteActivityImpl.class);
				c.add(Expression.eq("siteId", eUpdate.getSiteId()));
				c.add(Expression.eq("eventId", eUpdate.getEventId()));
				c.add(Expression.eq("date", eUpdate.getDate()));
				try{
					eExisting = (SiteActivity) c.uniqueResult();
				}catch(HibernateException ex){
					try{
						List events = c.list();
						if ((events!=null) && (events.size()>0)){
							LOG.debug("More than 1 result when unique result expected.", ex);
							eExisting = (SiteActivity) c.list().get(0);
						}else{
							LOG.debug("No result found", ex);
							eExisting = null;
						}
					}catch(Exception ex3){
						eExisting = null;
					}
				}catch(Exception ex2){
					LOG.debug("Probably ddbb error when loading data at java object", ex2);
					System.out.println("Probably ddbb error when loading data at java object!!!!!!!!");
					
				}
				if(eExisting == null) 
					eExisting = eUpdate;
				else
					eExisting.setCount(eExisting.getCount() + eUpdate.getCount());
	
				eExistingSiteId = eExisting.getSiteId();
			}catch(Exception e){
				e.printStackTrace();
			}
			
			if ((eExistingSiteId!=null) && (eExistingSiteId.trim().length()>0))
					session.saveOrUpdate(eExisting);
		}
	}
	
	private void doUpdateSiteVisitsObjects(Session session, Collection<SiteVisits> objects, Map<UniqueVisitsKey, Integer> map) {
		if(objects == null) return;
		Iterator<SiteVisits> i = objects.iterator();
		while(i.hasNext()){
			SiteVisits eUpdate = i.next();
			SiteVisits eExisting = null;
			String eExistingSiteId = null;
			try{
				Criteria c = session.createCriteria(SiteVisitsImpl.class);
				c.add(Expression.eq("siteId", eUpdate.getSiteId()));
				c.add(Expression.eq("date", eUpdate.getDate()));
				try{
					eExisting = (SiteVisits) c.uniqueResult();
				}catch(HibernateException ex){
					try{
						List events = c.list();
						if ((events!=null) && (events.size()>0)){
							LOG.debug("More than 1 result when unique result expected.", ex);
							eExisting = (SiteVisits) c.list().get(0);
						}else{
							LOG.debug("No result found", ex);
							eExisting = null;
						}
					}catch(Exception ex3){
						eExisting = null;
					}
				}catch(Exception ex2){
					LOG.debug("Probably ddbb error when loading data at java object", ex2);
					System.out.println("Probably ddbb error when loading data at java object!!!!!!!!");
					
				}
				if(eExisting == null){
					eExisting = eUpdate;
				}else{
					eExisting.setTotalVisits(eExisting.getTotalVisits() + eUpdate.getTotalVisits());
				}
				Integer mapUV = map.get(new UniqueVisitsKey(eExisting.getSiteId(), eExisting.getDate()));
				eExisting.setTotalUnique(mapUV == null? 1 : mapUV.longValue());
	
				eExistingSiteId = eExisting.getSiteId();
			}catch(Exception e){
				e.printStackTrace();
			}
			if ((eExistingSiteId!=null) && (eExistingSiteId.trim().length()>0))
					session.saveOrUpdate(eExisting);
		}
	}
	
	private void doUpdateSiteVisitTimeObjects(Session session, Collection<SitePresence> objects) {
		if(objects == null) return;
		Iterator<SitePresence> i = objects.iterator();
		while(i.hasNext()){
			SitePresence eUpdate = i.next();
			SitePresence eExisting = null;
			String eExistingSiteId = null;
			try{
				Criteria c = session.createCriteria(SitePresenceImpl.class);
				c.add(Expression.eq("siteId", eUpdate.getSiteId()));
				c.add(Expression.eq("userId", eUpdate.getUserId()));
				c.add(Expression.eq("date", eUpdate.getDate()));
				try{
					eExisting = (SitePresence) c.uniqueResult();
				}catch(HibernateException ex){
					try{
						List<SitePresence> events = (List<SitePresence>) c.list();
						if ((events!=null) && (events.size()>0)){
							LOG.debug("More than 1 result when unique result expected.", ex);
							eExisting = (SitePresence) c.list().get(0);
						}else{
							LOG.debug("No result found", ex);
							eExisting = null;
						}
					}catch(Exception ex3){
						eExisting = null;
					}
				}catch(Exception ex2){
					LOG.debug("Probably ddbb error when loading data at java object", ex2);
					System.out.println("Probably ddbb error when loading data at java object!!!!!!!!");
					
				}
				if(eExisting == null){
					eExisting = eUpdate;
				}else{
					eExisting.setDuration(eExisting.getDuration() + eUpdate.getDuration());
				}
				eExistingSiteId = eExisting.getSiteId();
			}catch(Exception e){
				e.printStackTrace();
			}
			if ((eExistingSiteId!=null) && (eExistingSiteId.trim().length()>0))
					session.saveOrUpdate(eExisting);
		}
	}
	
	private void doUpdateServerStatObjects(Session session, Collection<ServerStat> objects) {
		if(objects == null) return;
		Iterator<ServerStat> i = objects.iterator();
		while(i.hasNext()){
			ServerStat eUpdate = i.next();
			ServerStat eExisting = null;
			try{
				Criteria c = session.createCriteria(ServerStatImpl.class);
				c.add(Expression.eq("eventId", eUpdate.getEventId()));
				c.add(Expression.eq("date", eUpdate.getDate()));
				try{
					eExisting = (ServerStat) c.uniqueResult();
				}catch(HibernateException ex){
					try{
						List events = c.list();
						if ((events!=null) && (events.size()>0)){
							LOG.debug("More than 1 result when unique result expected.", ex);
							eExisting = (ServerStat) c.list().get(0);
						}else{
							LOG.debug("No result found", ex);
							eExisting = null;
						}
					}catch(Exception ex3){
						eExisting = null;
					}
				}catch(Exception ex2){
					LOG.debug("Probably ddbb error when loading data at java object", ex2);
					System.out.println("Probably ddbb error when loading data at java object!!!!!!!!");
					
				}
				if(eExisting == null) {
					eExisting = eUpdate;
				}else{
					eExisting.setCount(eExisting.getCount() + eUpdate.getCount());
				}
				
			}catch(Exception e){
				e.printStackTrace();
			}
			session.saveOrUpdate(eExisting);
		}
	}
	
	private void doUpdateUserStatObjects(Session session, Collection<UserStat> objects) {
		if(objects == null) return;
		Iterator<UserStat> i = objects.iterator();
		while(i.hasNext()){
			UserStat eUpdate = i.next();
			UserStat eExisting = null;
			String eExistingUserId = null;
			try{
				Criteria c = session.createCriteria(UserStatImpl.class);
				c.add(Expression.eq("userId", eUpdate.getUserId()));
				c.add(Expression.eq("date", eUpdate.getDate()));
				try{
					eExisting = (UserStat) c.uniqueResult();
				}catch(HibernateException ex){
					try{
						List events = c.list();
						if ((events!=null) && (events.size()>0)){
							LOG.debug("More than 1 result when unique result expected.", ex);
							eExisting = (UserStat) c.list().get(0);
						}else{
							LOG.debug("No result found", ex);
							eExisting = null;
						}
					}catch(Exception ex3){
						eExisting = null;
					}
				}catch(Exception ex2){
					LOG.debug("Probably ddbb error when loading data at java object", ex2);
					System.out.println("Probably ddbb error when loading data at java object!!!!!!!!");
					
				}
				if(eExisting == null) {
					eExisting = eUpdate;
				}else{
					eExisting.setCount(eExisting.getCount() + eUpdate.getCount());
				}
				
				eExistingUserId = eExisting.getUserId();
				
			}catch(Exception e){
				e.printStackTrace();
			}
			
			if(StringUtils.isNotBlank(eExistingUserId)) {
				session.saveOrUpdate(eExisting);
			}
			
		}
	}
	
	private Map<UniqueVisitsKey, Integer> doGetSiteUniqueVisits(Session session, Map<UniqueVisitsKey, Integer> map) {
		Iterator<UniqueVisitsKey> i = map.keySet().iterator();
		while(i.hasNext()){
			UniqueVisitsKey key = i.next();
			Query q = session.createQuery("select count(distinct s.userId) " + 
					"from EventStatImpl as s " +
					"where s.siteId = :siteid " +
					"and s.eventId = 'pres.begin' " +
					"and s.date = :idate");
			q.setString("siteid", key.siteId);
			q.setDate("idate", key.date);
			Integer uv = 1;
			try{
				uv = (Integer) q.uniqueResult();
			}catch(ClassCastException ex){
				uv = (int) ((Long) q.uniqueResult()).longValue();
			}catch(HibernateException ex){
				try{
					List visits = q.list();
					if ((visits!=null) && (visits.size()>0)){
						LOG.debug("More than 1 result when unique result expected.", ex);
						uv = (Integer) q.list().get(0);
					}else{
						LOG.debug("No result found", ex);
						uv = 1;
					}
				}catch (Exception e3){
					uv = 1;
				}
				
			}catch(Exception ex2){
				LOG.debug("Probably ddbb error when loading data at java object", ex2);
			}
			int uniqueVisits = uv == null? 1 : uv.intValue();
			map.put(key, Integer.valueOf((int)uniqueVisits));			
		}
		return map;
	}
	
	private void doUpdateSitePresencesObjects(Session session, Collection<SitePresenceConsolidation> objects) {
		if(objects == null) return;
		Iterator<SitePresenceConsolidation> i = objects.iterator();
		while(i.hasNext()){
			try{
				SitePresenceConsolidation spc = i.next();
				SitePresence sp = spc.sitePresence;
				SitePresence spExisting = doGetSitePresence(session, sp.getSiteId(), sp.getUserId(), sp.getDate());
				if(spExisting == null) {
					session.save(sp);
				}else{
					long previousTotalPresence = spExisting.getDuration();
					long previousPresence = 0;
					long newTotalPresence = 0;
					if(spc.firstEventIsPresEnd) {
						if(spExisting.getLastVisitStartTime() != null)
							previousPresence = spc.firstPresEndDate.getTime() - spExisting.getLastVisitStartTime().getTime();
						else
							throw new Exception("No initial visit start time found - skipping");
					} 
					newTotalPresence = previousTotalPresence + previousPresence + sp.getDuration();
					spExisting.setDuration(newTotalPresence);				
					spExisting.setLastVisitStartTime(sp.getLastVisitStartTime());
					session.update(spExisting);
				}
			}catch(HibernateException e){
				LOG.debug("Probably ddbb error when loading data at java object", e);
			}catch(Exception e){
				LOG.debug("Unknow error while consolidating presence events", e);
			}
		}
	}
	

	// ################################################################
	// Special site presence methods (visit time tracking)
	// ################################################################
	@SuppressWarnings("unchecked")
	private SitePresence doGetSitePresence(Session session, String siteId, String userId, Date date) {
		SitePresence eDb = null;
		Criteria c = session.createCriteria(SitePresenceImpl.class);
		c.add(Expression.eq("siteId", siteId));
		c.add(Expression.eq("userId", userId));
		c.add(Expression.eq("date", date));
		
		try{
			eDb = (SitePresence) c.uniqueResult();
		}catch(HibernateException ex){
			try{
				List es = c.list();
				if(es != null && es.size() > 0){
					LOG.debug("More than 1 result when unique result expected.", ex);
					eDb = (SitePresence) es.get(0);
				}else{
					eDb = null;
				}
			}catch (Exception e3){
				LOG.debug("Probably ddbb error when loading data at java object", e3);
				eDb = null;
			}
			
		}catch(Exception ex2){
			LOG.debug("Probably ddbb error when loading data at java object", ex2);
		}
		return eDb;
	}
	

	// ################################################################
	// Utility methods
	// ################################################################	
	private synchronized boolean isValidEvent(Event e) {
		if(e.getEvent().startsWith(StatsManager.RESOURCE_EVENTID_PREFIX)){
			String ref = e.getResource();	
			if(ref.trim().equals("")) return false;			
			try{
				String parts[] = ref.split("\\/");		
				if(parts[2].equals("user")){
					// workspace (ignore)
					return false;
				}else if(parts[2].equals("attachment") && parts.length < 6){
					// ignore mail attachments (no reference to site)
					return false;
				}else if(parts[2].equals("group")){
					// resources
					if(parts.length <= 4) return false;	
				}else if(parts[2].equals("group-user")){
					// drop-box
					if(parts.length <= 5) return false;
				}else if ((parts.length >= 3) && (parts[2].equals("private"))) {
          // discard
					LOG.debug("Discarding content event in private area.");
					return false;
        }
      }catch(Exception ex){
				return false;
			}
		}
		return true;
	}
	
	private Event fixMalFormedEvents(Event e){
		String event = e.getEvent();
		String resource = e.getResource();
		
		// OBSOLETE: fix bad reference (resource) format
		// => Use <eventParserTip> instead
			//if(!resource.startsWith("/"))
			//	resource = '/' + resource;
		
		// MessageCenter (OLD) CASE: Handle old MessageCenter events */
		if (event!=null){
			if(event.startsWith(StatsManager.RESOURCE_EVENTID_PREFIX) && resource.startsWith("MessageCenter")) {
				resource = resource.replaceFirst("MessageCenter::", "/MessageCenter/site/");
				resource = resource.replaceAll("::", "/");
				return M_ets.newEvent(
						event.replaceFirst("content.", "msgcntr."), 
						resource, 
						false);
			}else{ 
				return e;
			}
		}else{
			return M_ets.newEvent("garbage.", resource, false);
		}
	}
	
	private String parseSiteId(Event e){
		String eventId = e.getEvent();
		
		// get contextId (siteId) from new Event.getContext() method, if available
		if(M_sm.isEventContextSupported()) {
			String contextId = null;
			try{
				contextId = (String) e.getClass().getMethod("getContext", null).invoke(e, null);
				// STAT-150 fix:
				String sitePrefix = "/site/";
				if(contextId != null && contextId.startsWith(sitePrefix)) {
					contextId = contextId.substring(sitePrefix.length());
				}
				LOG.debug("Context read from Event.getContext() for event: " + eventId + " - context: " + contextId);
			}catch(Exception ex){
				LOG.warn("Unable to get Event.getContext() for event: " + eventId, ex);
			}
			if(contextId != null)
				return contextId; 
		}
		
		// get contextId (siteId) from event reference
		String eventRef = e.getResource();
		if(eventRef != null){
			try{
				if(StatsManager.SITEVISIT_EVENTID.equals(eventId)
						|| StatsManager.SITEVISITEND_EVENTID.equals(eventId)){
					// presence (site visit) syntax (/presence/SITE_ID-presence)
					String[] parts = eventRef.split("/");
					if(parts.length > 2 && parts[2].endsWith(PRESENCE_SUFFIX)) {
						return parts[2].substring(0, parts[2].length() - PRESENCE_SUFFIX_LENGTH);
					}

				}else{
					// use <eventParserTip>
					ToolInfo toolInfo = getEventIdToolMap().get(eventId);
					EventParserTip parserTip = toolInfo.getEventParserTip();
					if(parserTip != null && parserTip.getFor().equals(StatsManager.PARSERTIP_FOR_CONTEXTID)){
						int index = Integer.parseInt(parserTip.getIndex());
						return eventRef.split(parserTip.getSeparator())[index];
					}else if(M_sm.isEventContextSupported()) {
						LOG.info("Context information unavailable for event: " + eventId + " (ignoring)");
					}else{
						LOG.info("<eventParserTip> is mandatory when Event.getContext() is unsupported! Ignoring event: " + eventId);
						// try with most common syntax (/abc/cde/SITE_ID/...)
						// return eventRef.split("/")[3];
					}
				}
			}catch(Exception ex){
				LOG.warn("Unable to parse contextId from event: " + eventId + " | " + eventRef, ex);
			}
		}
		return null;
	}
	
	private Site getSite(String siteId) {
		Site site = null;
		try{
			// is it a site id?
			site = M_ss.getSite(siteId);
		}catch(IdUnusedException e1){
			// is it an alias?
			try{
				String alias = siteId;
				String target = M_as.getTarget(alias);
				if(target != null) {
					String newSiteId = M_em.newReference(target).getId();
					LOG.debug(alias + " is an alias targetting site id: "+newSiteId);
					site = M_ss.getSite(newSiteId);
				}else{
					throw new IdUnusedException(siteId);
				}
			}catch(IdUnusedException e2){
				// not a valid site
				LOG.debug(siteId + " is not a valid site.", e2);
			}
		}catch(Exception ex) {
			// not a valid site
			LOG.debug(siteId + " is not a valid site.", ex);
		}
		return site;
	}
	
	/** Get all registered events */
	private Collection<String> getRegisteredEvents() {
		return M_ers.getEventIds();
	}
	
	/** Get all server events **/
	private Collection<String> getServerEvents() {
		return M_ers.getServerEventIds();
	}
	
	/** Get eventId -> ToolInfo map */
	private Map<String, ToolInfo> getEventIdToolMap() {
		return M_ers.getEventIdToolMap();
	}	
	
	/**
	 * Get the date for today (time of 00:00:00).
	 * This is used when we are grouping event by day.
	 */
	private Date getToday() {
		return new Date();
	}
	
	private boolean isUserLoginEvent(Event e) {
		return StringUtils.equals(StatsManager.LOGIN_EVENTID, e.getEvent());
	}
	
	private Date getTruncatedDate(Date date) {
		if(date == null) return null;
		Calendar c = Calendar.getInstance();
		c.setTime(date);
		c.set(Calendar.HOUR_OF_DAY, 0);
		c.set(Calendar.MINUTE, 0);
		c.set(Calendar.SECOND, 0);
		return c.getTime();
	}

	private static class UniqueVisitsKey {
		public String siteId;
		public Date date;
		
		public UniqueVisitsKey(String siteId, Date date){
			this.siteId = siteId;
			this.date = resetToDay(date);
		}

		@Override
		public boolean equals(Object o) {
			if(o instanceof UniqueVisitsKey) {
				UniqueVisitsKey u = (UniqueVisitsKey) o;
				return siteId.equals(u.siteId) && date.equals(u.date);
			}
			return false;
		}

		@Override
		public int hashCode() {
			return siteId.hashCode() + date.hashCode();
		}
		
		private Date resetToDay(Date date){
			Calendar c = Calendar.getInstance();
			c.setTime(date);
			c.set(Calendar.HOUR_OF_DAY, 0);
			c.set(Calendar.MINUTE, 0);
			c.set(Calendar.SECOND, 0);
			c.set(Calendar.MILLISECOND, 0);
			return c.getTime();
		}
	}
	
	private static class SitePresenceConsolidation {
		public boolean firstEventIsPresEnd;
		public Date firstPresEndDate;
		public SitePresence sitePresence;
		
		public SitePresenceConsolidation(SitePresence sitePresence) {
			this(sitePresence, null);
		}
		
		public SitePresenceConsolidation(SitePresence sitePresence, Date firstPresEndDate) {
			this.sitePresence = sitePresence;
			if(firstPresEndDate == null) {
				this.firstEventIsPresEnd = false;
				this.firstPresEndDate = null;
			}else{
				this.firstEventIsPresEnd = true;
				this.firstPresEndDate = firstPresEndDate;
			}
		}

		@Override
		public boolean equals(Object o) {
			if(o instanceof SitePresenceConsolidation) {
				SitePresenceConsolidation u = (SitePresenceConsolidation) o;
				return firstEventIsPresEnd == u.firstEventIsPresEnd
					&& ( (firstPresEndDate == null && u.firstPresEndDate == null) || (firstPresEndDate != null && firstPresEndDate.equals(u.firstPresEndDate)) )
					&& ( (sitePresence == null && u.sitePresence == null) || (sitePresence != null && sitePresence.equals(u.sitePresence)) );
			}
			return false;
		}

		@Override
		public int hashCode() {
			return Boolean.valueOf(firstEventIsPresEnd).hashCode() + firstPresEndDate.hashCode() + sitePresence.hashCode();
		}
		
		@Override
		public String toString() {
			StringBuilder buff = new StringBuilder();
			buff.append("firstPresEndDate: ");
			buff.append(firstPresEndDate);
			buff.append(" | sitePresence => ");
			if(sitePresence != null) buff.append(sitePresence.toString());
			else buff.append("null");
			return buff.toString();
		}
		
	}
	
}