/**
* $URL: https://source.sakaiproject.org/svn/sitestats/trunk/sitestats-impl/src/java/org/sakaiproject/sitestats/impl/StatsAggregateJobImpl.java $
* $Id: StatsAggregateJobImpl.java 105078 2012-02-24 23:00:38Z ottenhoff@longsight.com $
*
* Copyright (c) 2006-2009 The Sakai Foundation
*
* Licensed under the Educational Community License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.opensource.org/licenses/ECL-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.sakaiproject.sitestats.impl;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.quartz.JobExecutionContext;
import org.quartz.JobExecutionException;
import org.quartz.SchedulerException;
import org.quartz.StatefulJob;
import org.sakaiproject.component.app.scheduler.jobs.SpringJobBeanWrapper;
import org.sakaiproject.db.api.SqlService;
import org.sakaiproject.event.api.Event;
import org.sakaiproject.sitestats.api.JobRun;
import org.sakaiproject.sitestats.api.StatsUpdateManager;
public class StatsAggregateJobImpl implements StatefulJob {
private Log LOG = LogFactory.getLog(StatsAggregateJobImpl.class);
// Spring fields
private int maxEventsPerRun = 0;
private int sqlBlockSize = 1000;
private long startEventId = -1;
private long lastEventIdInTable = -1;
private String driverClassName = null;
private String url = null;
private String username = null;
private String password = null;
// Relevant job fields
private JobRun jobRun = null;
private Object extDbdriver = null;
private String sqlGetEvent = null;
private String sqlPastSiteEvents = null;
private boolean isOracle = false;
private boolean isEventContextSupported = false;
private final static String LAST_EVENT_ID = "select max(EVENT_ID) LAST_ID from SAKAI_EVENT";
private final static String MYSQL_DEFAULT_COLUMNS = "EVENT_ID as EVENT_ID,EVENT_DATE as EVENT_DATE,EVENT as EVENT,REF as REF,SESSION_USER as SESSION_USER,e.SESSION_ID as SESSION_ID";
private final static String ORACLE_DEFAULT_COLUMNS = "EVENT_ID,EVENT_DATE,EVENT,REF,SESSION_USER,e.SESSION_ID SESSION_ID";
private final static String MYSQL_CHECK_FOR_CONTEXT = "show columns from SAKAI_EVENT like 'CONTEXT'";
private final static String ORACLE_CHECK_FOR_CONTEXT = "select column_name from USER_TAB_COLUMNS where table_name='SAKAI_EVENT' and column_name='CONTEXT'";
private final static String MYSQL_CONTEXT_COLUMN = ",CONTEXT as CONTEXT";
private final static String ORACLE_CONTEXT_COLUMN = ",CONTEXT";
private String MYSQL_GET_EVENT = "select " + MYSQL_DEFAULT_COLUMNS + MYSQL_CONTEXT_COLUMN + " " +
"from SAKAI_EVENT e join SAKAI_SESSION s on e.SESSION_ID=s.SESSION_ID " +
"where EVENT_ID >= ? and EVENT_ID < ? " +
"order by EVENT_ID asc ";
private String ORACLE_GET_EVENT = "SELECT * FROM ( " +
"SELECT " +
" ROW_NUMBER() OVER (ORDER BY EVENT_ID ASC) AS rn, " +
ORACLE_DEFAULT_COLUMNS + ORACLE_CONTEXT_COLUMN + " " +
"from SAKAI_EVENT e join SAKAI_SESSION s on e.SESSION_ID=s.SESSION_ID " +
"where EVENT_ID >= ? " +
") " +
"WHERE rn BETWEEN ? AND ?";
private String MYSQL_PAST_SITE_EVENTS = "select " + MYSQL_DEFAULT_COLUMNS + MYSQL_CONTEXT_COLUMN + " " +
"from SAKAI_EVENT e join SAKAI_SESSION s on e.SESSION_ID=s.SESSION_ID " +
"where (CONTEXT = ? or (EVENT in ('pres.begin','pres.end') and REF = ?)) " +
"and EVENT_DATE >= ? and EVENT_DATE <= ?";
private String ORACLE_PAST_SITE_EVENTS = "SELECT " + ORACLE_DEFAULT_COLUMNS + ORACLE_CONTEXT_COLUMN + " " +
"from SAKAI_EVENT e join SAKAI_SESSION s on e.SESSION_ID=s.SESSION_ID " +
"where (CONTEXT = ? or (EVENT in ('pres.begin','pres.end') and REF = ?)) " +
"and EVENT_DATE >= ? and EVENT_DATE <= ?";
// Services
private StatsUpdateManager statsUpdateManager = null;
private SqlService sqlService = null;
public void init(){
doInitialCheck();
LOG.info("StatsAggregateJobImpl.init()");
}
private void doInitialCheck() {
JobRun lastJobRun = null;
try{
lastJobRun = getLastJobRun();
}catch(Exception e){
LOG.error("Make sure SST_JOB_RUN table is created before running the StatsAggregateJob job.");
}
if(lastJobRun == null && !statsUpdateManager.isCollectThreadEnabled()){
if(getStartEventId() < 0){
long lastEventIdInTable = 0;
try{
lastEventIdInTable = getLastEventIdInTable();
}catch(SQLException e){
LOG.warn("Unable to check last eventId in table SAKAI_EVENT --> assuming 0.", e);
}
LOG.warn("First StatsAggregateJob job run will use last SAKAI_EVENT.EVENT_ID (id = "+lastEventIdInTable+"). To override this, please specify a new eventId in sakai.properties (property: startEventId@org.sakaiproject.sitestats.api.StatsAggregateJob=n, where n>=0). This value is for the first job run only.");
}else{
LOG.warn("First StatsAggregateJob job run will use 'startEventId' ("+getStartEventId()+") specified in sakai.properties. This value is for the first job run only.");
}
}
}
// ################################################################
// Job related methods
// ################################################################
public void execute(JobExecutionContext context) throws JobExecutionException {
String result = null;
String jobName = context.getJobDetail().getFullName();
// ABORT if job is currently running in this cluster node.
// -> Required as StatefullJob is only correctly supported in trunk!
// WARNING: I cannot currently check if this is running in OTHER cluster nodes!!!
try{
while(isJobCurrentlyRunning(context)) {
String beanId = context.getJobDetail().getJobDataMap().getString(SpringJobBeanWrapper.SPRING_BEAN_NAME);
LOG.warn("Another instance of "+beanId+" is currently running - Execution aborted.");
return;
}
}catch(SchedulerException e){
LOG.error("Aborting job execution due to "+e.toString(), e);
return;
}
LOG.info("Starting job: " + jobName);
// check for SAKAI_EVENT.CONTEXT column
try{
checkForContextColumn();
LOG.debug("SAKAI_EVENT.CONTEXT exists? "+isEventContextSupported);
}catch(SQLException e1){
LOG.warn("Unable to check existence of SAKAI_EVENT.CONTEXT", e1);
}
// configure job
JobRun lastJobRun;
try{
lastJobRun = getLastJobRun();
}catch(Exception e){
LOG.error("Error accessing SST_JOB_RUN table. Does this table exists? Aborting job...");
return;
}
jobRun = new JobRunImpl();
jobRun.setJobStartDate(new Date(System.currentTimeMillis()));
if(lastJobRun != null){
jobRun.setStartEventId(lastJobRun.getEndEventId() + 1);
}else if(getStartEventId() >= 0){
LOG.info("First job run: using 'startEventId' ("+getStartEventId()+") specified in sakai.properties. This value is for the first job run only.");
jobRun.setStartEventId(getStartEventId());
}else{
long lastEventIdInTable = 0;
try{
lastEventIdInTable = getLastEventIdInTable();
}catch(SQLException e){
LOG.warn("Unable to check last eventId in table SAKAI_EVENT --> assuming 0.", e);
}
LOG.info("First job run: no 'startEventId' specified in sakai.properties; using last SAKAI_EVENT.EVENT_ID (id = "+lastEventIdInTable+"). This value is for the first job run only.");
jobRun.setStartEventId(lastEventIdInTable);
}
// start job
try{
result = startJob();
LOG.info("Summary: " + result);
}catch(SQLException e){
LOG.error("Summary: job run failed", e);
}
// finish
LOG.info("Finishing job: " + jobName);
}
private boolean isJobCurrentlyRunning(JobExecutionContext context) throws SchedulerException {
String beanId = context.getJobDetail().getJobDataMap().getString(SpringJobBeanWrapper.SPRING_BEAN_NAME);
List<JobExecutionContext> jobsRunning = context.getScheduler().getCurrentlyExecutingJobs();
int jobsCount = 0;
for(JobExecutionContext j : jobsRunning)
if(beanId.equals(j.getJobDetail().getJobDataMap().getString(SpringJobBeanWrapper.SPRING_BEAN_NAME))) {
jobsCount++;
}
if(jobsCount > 1)
return true;
return false;
}
private long getLastEventIdInTable() throws SQLException {
if(lastEventIdInTable == -1) {
Connection connection = null;
Statement st = null;
ResultSet rs = null;
try{
connection = getEventDbConnection();
st = connection.createStatement();
rs = st.executeQuery(LAST_EVENT_ID);
if(rs.next()){
lastEventIdInTable = rs.getLong("LAST_ID");
}
}catch(SQLException e){
LOG.error("Unable to retrieve events", e);
}finally{
try{
if(rs != null)
rs.close();
}finally{
try{
if(st != null)
st.close();
}finally{
closeEventDbConnection(connection);
}
}
}
}
return lastEventIdInTable;
}
private String startJob() throws SQLException {
List<Event> eventsQueue = new ArrayList<Event>();
long counter = 0;
long offset = 0;
long lastProcessedEventId = 0;
long lastProcessedEventIdWithSuccess = 0;
long firstEventIdProcessed = -1;
long firstEventIdProcessedInBlock = -1;
Date lastEventDate = null;
Date lastEventDateWithSuccess = null;
boolean abortIteration = false;
long start = System.currentTimeMillis();
boolean sqlError = false;
String returnMessage = null;
Connection connection = getEventDbConnection();
long eventIdLowerLimit = getEventIdLowerLimit();
PreparedStatement st = null;
ResultSet rs = null;
try{
st = connection.prepareStatement(sqlGetEvent);
rs = null;
while(!abortIteration) {
abortIteration = true;
st.clearParameters();
if(!isOracle){
if(firstEventIdProcessed == -1)
offset = eventIdLowerLimit;
st.setLong(1, offset); // MySQL >= startId
st.setLong(2, sqlBlockSize + offset); // MySQL < endId
}else{
st.setLong(1, eventIdLowerLimit); // Oracle lower limit
st.setLong(2, offset); // Oracle offset
st.setLong(3, sqlBlockSize + offset); // Oracle limit
}
rs = st.executeQuery();
while(rs.next()){
abortIteration = false;
Date date = null;
String event = null;
String ref = null;
String context = null;
String sessionUser = null;
String sessionId = null;
try{
//If an exception is launched, iteration is not aborted but no event is added to event queue
date = new Date(rs.getTimestamp("EVENT_DATE").getTime());
event = rs.getString("EVENT");
ref = rs.getString("REF");
sessionUser = rs.getString("SESSION_USER");
sessionId = rs.getString("SESSION_ID");
if(isEventContextSupported)
context = rs.getString("CONTEXT");
eventsQueue.add( statsUpdateManager.buildEvent(date, event, ref, context, sessionUser, sessionId) );
counter++;
lastProcessedEventId = rs.getInt("EVENT_ID");
lastEventDate = date;
if(firstEventIdProcessed == -1)
firstEventIdProcessed = jobRun.getStartEventId(); //was: lastProcessedEventId;
if(firstEventIdProcessedInBlock == -1)
firstEventIdProcessedInBlock = lastProcessedEventId;
}catch(Exception e){
if(LOG.isDebugEnabled())
LOG.debug("Ignoring "+event+", "+ref+", "+date+", "+sessionUser+", "+sessionId+" due to: "+e.toString());
}
}
rs.close();
if(!abortIteration){
// process events
boolean processedOk = statsUpdateManager.collectEvents(eventsQueue);
eventsQueue.clear();
if(processedOk){
lastProcessedEventIdWithSuccess = lastProcessedEventId;
lastEventDateWithSuccess = lastEventDate;
jobRun.setStartEventId(firstEventIdProcessed);
jobRun.setEndEventId(lastProcessedEventIdWithSuccess);
jobRun.setLastEventDate(lastEventDateWithSuccess);
jobRun.setJobEndDate(new Date(System.currentTimeMillis()));
saveJobRun(jobRun);
firstEventIdProcessedInBlock = -1;
if(counter >= getMaxEventsPerRun()){
abortIteration = true;
}else if(counter + sqlBlockSize < getMaxEventsPerRun()){
offset += sqlBlockSize;
}else{
offset += getMaxEventsPerRun() - counter;
}
}else{
returnMessage = "An error occurred while processing/persisting events to db. Please check your logs, fix possible problems and re-run this job (will start after last successful processed event).";
LOG.error(returnMessage);
throw new Exception(returnMessage);
}
}
}
}catch(SQLException e){
sqlError = true;
if(returnMessage == null) {
returnMessage = "Unable to retrieve events due to: " + e.getMessage();
LOG.error("Unable to retrieve events", e);
}
}catch(Exception e){
sqlError = true;
if(returnMessage == null) {
returnMessage = "Unable to retrieve events due to: " + e.getMessage();
LOG.error("Unable to retrieve events due to an unknown cause", e);
}
}finally{
try{
if(rs != null)
rs.close();
}finally{
try{
if(st != null)
st.close();
}finally{
closeEventDbConnection(connection);
}
}
}
// error occurred
if(sqlError) {
return returnMessage;
}
long processingTime = (System.currentTimeMillis() - start) / 1000;
if(firstEventIdProcessed == -1 && jobRun != null){
// no data was processed: do not persist to DB
// long eventId = jobRun.getEndEventId();
// firstEventIdProcessed = eventId;
// lastProcessedEventIdWithSuccess = eventId;
// jobRun.setEndEventId(lastProcessedEventId > 0 ? lastProcessedEventId : jobRun.getStartEventId());
// jobRun.setLastEventDate(lastEventDate != null ? lastEventDate : null);
// jobRun.setJobEndDate(new Date(System.currentTimeMillis()));
return "0 events processed in "+processingTime+"s (no entry will be added to SST_JOB_RUN; only events associated with a session are processed)";
}else{
saveJobRun(jobRun);
}
return counter + " events processed (ids: "+firstEventIdProcessed+" - "+lastProcessedEventIdWithSuccess+") in "+processingTime+"s (only events associated with a session are processed)";
}
private long getEventIdLowerLimit() {
long start = getStartEventId();
long nextEventId = jobRun.getStartEventId();
if(nextEventId > start)
start = nextEventId;
return start;
}
private JobRun getLastJobRun() throws Exception {
return statsUpdateManager.getLatestJobRun();
}
private boolean saveJobRun(JobRun jobRun) {
boolean ok = false;
try{
ok = statsUpdateManager.saveJobRun(jobRun);
}catch(Exception e){
LOG.error("Unable to persist last job information to db.", e);
}
return ok;
}
public long collectPastSiteEvents(String siteId, Date initialDate, Date finalDate) {
List<Event> eventsQueue = new ArrayList<Event>();
Connection connection = getEventDbConnection();
PreparedStatement st = null;
ResultSet rs = null;
long count = 0;
long opStart = System.currentTimeMillis();
try{
st = connection.prepareStatement(sqlPastSiteEvents);
st.setString(1, siteId); // CONTEXT = ?
st.setString(2, "/presence/"+siteId+"-presence"); // REF = ?
st.setDate(3, new java.sql.Date(initialDate.getTime())); // EVENT_DATE >= ?
st.setDate(4, new java.sql.Date(finalDate.getTime())); // EVENT_DATE <= ?
rs = st.executeQuery();
while(rs.next()){
Date date = null;
String event = null;
String ref = null;
String context = null;
String sessionUser = null;
String sessionId = null;
try{
//If an exception is launched, iteration is not aborted but no event is added to event queue
date = new Date(rs.getTimestamp("EVENT_DATE").getTime());
event = rs.getString("EVENT");
ref = rs.getString("REF");
sessionUser = rs.getString("SESSION_USER");
sessionId = rs.getString("SESSION_ID");
context = rs.getString("CONTEXT");
eventsQueue.add( statsUpdateManager.buildEvent(date, event, ref, context, sessionUser, sessionId) );
count++;
}catch(Exception e){
if(LOG.isDebugEnabled())
LOG.debug("Ignoring "+event+", "+ref+", "+date+", "+sessionUser+", "+sessionId+" due to: "+e.toString());
}
}
// process events
boolean processedOk = statsUpdateManager.collectEvents(eventsQueue);
eventsQueue.clear();
if(!processedOk){
String returnMessage = "An error occurred while processing/persisting events to db - please check your logs.";
LOG.error(returnMessage);
throw new Exception(returnMessage);
}
}catch(SQLException e){
LOG.error("Unable to collect past site events", e);
}catch(Exception e){
LOG.error("Unable to collect past site due to an unknown cause", e);
}finally{
try{
if(rs != null)
try{
rs.close();
}catch(SQLException e){ }
}finally{
try{
if(st != null)
try{
st.close();
}catch(SQLException e){ }
}finally{
closeEventDbConnection(connection);
}
}
}
long opEnd = System.currentTimeMillis();
LOG.info("Collected "+count+" past events for site "+siteId+" in "+(opEnd-opStart)/1000+" seconds.");
return count;
}
// ################################################################
// Util methods
// ################################################################
private Connection getEventDbConnection() {
Connection connection = null;
if(getUrl() == null){
// SAKAI_EVENT and SAKAI_SESSION are on the same database
try{
connection = sqlService.borrowConnection();
if(sqlService.getVendor().equals("oracle")){
isOracle = true;
if(isEventContextSupported)
sqlGetEvent = ORACLE_GET_EVENT;
else
sqlGetEvent = ORACLE_GET_EVENT.replaceAll(ORACLE_CONTEXT_COLUMN, "");
// this feature is only available for Sakai >= 2.6.x
sqlPastSiteEvents = ORACLE_PAST_SITE_EVENTS;
}else{
isOracle = false;
if(isEventContextSupported)
sqlGetEvent = MYSQL_GET_EVENT;
else
sqlGetEvent = MYSQL_GET_EVENT.replaceAll(MYSQL_CONTEXT_COLUMN, "");
// this feature is only available for Sakai >= 2.6.x
sqlPastSiteEvents = MYSQL_PAST_SITE_EVENTS;
}
}catch(SQLException e){
LOG.error("Unable to connect Sakai Db", e);
return null;
}catch(Exception e){
LOG.error("Unable to connect to Sakai Db", e);
return null;
}
}else{
// SAKAI_EVENT and SAKAI_SESSION are on different database
try{
if(extDbdriver == null){
extDbdriver = Class.forName(getDriverClassName()).newInstance();
if(getDriverClassName().equals("oracle.jdbc.driver.OracleDriver")){
isOracle = true;
if(isEventContextSupported)
sqlGetEvent = ORACLE_GET_EVENT;
else
sqlGetEvent = ORACLE_GET_EVENT.replaceAll(ORACLE_CONTEXT_COLUMN, "");
}else{
isOracle = false;
if(isEventContextSupported)
sqlGetEvent = MYSQL_GET_EVENT;
else
sqlGetEvent = MYSQL_GET_EVENT.replaceAll(MYSQL_CONTEXT_COLUMN, "");
}
}
connection = DriverManager.getConnection(getUrl(), getUsername(), getPassword());
}catch(SQLException e){
LOG.error("Unable to connect to " + getUrl(), e);
return null;
}catch(Exception e){
LOG.error("Unable to connect to " + getUrl(), e);
return null;
}
}
return connection;
}
private void closeEventDbConnection(Connection connection) {
if(getUrl() == null){
if(connection != null){
sqlService.returnConnection(connection);
}
}else{
try{
if(connection != null && !connection.isClosed()){
connection.close();
}
}catch(SQLException e){
LOG.error("Unable to close connection " + getUrl(), e);
}
}
}
private void checkForContextColumn() throws SQLException {
// Check for SAKAI_EVENT.CONTEXT table column
Connection connection = null;
PreparedStatement st = null;
ResultSet rs = null;
String sqlCheckForContext = null;
try{
connection = getEventDbConnection();
if(isOracle)
sqlCheckForContext = ORACLE_CHECK_FOR_CONTEXT;
else
sqlCheckForContext = MYSQL_CHECK_FOR_CONTEXT;
st = connection.prepareStatement(sqlCheckForContext);
rs = st.executeQuery();
if(rs.next()){
LOG.debug("SAKAI_EVENT.CONTEXT IS present.");
isEventContextSupported = true;
} else {
LOG.debug("SAKAI_EVENT.CONTEXT is NOT present.");
isEventContextSupported = false;
}
}catch(SQLException e){
LOG.error("Unable to determine if SAKAI_EVENT.CONTEXT is present", e);
isEventContextSupported = false;
}finally{
try{
if(rs != null)
rs.close();
}finally{
try{
if(st != null)
st.close();
}finally{
closeEventDbConnection(connection);
}
}
}
}
// ################################################################
// Spring related methods
// ################################################################
public int getMaxEventsPerRun() {
return maxEventsPerRun;
}
public void setMaxEventsPerRun(int maxEventsPerRun) {
this.maxEventsPerRun = maxEventsPerRun;
}
public int getSqlBlockSize() {
return sqlBlockSize;
}
public void setSqlBlockSize(int sqlBlockSize) {
this.sqlBlockSize = sqlBlockSize;
}
public long getStartEventId() {
return startEventId;
}
public void setStartEventId(long startEventId) {
this.startEventId = startEventId;
}
public String getDriverClassName() {
return driverClassName;
}
public void setDriverClassName(String driverClassName) {
this.driverClassName = driverClassName;
}
public String getUrl() {
return url;
}
public void setUrl(String url) {
this.url = url;
}
public String getUsername() {
return username;
}
public void setUsername(String username) {
this.username = username;
}
public String getPassword() {
return password;
}
public void setPassword(String password) {
this.password = password;
}
public StatsUpdateManager getStatsUpdateManager() {
return statsUpdateManager;
}
public void setStatsUpdateManager(StatsUpdateManager statsUpdateManager) {
this.statsUpdateManager = statsUpdateManager;
}
public SqlService getSqlService() {
return sqlService;
}
public void setSqlService(SqlService sqlService) {
this.sqlService = sqlService;
}
}