/*
* Seldon -- open source prediction engine
* =======================================
*
* Copyright 2011-2015 Seldon Technologies Ltd and Rummble Ltd (http://www.seldon.io/)
*
* ********************************************************************************************
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
* ********************************************************************************************
*/
package io.seldon.api.service.async;
import io.seldon.api.Constants;
import io.seldon.db.jdbc.JDBCConnectionFactory;
import io.seldon.general.Action;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import org.apache.log4j.Logger;
/**
* Provide an batched insert of Actions (including creating new users and items).
* Uses a LinkedBlockingQueue in which Action objects are added. A thread runs reading actions off this
* queue and adding them to the db using SQL batch statements. To run fast needs the Mysql extension rewriteBatchedStatements=true.
*
* <ul>
* <li> new users are identified by having user_id 0
* <li> new items are identified by having item_id 0
* <li> basic validation checks carried out on client_user_id and client_item_id
* <li> All exceptions are caught in an effort to never allow the thread to die
* <li> batch size can be configured
* <li> max wait timeout can be configured
* <li> The SQL will be run if a) batch size is exceeeded, b) the max time between sql updates is exceeded
* </ul>
* @author rummble
*
*/
public class JdoAsyncActionQueue implements Runnable, AsyncActionQueue {
private static Logger logger = Logger.getLogger(JdoAsyncActionQueue.class.getName());
public static final int MAX_CLIENT_USER_ID_LEN = 254;
public static final int MAX_CLIENT_ITEM_ID_LEN = 254;
private String client;
private int timeout;
private LinkedBlockingQueue<Action> queue;
private int batchSize; // batch size for sql statements
private int maxDBRetries = 1; // max # of times to try sql statement on exception
boolean keepRunning;
Connection connection = null;
PreparedStatement actionPreparedStatement;
private int actionsAdded = 0; // actions added so far to sql statement
private int actionsProcessed = 0; // number of actions processed
PreparedStatement userPreparedStatement;
private int usersAdded = 0; // users added so far to sql statement
boolean updateUsers = false;
PreparedStatement itemPreparedStatement;
private int itemsAdded = 0; // users added so far to sql statement
boolean updateItems = false;
boolean runUserItemUpdates = true;
boolean runUpdateIdsInActionTable = true;
long lastSqlRunTime = 0;
int badActions = 0;
boolean insertActions = true;
public JdoAsyncActionQueue(String client, int qTimeoutSecs, int batchSize, int maxQSize,int maxDBRetries,boolean runUserItemUpdates,boolean runUpdateIdsInActionTable,boolean insertActions) {
this.client = client;
this.batchSize = batchSize;
this.maxDBRetries = maxDBRetries;
this.queue = new LinkedBlockingQueue<>(maxQSize);
this.timeout = qTimeoutSecs;
this.runUserItemUpdates = runUserItemUpdates;
this.runUpdateIdsInActionTable = runUpdateIdsInActionTable;
this.insertActions = insertActions;
logger.info("AsyncQ created for client "+client+" qTimeout:"+qTimeoutSecs+" batchSize:"+batchSize+" maxQSize:"+maxQSize+" maxDBRetries:"+maxDBRetries+" userItemUpdates:"+runUserItemUpdates+" rumActionIdUpdates:"+this.runUpdateIdsInActionTable+" insertActions:"+this.insertActions);
}
public void run() {
keepRunning = true;
this.lastSqlRunTime = System.currentTimeMillis();
while (true)
{
try
{
Action action = queue.poll(timeout, TimeUnit.SECONDS);
if (action != null)
addSQL(action);
long timeSinceLastSQLRun = (System.currentTimeMillis() - this.lastSqlRunTime)/1000;
boolean runSQL = false;
if ((action == null && actionsProcessed > 0))
{
runSQL = true;
logger.info("Run sql as timeout on poll and actionsProcessed > 0");
}
else if (actionsProcessed >= batchSize)
{
runSQL = true;
logger.info("Run sql as batch size exceeded");
}
else if (timeSinceLastSQLRun > timeout && actionsProcessed > 0)
{
runSQL = true;
logger.info("Run sql as time between sql runs exceeded");
}
if (runSQL)
runSQL();
if (!keepRunning && action == null)
{
logger.warn("Asked to stop as keepRunning is false");
return;
}
}
catch (InterruptedException e) {
logger.error("Received interrupted exception - will stop",e);
return;
}
catch (Exception e)
{
logger.error("Caught exception while running ", e);
resetState();
logger.warn("\\-> Reset buffers.");
}
catch (Throwable t)
{
logger.error("Caught throwable while running ", t);
resetState();
logger.warn("\\-> Reset buffers.");
}
}
}
private void resetState() {
clearSQLState();
actionsAdded = 0;
actionsProcessed = 0;
itemsAdded = 0;
usersAdded = 0;
updateUsers = false;
updateItems = false;
this.lastSqlRunTime = System.currentTimeMillis();
}
private void clearSQLState()
{
try
{
if (connection != null)
{
try{connection.close();}
catch( SQLException exception )
{
logger.error("Unable to close connection",exception);
}
}
if (actionPreparedStatement != null)
{
try{actionPreparedStatement.close();}
catch( SQLException exception )
{
logger.error("Unable to close action perpared statment",exception);
}
}
if (userPreparedStatement != null)
{
try{userPreparedStatement.close();}
catch( SQLException exception )
{
logger.error("Unable to close user perpared statment",exception);
}
}
if (itemPreparedStatement != null)
{
try{itemPreparedStatement.close();}
catch( SQLException exception )
{
logger.error("Unable to close item perpared statment",exception);
}
}
}
finally
{
connection = null;
actionPreparedStatement = null;
userPreparedStatement = null;
itemPreparedStatement = null;
}
}
private void updateUsersIfNeeded() throws SQLException
{
if (updateUsers && insertActions) //update user ids - if we have added new users or added users with 0 user_id
{
logger.info("Updating users");
PreparedStatement s = connection.prepareStatement("update actions a join users u on a.client_user_id=u.client_user_id set a.user_id=u.user_id where a.user_id=0");
try
{
s.executeUpdate();
connection.commit();
updateUsers = false;
}
finally
{
if (s!= null)
s.close();
}
}
}
private void updateItemsIfNeeded() throws SQLException
{
if (updateItems && insertActions) //update user ids - if we have added new users or added users with 0 user_id
{
logger.info("Updating items");
PreparedStatement s = connection.prepareStatement("update actions a join items i on a.client_item_id=i.client_item_id set a.item_id=i.item_id where a.item_id=0");
try
{
s.executeUpdate();
connection.commit();
updateItems = false;
}
finally
{
if (s!= null)
s.close();
}
}
}
private void executeBatch() throws SQLException
{
if (actionsProcessed > 0)
{
if (actionsAdded > 0)
{
actionPreparedStatement.executeBatch();
}
if (usersAdded > 0)
{
userPreparedStatement.executeBatch();
}
if (itemsAdded > 0)
{
itemPreparedStatement.executeBatch();
}
connection.commit();
if (actionPreparedStatement != null)
actionPreparedStatement.close();
if (usersAdded > 0)
{
usersAdded = 0;
userPreparedStatement.close();
}
if (itemsAdded > 0)
{
itemsAdded = 0;
itemPreparedStatement.close();
}
actionsAdded = 0;
actionsProcessed = 0;
}
}
private void rollBack()
{
try
{
connection.rollback();
}
catch( SQLException re )
{
logger.error("Can't roll back transaction",re);
}
}
private void runSQL()
{
long t1 = System.currentTimeMillis();
int localActionsAdded = this.actionsAdded;
// batch update actions
boolean success = false;
for (int i = 0; i < this.maxDBRetries && !success; i++)
{
try
{
executeBatch();
success = true;
break;
}
catch (SQLException e) {
logger.error("Failed to run batch update ",e);
rollBack();
}
}
if (success)
{
// Update users if needed
success = false;
for (int i = 0; i < this.maxDBRetries && !success; i++)
{
try
{
updateUsersIfNeeded();
success = true;
break;
}
catch (SQLException e) {
logger.error("Failed to run user update ",e);
rollBack();
}
}
//Update items if needed
success = false;
for (int i = 0; i < this.maxDBRetries && !success; i++)
{
try
{
updateItemsIfNeeded();
success = true;
break;
}
catch (SQLException e) {
logger.error("Failed to run item update ",e);
rollBack();
}
}
}
else {
// logger.error("Failed to add batch actions so not running user/item update");
final String message = "Failed to add batch actions so not running user/item update";
logger.error(message, new Exception(message));
}
resetState();
long t2 = System.currentTimeMillis();
//log q size
logger.info("AsynAction Q for "+client+" at size:"+queue.size()+" actions added "+localActionsAdded+" time to process:"+(t2-t1));
}
/**
* Allowed operations to fill in nulls in Action
* @param action
*/
private void repairAction(Action action)
{
if (action.getTimes() == null)
action.setTimes(1);
}
/**
* Check the values of an action to ensure we don't try to manipulate bad data
* @param action
* @return
*/
private boolean checkActionOK(Action action)
{
repairAction(action);
if (action.getType() == null
// || action.getTimes() == null
// || action.getValue() == null
|| action.getClientUserId() == null
|| action.getClientItemId() == null
|| action.getClientUserId().length() > MAX_CLIENT_USER_ID_LEN
|| action.getClientItemId().length() > MAX_CLIENT_ITEM_ID_LEN
)
{
badActions++;
return false;
}
else
return true;
}
private void getConnectionIfNeeded() throws SQLException
{
if (connection == null)
{
connection = JDBCConnectionFactory.get().getConnection(client);
connection.setAutoCommit( false );
}
}
private void addUserBatch(Action action) throws SQLException
{
userPreparedStatement.setString(1, ""); // Username hardwired to empty string
if (action.getDate() != null)
userPreparedStatement.setTimestamp(2, new java.sql.Timestamp(action.getDate().getTime()));
else
userPreparedStatement.setNull(2, java.sql.Types.TIMESTAMP);
if (action.getDate() != null)
userPreparedStatement.setTimestamp(3, new java.sql.Timestamp(action.getDate().getTime()));
else
userPreparedStatement.setNull(3, java.sql.Types.TIMESTAMP);
userPreparedStatement.setInt(4, Constants.DEFAULT_USER_TYPE); // hardwired user type
userPreparedStatement.setString(5, action.getClientUserId());
}
private void addItemBatch(Action action) throws SQLException
{
itemPreparedStatement.setString(1, ""); // item names hardwired to empty string
if (action.getDate() != null)
itemPreparedStatement.setTimestamp(2, new java.sql.Timestamp(action.getDate().getTime()));
else
itemPreparedStatement.setNull(2, java.sql.Types.TIMESTAMP);
if (action.getDate() != null)
itemPreparedStatement.setTimestamp(3, new java.sql.Timestamp(action.getDate().getTime()));
else
itemPreparedStatement.setNull(3, java.sql.Types.TIMESTAMP);
itemPreparedStatement.setString(4, action.getClientItemId());
}
private void addActionBatch(Action action) throws SQLException
{
actionPreparedStatement.setLong(1, action.getUserId());
actionPreparedStatement.setLong(2, action.getItemId());
if (action.getType() != null)
actionPreparedStatement.setInt(3, action.getType());
else
actionPreparedStatement.setNull(3, java.sql.Types.INTEGER);
if (action.getTimes() != null)
actionPreparedStatement.setInt(4, action.getTimes());
else
actionPreparedStatement.setNull(4, java.sql.Types.INTEGER);
if (action.getDate() != null)
actionPreparedStatement.setTimestamp(5, new java.sql.Timestamp(action.getDate().getTime()));
else
actionPreparedStatement.setNull(5, java.sql.Types.TIMESTAMP);
if (action.getValue() != null)
actionPreparedStatement.setDouble(6, action.getValue());
else
actionPreparedStatement.setNull(6, java.sql.Types.DOUBLE);
actionPreparedStatement.setString(7, action.getClientUserId());
actionPreparedStatement.setString(8, action.getClientItemId());
}
private void addSQL(Action action) throws SQLException {
if (!checkActionOK(action))
{
logger.warn("Bad Action "+action.toString());
return;
}
else
{
getConnectionIfNeeded();
if (this.insertActions)
{
// Add action batch
if (actionPreparedStatement == null)
actionPreparedStatement = connection.prepareStatement("insert into actions (action_id,user_id,item_id,type,times,date,value,client_user_id,client_item_id) values (0,?,?,?,?,?,?,?,?)");
addActionBatch(action);
actionPreparedStatement.addBatch();
actionsAdded++;
}
if (runUserItemUpdates)
{
if (action.getUserId() == 0)
{
if (isRunUpdateIdsInActionTable())
updateUsers = true;
if (userPreparedStatement == null)
userPreparedStatement = connection.prepareStatement("insert ignore into users (user_id,username,first_op,last_op,type,num_op,active,client_user_id,avgrating,stddevrating) values (0,?,?,?,?,1,1,?,0,0)");
addUserBatch(action);
userPreparedStatement.addBatch();
usersAdded++;
}
if (action.getItemId() == 0)
{
if (isRunUpdateIdsInActionTable())
updateItems = true;
if (itemPreparedStatement == null)
itemPreparedStatement = connection.prepareStatement("insert ignore into items (item_id,name,first_op,last_op,popular,client_item_id,type,avgrating,stddevrating,num_op) values (0,?,?,?,0,?,0,0,0,0)");
addItemBatch(action);
itemPreparedStatement.addBatch();
itemsAdded++;
}
}
actionsProcessed++;
}
}
public void put(Action action) {
queue.add(action);
}
public String getClient() {
return client;
}
public void setClient(String client) {
this.client = client;
}
public int getTimeout() {
return timeout;
}
public void setTimeout(int timeout) {
this.timeout = timeout;
}
public int getActionsAdded() {
return actionsAdded;
}
public int getActionsProcessed() {
return actionsProcessed;
}
public int getBatchSize() {
return batchSize;
}
public void setBatchSize(int batchSize) {
this.batchSize = batchSize;
}
public int getMaxDBRetries() {
return maxDBRetries;
}
public void setMaxDBRetries(int maxDBRetries) {
this.maxDBRetries = maxDBRetries;
}
public boolean isKeepRunning() {
return keepRunning;
}
public void setKeepRunning(boolean keepRunning) {
this.keepRunning = keepRunning;
}
public int getBadActions() {
return badActions;
}
public boolean isInsertActions() {
return insertActions;
}
public synchronized boolean isRunUpdateIdsInActionTable() {
return runUpdateIdsInActionTable;
}
public synchronized void setRunUpdateIdsInActionTable(
boolean runUpdateIdsInActionTable) {
this.runUpdateIdsInActionTable = runUpdateIdsInActionTable;
}
}