package com.linkedin.thirdeye.datalayer.dao; import com.linkedin.thirdeye.anomaly.utils.ThirdeyeMetricsUtil; import com.linkedin.thirdeye.datalayer.entity.AlertConfigIndex; import com.linkedin.thirdeye.datalayer.entity.ClassificationConfigIndex; import com.linkedin.thirdeye.datalayer.entity.EventIndex; import com.linkedin.thirdeye.datalayer.entity.AutotuneConfigIndex; import com.linkedin.thirdeye.datalayer.entity.GroupedAnomalyResultsIndex; import com.linkedin.thirdeye.datalayer.entity.OverrideConfigIndex; import com.linkedin.thirdeye.datalayer.pojo.AlertConfigBean; import com.linkedin.thirdeye.datalayer.pojo.ClassificationConfigBean; import com.linkedin.thirdeye.datalayer.pojo.EventBean; import com.linkedin.thirdeye.datalayer.pojo.AutotuneConfigBean; import com.linkedin.thirdeye.datalayer.pojo.GroupedAnomalyResultsBean; import com.linkedin.thirdeye.datalayer.pojo.OverrideConfigBean; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Timestamp; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; import javax.sql.DataSource; import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.commons.collections.CollectionUtils; import org.modelmapper.ModelMapper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.collect.Sets; import com.google.inject.Inject; import com.linkedin.thirdeye.datalayer.entity.AbstractEntity; import com.linkedin.thirdeye.datalayer.entity.AbstractIndexEntity; import com.linkedin.thirdeye.datalayer.entity.AbstractJsonEntity; import com.linkedin.thirdeye.datalayer.entity.AnomalyFeedbackIndex; import com.linkedin.thirdeye.datalayer.entity.AnomalyFunctionIndex; import com.linkedin.thirdeye.datalayer.entity.DashboardConfigIndex; import com.linkedin.thirdeye.datalayer.entity.DataCompletenessConfigIndex; import com.linkedin.thirdeye.datalayer.entity.DatasetConfigIndex; import com.linkedin.thirdeye.datalayer.entity.DetectionStatusIndex; import com.linkedin.thirdeye.datalayer.entity.EmailConfigurationIndex; import com.linkedin.thirdeye.datalayer.entity.EntityToEntityMappingIndex; import com.linkedin.thirdeye.datalayer.entity.GenericJsonEntity; import com.linkedin.thirdeye.datalayer.entity.IngraphDashboardConfigIndex; import com.linkedin.thirdeye.datalayer.entity.IngraphMetricConfigIndex; import com.linkedin.thirdeye.datalayer.entity.JobIndex; import com.linkedin.thirdeye.datalayer.entity.MergedAnomalyResultIndex; import com.linkedin.thirdeye.datalayer.entity.MetricConfigIndex; import com.linkedin.thirdeye.datalayer.entity.RawAnomalyResultIndex; import com.linkedin.thirdeye.datalayer.entity.TaskIndex; import com.linkedin.thirdeye.datalayer.pojo.AbstractBean; import com.linkedin.thirdeye.datalayer.pojo.AnomalyFeedbackBean; import com.linkedin.thirdeye.datalayer.pojo.AnomalyFunctionBean; import com.linkedin.thirdeye.datalayer.pojo.DashboardConfigBean; import com.linkedin.thirdeye.datalayer.pojo.DataCompletenessConfigBean; import com.linkedin.thirdeye.datalayer.pojo.DatasetConfigBean; import com.linkedin.thirdeye.datalayer.pojo.DetectionStatusBean; import com.linkedin.thirdeye.datalayer.pojo.EmailConfigurationBean; import com.linkedin.thirdeye.datalayer.pojo.EntityToEntityMappingBean; import com.linkedin.thirdeye.datalayer.pojo.IngraphDashboardConfigBean; import com.linkedin.thirdeye.datalayer.pojo.IngraphMetricConfigBean; import com.linkedin.thirdeye.datalayer.pojo.JobBean; import com.linkedin.thirdeye.datalayer.pojo.MergedAnomalyResultBean; import com.linkedin.thirdeye.datalayer.pojo.MetricConfigBean; import com.linkedin.thirdeye.datalayer.pojo.RawAnomalyResultBean; import com.linkedin.thirdeye.datalayer.pojo.TaskBean; import com.linkedin.thirdeye.datalayer.util.GenericResultSetMapper; import com.linkedin.thirdeye.datalayer.util.Predicate; import com.linkedin.thirdeye.datalayer.util.SqlQueryBuilder; public class GenericPojoDao { private static final Logger LOG = LoggerFactory.getLogger(GenericPojoDao.class); private static boolean IS_DEBUG = LOG.isDebugEnabled(); static Map<Class<? extends AbstractBean>, PojoInfo> pojoInfoMap = new HashMap<Class<? extends AbstractBean>, GenericPojoDao.PojoInfo>(); static String DEFAULT_BASE_TABLE_NAME = "GENERIC_JSON_ENTITY"; static { pojoInfoMap.put(AnomalyFeedbackBean.class, newPojoInfo(DEFAULT_BASE_TABLE_NAME, AnomalyFeedbackIndex.class)); pojoInfoMap.put(AnomalyFunctionBean.class, newPojoInfo(DEFAULT_BASE_TABLE_NAME, AnomalyFunctionIndex.class)); pojoInfoMap.put(EmailConfigurationBean.class, newPojoInfo(DEFAULT_BASE_TABLE_NAME, EmailConfigurationIndex.class)); pojoInfoMap.put(JobBean.class, newPojoInfo(DEFAULT_BASE_TABLE_NAME, JobIndex.class)); pojoInfoMap.put(TaskBean.class, newPojoInfo(DEFAULT_BASE_TABLE_NAME, TaskIndex.class)); pojoInfoMap.put(MergedAnomalyResultBean.class, newPojoInfo(DEFAULT_BASE_TABLE_NAME, MergedAnomalyResultIndex.class)); pojoInfoMap.put(RawAnomalyResultBean.class, newPojoInfo(DEFAULT_BASE_TABLE_NAME, RawAnomalyResultIndex.class)); pojoInfoMap.put(DatasetConfigBean.class, newPojoInfo(DEFAULT_BASE_TABLE_NAME, DatasetConfigIndex.class)); pojoInfoMap.put(MetricConfigBean.class, newPojoInfo(DEFAULT_BASE_TABLE_NAME, MetricConfigIndex.class)); pojoInfoMap.put(DashboardConfigBean.class, newPojoInfo(DEFAULT_BASE_TABLE_NAME, DashboardConfigIndex.class)); pojoInfoMap.put(IngraphDashboardConfigBean.class, newPojoInfo(DEFAULT_BASE_TABLE_NAME, IngraphDashboardConfigIndex.class)); pojoInfoMap.put(IngraphMetricConfigBean.class, newPojoInfo(DEFAULT_BASE_TABLE_NAME, IngraphMetricConfigIndex.class)); pojoInfoMap.put(OverrideConfigBean.class, newPojoInfo(DEFAULT_BASE_TABLE_NAME, OverrideConfigIndex.class)); pojoInfoMap.put(EventBean.class, newPojoInfo(DEFAULT_BASE_TABLE_NAME, EventIndex.class)); pojoInfoMap.put(AlertConfigBean.class, newPojoInfo(DEFAULT_BASE_TABLE_NAME, AlertConfigIndex.class)); pojoInfoMap.put(DataCompletenessConfigBean.class, newPojoInfo(DEFAULT_BASE_TABLE_NAME, DataCompletenessConfigIndex.class)); pojoInfoMap.put(DetectionStatusBean.class, newPojoInfo(DEFAULT_BASE_TABLE_NAME, DetectionStatusIndex.class)); pojoInfoMap.put(AutotuneConfigBean.class, newPojoInfo(DEFAULT_BASE_TABLE_NAME, AutotuneConfigIndex.class)); pojoInfoMap.put(ClassificationConfigBean.class, newPojoInfo(DEFAULT_BASE_TABLE_NAME, ClassificationConfigIndex.class)); pojoInfoMap.put(EntityToEntityMappingBean.class, newPojoInfo(DEFAULT_BASE_TABLE_NAME, EntityToEntityMappingIndex.class)); pojoInfoMap.put(GroupedAnomalyResultsBean.class, newPojoInfo(DEFAULT_BASE_TABLE_NAME, GroupedAnomalyResultsIndex.class)); } private static PojoInfo newPojoInfo(String baseTableName, Class<? extends AbstractIndexEntity> indexEntityClass) { PojoInfo pojoInfo = new PojoInfo(); pojoInfo.baseTableName = baseTableName; pojoInfo.indexEntityClass = indexEntityClass; return pojoInfo; } @Inject DataSource dataSource; @Inject SqlQueryBuilder sqlQueryBuilder; @Inject GenericResultSetMapper genericResultSetMapper; static ModelMapper MODEL_MAPPER = new ModelMapper(); static ObjectMapper OBJECT_MAPPER = new ObjectMapper(); public GenericPojoDao() { } /** * Use at your own risk!!! Ensure to close the connection after using it or it can cause a leak. * * @return * * @throws SQLException */ public Connection getConnection() throws SQLException { ThirdeyeMetricsUtil.dbCallCounter.inc(); // ensure to close the connection return dataSource.getConnection(); } public <E extends AbstractBean> Long put(final E pojo) { //insert into its base table //get the generated id //update indexes return runTask(new QueryTask<Long>() { @Override public Long handle(Connection connection) throws Exception { PojoInfo pojoInfo = pojoInfoMap.get(pojo.getClass()); AbstractJsonEntity genericJsonEntity = new GenericJsonEntity(); genericJsonEntity.setCreateTime(new Timestamp(System.currentTimeMillis())); genericJsonEntity.setUpdateTime(new Timestamp(System.currentTimeMillis())); genericJsonEntity.setVersion(1); genericJsonEntity.setBeanClass(pojo.getClass().getName()); String jsonVal = OBJECT_MAPPER.writeValueAsString(pojo); genericJsonEntity.setJsonVal(jsonVal); try (PreparedStatement baseTableInsertStmt = sqlQueryBuilder.createInsertStatement(connection, genericJsonEntity)) { int affectedRows = baseTableInsertStmt.executeUpdate(); if (affectedRows == 1) { try (ResultSet generatedKeys = baseTableInsertStmt.getGeneratedKeys()) { if (generatedKeys.next()) { pojo.setId(generatedKeys.getLong(1)); } } if (pojoInfo.indexEntityClass != null) { AbstractIndexEntity abstractIndexEntity = pojoInfo.indexEntityClass.newInstance(); MODEL_MAPPER.map(pojo, abstractIndexEntity); abstractIndexEntity.setBaseId(pojo.getId()); abstractIndexEntity.setCreateTime(new Timestamp(System.currentTimeMillis())); abstractIndexEntity.setUpdateTime(new Timestamp(System.currentTimeMillis())); abstractIndexEntity.setVersion(1); int numRowsCreated; try (PreparedStatement indexTableInsertStatement = sqlQueryBuilder.createInsertStatement(connection, abstractIndexEntity)) { numRowsCreated = indexTableInsertStatement.executeUpdate(); } if (numRowsCreated == 1) { return pojo.getId(); } } else { return pojo.getId(); } } } return null; } }, null); } public <E extends AbstractBean> int update(E pojo) { return update(pojo, Predicate.EQ("id", pojo.getId())); } public <E extends AbstractBean> int update(final E pojo, final Predicate predicate) { //update base table //update indexes return runTask(new QueryTask<Integer>() { @Override public Integer handle(Connection connection) throws Exception { PojoInfo pojoInfo = pojoInfoMap.get(pojo.getClass()); String jsonVal = OBJECT_MAPPER.writeValueAsString(pojo); AbstractJsonEntity genericJsonEntity = new GenericJsonEntity(); genericJsonEntity.setUpdateTime(new Timestamp(System.currentTimeMillis())); genericJsonEntity.setJsonVal(jsonVal); genericJsonEntity.setId(pojo.getId()); genericJsonEntity.setVersion(pojo.getVersion()); dumpTable(connection, GenericJsonEntity.class); Set<String> fieldsToUpdate = Sets.newHashSet("jsonVal", "updateTime", "version"); int affectedRows; try (PreparedStatement baseTableInsertStmt = sqlQueryBuilder.createUpdateStatement(connection, genericJsonEntity, fieldsToUpdate, predicate)) { affectedRows = baseTableInsertStmt.executeUpdate(); } if (affectedRows == 1) { if (pojoInfo.indexEntityClass != null) { AbstractIndexEntity abstractIndexEntity = pojoInfo.indexEntityClass.newInstance(); MODEL_MAPPER.map(pojo, abstractIndexEntity); abstractIndexEntity.setBaseId(pojo.getId()); abstractIndexEntity.setUpdateTime(new Timestamp(System.currentTimeMillis())); //updates all columns in the index table by default try (PreparedStatement indexTableInsertStatement = sqlQueryBuilder.createUpdateStatementForIndexTable(connection, abstractIndexEntity)) { int numRowsUpdated = indexTableInsertStatement.executeUpdate(); LOG.debug("numRowsUpdated: {}", numRowsUpdated); return numRowsUpdated; } } } return affectedRows; } }, 0); } public <E extends AbstractBean> List<E> getAll(final Class<E> beanClass) { return runTask(new QueryTask<List<E>>() { @Override public List<E> handle(Connection connection) throws Exception { Predicate predicate = Predicate.EQ("beanClass", beanClass.getName()); List<GenericJsonEntity> entities; try (PreparedStatement selectStatement = sqlQueryBuilder.createFindByParamsStatement(connection, GenericJsonEntity.class, predicate)) { try (ResultSet resultSet = selectStatement.executeQuery()) { entities = genericResultSetMapper.mapAll(resultSet, GenericJsonEntity.class); } } List<E> ret = new ArrayList<>(); if (CollectionUtils.isNotEmpty(entities)) { for (GenericJsonEntity entity : entities) { E e = OBJECT_MAPPER.readValue(entity.getJsonVal(), beanClass); e.setId(entity.getId()); ret.add(e); } } return ret; } }, Collections.<E>emptyList()); } public <E extends AbstractBean> E get(final Long id, final Class<E> pojoClass) { return runTask(new QueryTask<E>() { @Override public E handle(Connection connection) throws Exception { GenericJsonEntity genericJsonEntity; try (PreparedStatement selectStatement = sqlQueryBuilder.createFindByIdStatement(connection, GenericJsonEntity.class, id)) { try (ResultSet resultSet = selectStatement.executeQuery()) { genericJsonEntity = genericResultSetMapper.mapSingle(resultSet, GenericJsonEntity.class); } } E e = null; if (genericJsonEntity != null) { e = OBJECT_MAPPER.readValue(genericJsonEntity.getJsonVal(), pojoClass); e.setId(genericJsonEntity.getId()); e.setVersion(genericJsonEntity.getVersion()); } return e; } }, null); } public <E extends AbstractBean> List<E> get(final List<Long> idList, final Class<E> pojoClass) { return runTask(new QueryTask<List<E>>() { @Override public List<E> handle(Connection connection) throws Exception { List<GenericJsonEntity> genericJsonEntities; try (PreparedStatement selectStatement = sqlQueryBuilder.createFindByIdStatement(connection, GenericJsonEntity.class, idList)) { try (ResultSet resultSet = selectStatement.executeQuery()) { genericJsonEntities = genericResultSetMapper.mapAll(resultSet, GenericJsonEntity.class); } } List<E> result = new ArrayList<>(); if (CollectionUtils.isNotEmpty(genericJsonEntities)) { for (GenericJsonEntity genericJsonEntity : genericJsonEntities) { E e = OBJECT_MAPPER.readValue(genericJsonEntity.getJsonVal(), pojoClass); e.setId(genericJsonEntity.getId()); e.setVersion(genericJsonEntity.getVersion()); result.add(e); } } return result; } }, Collections.<E>emptyList()); } /** * * @param parameterizedSQL second part of the sql (omit select from table section) * @param parameterMap * @return */ public <E extends AbstractBean> List<E> executeParameterizedSQL(final String parameterizedSQL, final Map<String, Object> parameterMap, final Class<E> pojoClass) { return runTask(new QueryTask<List<E>>() { @Override public List<E> handle(Connection connection) throws Exception { PojoInfo pojoInfo = pojoInfoMap.get(pojoClass); dumpTable(connection, pojoInfo.indexEntityClass); List<? extends AbstractIndexEntity> indexEntities; try (PreparedStatement findMatchingIdsStatement = sqlQueryBuilder.createStatementFromSQL( connection, parameterizedSQL, parameterMap, pojoInfo.indexEntityClass)) { try (ResultSet rs = findMatchingIdsStatement.executeQuery()) { indexEntities = genericResultSetMapper.mapAll(rs, pojoInfo.indexEntityClass); } } List<Long> idsToFind = new ArrayList<>(); if (CollectionUtils.isNotEmpty(indexEntities)) { for (AbstractIndexEntity entity : indexEntities) { idsToFind.add(entity.getBaseId()); } } List<E> ret = new ArrayList<>(); //fetch the entities if (!idsToFind.isEmpty()) { List<GenericJsonEntity> entities; try (PreparedStatement selectStatement = sqlQueryBuilder.createFindByIdStatement(connection, GenericJsonEntity.class, idsToFind)) { try (ResultSet resultSet = selectStatement.executeQuery()) { entities = genericResultSetMapper.mapAll(resultSet, GenericJsonEntity.class); } } if (CollectionUtils.isNotEmpty(entities)) { for (GenericJsonEntity entity : entities) { E bean = OBJECT_MAPPER.readValue(entity.getJsonVal(), pojoClass); bean.setId(entity.getId()); bean.setVersion(entity.getVersion()); ret.add(bean); } } } return ret; } }, Collections.<E>emptyList()); } public <E extends AbstractBean> List<E> get(Map<String, Object> filterParams, Class<E> pojoClass) { Predicate[] childPredicates = new Predicate[filterParams.size()]; int index = 0; for (Entry<String, Object> entry : filterParams.entrySet()) { childPredicates[index] = Predicate.EQ(entry.getKey(), entry.getValue()); index = index + 1; } return get(Predicate.AND(childPredicates), pojoClass); } public <E extends AbstractBean> List<E> get(final Predicate predicate, final Class<E> pojoClass) { //apply the predicates and fetch the primary key ids //look up the id and convert them to bean return runTask(new QueryTask<List<E>>() { @Override public List<E> handle(Connection connection) throws Exception { PojoInfo pojoInfo = pojoInfoMap.get(pojoClass); //find the matching ids to delete List<? extends AbstractIndexEntity> indexEntities; try (PreparedStatement findByParamsStatement = sqlQueryBuilder .createFindByParamsStatement(connection, pojoInfo.indexEntityClass, predicate)) { try (ResultSet rs = findByParamsStatement.executeQuery()) { indexEntities = genericResultSetMapper.mapAll(rs, pojoInfo.indexEntityClass); } } List<Long> idsToFind = new ArrayList<>(); if (CollectionUtils.isNotEmpty(indexEntities)) { for (AbstractIndexEntity entity : indexEntities) { idsToFind.add(entity.getBaseId()); } } dumpTable(connection, pojoInfo.indexEntityClass); //fetch the entities List<E> ret = new ArrayList<>(); if (!idsToFind.isEmpty()) { List<GenericJsonEntity> entities; try (PreparedStatement selectStatement = sqlQueryBuilder.createFindByIdStatement(connection, GenericJsonEntity.class, idsToFind)) { try (ResultSet resultSet = selectStatement.executeQuery()) { entities = genericResultSetMapper.mapAll(resultSet, GenericJsonEntity.class); } if (CollectionUtils.isNotEmpty(entities)) { for (GenericJsonEntity entity : entities) { E bean = OBJECT_MAPPER.readValue(entity.getJsonVal(), pojoClass); bean.setId(entity.getId()); bean.setVersion(entity.getVersion()); ret.add(bean); } } } } return ret; } }, Collections.<E>emptyList()); } private void dumpTable(Connection connection, Class<? extends AbstractEntity> entityClass) throws Exception { if (IS_DEBUG) { try (PreparedStatement findAllStatement = sqlQueryBuilder.createFindAllStatement(connection, entityClass)) { try (ResultSet resultSet = findAllStatement.executeQuery()) { List<? extends AbstractEntity> entities = genericResultSetMapper.mapAll(resultSet, entityClass); for (AbstractEntity entity : entities) { LOG.debug("{}", entity); } } } } } public <E extends AbstractBean> int delete(final Long id, final Class<E> pojoClass) { return runTask(new QueryTask<Integer>() { @Override public Integer handle(Connection connection) throws Exception { PojoInfo pojoInfo = pojoInfoMap.get(pojoClass); Map<String, Object> filters = new HashMap<>(); filters.put("id", id); try (PreparedStatement deleteStatement = sqlQueryBuilder.createDeleteByIdStatement(connection, GenericJsonEntity.class, filters)) { deleteStatement.executeUpdate(); } filters.clear(); filters.put("baseId", id); try (PreparedStatement deleteIndexStatement = sqlQueryBuilder.createDeleteByIdStatement(connection, pojoInfo.indexEntityClass, filters)) { return deleteIndexStatement.executeUpdate(); } } }, 0); } public <E extends AbstractBean> int deleteByParams(final Map<String, Object> filters, final Class<E> pojoClass) { return runTask(new QueryTask<Integer>() { @Override public Integer handle(Connection connection) throws Exception { PojoInfo pojoInfo = pojoInfoMap.get(pojoClass); //find the matching ids to delete List<? extends AbstractIndexEntity> indexEntities; try (PreparedStatement findByParamsStatement = sqlQueryBuilder .createFindByParamsStatement(connection, pojoInfo.indexEntityClass, filters)) { try (ResultSet rs = findByParamsStatement.executeQuery()) { indexEntities = genericResultSetMapper.mapAll(rs, pojoInfo.indexEntityClass); } } List<Long> idsToDelete = new ArrayList<>(); if (CollectionUtils.isNotEmpty(indexEntities)) { for (AbstractIndexEntity entity : indexEntities) { idsToDelete.add(entity.getBaseId()); } } int baseRowsDeleted = 0; if (!idsToDelete.isEmpty()) { //delete the ids from both base table and index table int indexRowsDeleted = 0; try (PreparedStatement statement = sqlQueryBuilder.createDeleteStatement(connection, pojoInfo.indexEntityClass, idsToDelete)) { indexRowsDeleted = statement.executeUpdate(); } try (PreparedStatement baseTableDeleteStatement = sqlQueryBuilder .createDeleteStatement(connection, GenericJsonEntity.class, idsToDelete)) { baseRowsDeleted = baseTableDeleteStatement.executeUpdate(); } assert (baseRowsDeleted == indexRowsDeleted); } return baseRowsDeleted; } }, 0); } private static interface QueryTask<T> { T handle(Connection connection) throws Exception; } <T> T runTask(QueryTask<T> task, T defaultReturnValue) { try (Connection connection = getConnection()) { T t = task.handle(connection); connection.commit(); return t; } catch (Exception e) { LOG.error("Exception while executing query task", e); return defaultReturnValue; } } static class PojoInfo { Class<? extends AbstractBean> pojoClass; String pojoName; String baseTableName; Class<AbstractJsonEntity> baseEntityClass; String indexTableName; Class<? extends AbstractIndexEntity> indexEntityClass; List<String> indexTableColumns; } }