/**
* Copyright 2014 Sunny Gleason and original author or authors
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.kazuki.v0.store.index;
import io.kazuki.v0.internal.availability.AvailabilityManager;
import io.kazuki.v0.internal.helper.IoHelper;
import io.kazuki.v0.internal.helper.JDBIHelper;
import io.kazuki.v0.internal.helper.LockManager;
import io.kazuki.v0.internal.helper.LogTranslation;
import io.kazuki.v0.internal.helper.OpaquePaginationHelper;
import io.kazuki.v0.internal.helper.SqlParamBindings;
import io.kazuki.v0.internal.v2schema.compact.FieldTransform;
import io.kazuki.v0.store.KazukiException;
import io.kazuki.v0.store.Key;
import io.kazuki.v0.store.index.query.QueryHelper;
import io.kazuki.v0.store.index.query.QueryTerm;
import io.kazuki.v0.store.index.query.ValueHolder;
import io.kazuki.v0.store.index.query.ValueType;
import io.kazuki.v0.store.keyvalue.KeyValueIterable;
import io.kazuki.v0.store.keyvalue.KeyValueIterator;
import io.kazuki.v0.store.keyvalue.KeyValuePair;
import io.kazuki.v0.store.keyvalue.KeyValueStore;
import io.kazuki.v0.store.keyvalue.KeyValueStoreIteration.SortDirection;
import io.kazuki.v0.store.keyvalue.KeyValueStoreIteratorJdbiImpl.KeyValueIterableJdbiImpl;
import io.kazuki.v0.store.keyvalue.KeyValueStoreRegistration;
import io.kazuki.v0.store.management.ComponentDescriptor;
import io.kazuki.v0.store.management.ComponentRegistrar;
import io.kazuki.v0.store.management.KazukiComponent;
import io.kazuki.v0.store.management.impl.ComponentDescriptorImpl;
import io.kazuki.v0.store.schema.SchemaStore;
import io.kazuki.v0.store.schema.SchemaStoreRegistration;
import io.kazuki.v0.store.schema.model.Attribute;
import io.kazuki.v0.store.schema.model.IndexAttribute;
import io.kazuki.v0.store.schema.model.IndexDefinition;
import io.kazuki.v0.store.schema.model.Schema;
import io.kazuki.v0.store.sequence.ResolvedKey;
import io.kazuki.v0.store.sequence.SequenceService;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import javax.annotation.Nullable;
import javax.inject.Inject;
import javax.sql.DataSource;
import org.skife.jdbi.v2.Handle;
import org.skife.jdbi.v2.IDBI;
import org.skife.jdbi.v2.Query;
import org.skife.jdbi.v2.TransactionCallback;
import org.skife.jdbi.v2.TransactionStatus;
import org.skife.jdbi.v2.Update;
import org.skife.jdbi.v2.exceptions.UnableToExecuteStatementException;
import org.slf4j.Logger;
import com.google.common.base.Throwables;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.Iterables;
public class SecondaryIndexStoreJdbiImpl implements SecondaryIndexSupport {
private final Logger log = LogTranslation.getLogger(getClass());
private final AvailabilityManager availability;
private final LockManager lockManager;
private final KazukiComponent<DataSource> dataSource;
private final IDBI database;
private final SequenceService sequence;
private final SchemaStore schemaStore;
private final KeyValueStore kvStore;
private final SecondaryIndexTableHelper tableHelper;
private final String groupName;
private final String storeName;
private final String partitionName;
private final ComponentDescriptor<SecondaryIndexStore> componentDescriptor;
@Inject
public SecondaryIndexStoreJdbiImpl(AvailabilityManager availability, LockManager lockManager,
KazukiComponent<DataSource> dataSource, IDBI database, SequenceService sequence,
SchemaStore schemaStore, KeyValueStore kvStore, SecondaryIndexTableHelper tableHelper,
String groupName, String storeName, String partitionName) {
this.availability = availability;
this.lockManager = lockManager;
this.dataSource = dataSource;
this.database = database;
this.sequence = sequence;
this.schemaStore = schemaStore;
this.kvStore = kvStore;
this.tableHelper = tableHelper;
this.groupName = groupName;
this.storeName = storeName;
this.partitionName = partitionName;
this.componentDescriptor =
new ComponentDescriptorImpl<SecondaryIndexStore>("KZ:SecondaryIndexStore:" + groupName
+ "-" + storeName + "-" + partitionName, SecondaryIndexStore.class,
(SecondaryIndexStore) this, new ImmutableList.Builder().add(
((KazukiComponent) this.lockManager).getComponentDescriptor(),
this.dataSource.getComponentDescriptor(),
((KazukiComponent) this.sequence).getComponentDescriptor(),
((KazukiComponent) this.kvStore).getComponentDescriptor()).build());
}
@Override
public ComponentDescriptor<SecondaryIndexStore> getComponentDescriptor() {
return this.componentDescriptor;
}
@Override
@Inject
public void registerAsComponent(ComponentRegistrar manager) {
manager.register(this.componentDescriptor);
}
@Inject
public void registerKeyValueStore(KeyValueStoreRegistration kvStore) {
kvStore.addListener(this);
}
@Inject
public void registerSchemaStore(SchemaStoreRegistration schemaStore) {
schemaStore.addListener(this);
}
@Override
public <T> KeyValueIterable<Key> queryWithoutPagination(String type, Class<T> clazz,
String indexName, List<QueryTerm> query, SortDirection sortDirection, @Nullable Long offset,
@Nullable Long limit) {
try {
return this.doIndexQuery(database, type, indexName, query, sortDirection, offset, limit,
false, schemaStore.retrieveSchema(type).getValue());
} catch (Exception e) {
throw Throwables.propagate(e);
}
}
@Override
public <T> KeyValueIterable<Key> queryWithoutPagination(String type, Class<T> clazz,
String indexName, String queryString, SortDirection sortDirection, @Nullable Long offset,
@Nullable Long limit) {
List<QueryTerm> query = QueryHelper.parseQuery(queryString);
return queryWithoutPagination(type, clazz, indexName, query, sortDirection, offset, limit);
}
@Override
public <T> QueryResultsPage<T> queryWithPagination(String type, Class<T> clazz, String indexName,
List<QueryTerm> query, SortDirection sortDirection, Boolean loadResults, PageToken token,
Long limit) {
try {
KeyValueIterable<Key> kvIter =
queryWithoutPagination(type, clazz, indexName, query, sortDirection,
OpaquePaginationHelper.decodeOpaqueCursor(token.getToken()), limit);
List<KeyValuePair<T>> kvPairs = new ArrayList<KeyValuePair<T>>();
if (loadResults) {
List<Key> toRetrieve = new ArrayList<Key>();
Iterables.addAll(toRetrieve, kvIter);
Map<Key, KeyValuePair<T>> resultMap = kvStore.multiRetrieveVersioned(toRetrieve, clazz);
for (Map.Entry<Key, KeyValuePair<T>> entry : resultMap.entrySet()) {
kvPairs.add(new KeyValuePair<T>(entry.getKey(), entry.getValue().getVersion(), entry
.getValue().getSchemaVersion(), entry.getValue().getValue()));
}
} else {
for (Key key : kvIter) {
kvPairs.add(new KeyValuePair<T>(key, null, null, null));
}
}
return new QueryResultsPageImpl<T>(kvPairs, loadResults);
} catch (Exception e) {
throw Throwables.propagate(e);
}
}
@Override
public <T> QueryResultsPage<T> queryWithPagination(String type, Class<T> clazz, String indexName,
String queryString, SortDirection sortDirection, Boolean loadResults, PageToken token,
Long limit) {
return queryWithPagination(type, clazz, indexName, QueryHelper.parseQuery(queryString),
sortDirection, loadResults, token, limit);
}
@Override
public Map<UniqueEntityDescription, Object> multiRetrieveUniqueEntities(
Collection<UniqueEntityDescription> entityDefinitions) {
Map<String, Schema> schemaMap = new LinkedHashMap<String, Schema>();
LinkedHashMap<UniqueEntityDescription, Object> inOrderResultMap = new LinkedHashMap<>();
try {
for (UniqueEntityDescription<?> desc : entityDefinitions) {
String type = desc.getType();
String indexName = desc.getIndexName();
Schema schema = schemaMap.get(type);
if (schema == null) {
schema = schemaStore.retrieveSchema(type).getValue();
schemaMap.put(type, schema);
}
List<QueryTerm> query = new ArrayList<QueryTerm>();
query.addAll(desc.getColumnDefinitions().values());
try (KeyValueIterator<Key> result =
this.queryWithoutPagination(type, desc.getClass(), indexName, query,
SortDirection.ASCENDING, 0L, 1L).iterator()) {
if (result.hasNext()) {
Key key = result.next();
inOrderResultMap.put(desc, kvStore.retrieve(key, desc.getClass()));
} else {
inOrderResultMap.put(desc, null);
}
}
}
} catch (KazukiException e) {
throw Throwables.propagate(e);
}
return Collections.unmodifiableMap(inOrderResultMap);
}
@Override
public Map<UniqueEntityDescription, Key> multiRetrieveUniqueKeys(
Collection<UniqueEntityDescription> entityDefinitions) {
Map<String, Schema> schemaMap = new LinkedHashMap<String, Schema>();
LinkedHashMap<UniqueEntityDescription, Key> inOrderResultMap = new LinkedHashMap<>();
try {
for (UniqueEntityDescription<?> desc : entityDefinitions) {
String type = desc.getType();
String indexName = desc.getIndexName();
Schema schema = schemaMap.get(type);
if (schema == null) {
schema = schemaStore.retrieveSchema(type).getValue();
schemaMap.put(type, schema);
}
List<QueryTerm> query = new ArrayList<QueryTerm>();
query.addAll(desc.getColumnDefinitions().values());
try (KeyValueIterator<Key> result =
this.queryWithoutPagination(type, desc.getClass(), indexName, query,
SortDirection.ASCENDING, 0L, 1L).iterator()) {
if (result.hasNext()) {
inOrderResultMap.put(desc, result.next());
} else {
inOrderResultMap.put(desc, null);
}
}
}
} catch (KazukiException e) {
throw Throwables.propagate(e);
}
return Collections.unmodifiableMap(inOrderResultMap);
}
@Override
public <T> void enforceUnique(String type, Class<T> clazz, Schema schema,
ResolvedKey resolvedKey, Map<String, Object> instance) throws KazukiException {
IndexDefinition uniqueIndexDef = getUniqueIndexDef(schema);
if (uniqueIndexDef != null) {
Map<String, ValueHolder> values = new LinkedHashMap<String, ValueHolder>();
for (String attr : uniqueIndexDef.getAttributeNames()) {
values.put(attr, new ValueHolder(ValueType.STRING, instance.get(attr).toString()));
}
UniqueEntityDescription uniqueDesc =
new UniqueEntityDescription(type, clazz, uniqueIndexDef.getName(), schema, values);
Key maybeExists = this.multiRetrieveUniqueKeys(ImmutableList.of(uniqueDesc)).get(uniqueDesc);
if (maybeExists != null && !sequence.resolveKey(maybeExists).equals(resolvedKey)) {
throw new KazukiException("unique index constraint violation");
}
}
}
@Override
public <T> void onCreate(Handle handle, String type, Class<T> clazz, Schema schema,
ResolvedKey resolvedKey, Map<String, Object> instance) {
try (LockManager toRelease = lockManager.acquire()) {
try {
for (IndexDefinition indexDef : schema.getIndexes()) {
this.insertEntity(handle, resolvedKey.getIdentifierLo(), instance, type,
indexDef.getName(), schema);
}
} catch (KazukiException e) {
throw Throwables.propagate(e);
}
}
}
@Override
public <T> void onUpdate(Handle handle, String type, Class<T> clazz, Schema schema,
ResolvedKey resolvedKey, Map<String, Object> newInstance, Map<String, Object> oldInstance) {
try (LockManager toRelease = lockManager.acquire()) {
try {
for (IndexDefinition indexDef : schema.getIndexes()) {
this.updateEntity(handle, resolvedKey.getIdentifierLo(), newInstance, oldInstance, type,
indexDef.getName(), schema);
}
} catch (KazukiException e) {
throw Throwables.propagate(e);
}
}
}
@Override
public <T> void onDelete(Handle handle, String type, Class<T> clazz, Schema schema,
ResolvedKey resolvedKey, Map<String, Object> oldInstance) {
try (LockManager toRelease = lockManager.acquire()) {
try {
for (IndexDefinition indexDef : schema.getIndexes()) {
this.deleteEntity(handle, resolvedKey.getIdentifierLo(), type, oldInstance,
indexDef.getName(), schema);
}
} catch (KazukiException e) {
throw Throwables.propagate(e);
}
}
}
@Override
public void clear(Handle handle, Map<String, Schema> typeToSchemaMap, boolean preserveSchema) {
try (LockManager toRelease = lockManager.acquire()) {
for (Map.Entry<String, Schema> entry : typeToSchemaMap.entrySet()) {
String type = entry.getKey();
Schema schema = entry.getValue();
for (IndexDefinition indexDef : schema.getIndexes()) {
if (preserveSchema) {
this.truncateTable(handle, type, indexDef.getName(), groupName, storeName,
partitionName);
} else {
this.dropTableAndIndex(handle, type, indexDef.getName());
}
}
}
}
}
@Override
public void onSchemaCreate(String type, Schema schema) {
try (LockManager toRelease = lockManager.acquire()) {
for (IndexDefinition indexDef : schema.getIndexes()) {
createTable(database, type, indexDef.getName(), schema);
createIndex(database, type, indexDef.getName(), schema);
}
}
}
@Override
public void onSchemaUpdate(final String type, final Schema newSchema, final Schema oldSchema,
final KeyValueIterable<KeyValuePair<LinkedHashMap>> entityCollection) {
try (LockManager toRelease = lockManager.acquire()) {
try {
for (final IndexDefinition indexDef : oldSchema.getIndexes()) {
database.inTransaction(new TransactionCallback<Void>() {
public Void inTransaction(Handle handle, TransactionStatus status) throws Exception {
dropTableAndIndex(handle, type, indexDef.getName());
return null;
};
});
}
for (final IndexDefinition indexDef : oldSchema.getIndexes()) {
createTable(database, type, indexDef.getName(), newSchema);
createIndex(database, type, indexDef.getName(), newSchema);
}
final FieldTransform fieldTransform = new FieldTransform(oldSchema);
database.inTransaction(new TransactionCallback<Void>() {
@Override
public Void inTransaction(Handle handle, TransactionStatus status) throws Exception {
for (KeyValuePair<LinkedHashMap> entity : entityCollection) {
Map<String, Object> fieldTransformed = fieldTransform.pack(entity.getValue());
SecondaryIndexStoreJdbiImpl.this.onCreate(handle, type, LinkedHashMap.class,
newSchema, sequence.resolveKey(entity.getKey()), entity.getValue());
}
return null;
}
});
} finally {
entityCollection.close();
}
}
}
@Override
public void onSchemaDelete(final String type, final Schema oldSchema) {
try (LockManager toRelease = lockManager.acquire()) {
for (final IndexDefinition indexDef : oldSchema.getIndexes()) {
database.inTransaction(new TransactionCallback<Void>() {
public Void inTransaction(Handle handle, TransactionStatus status) throws Exception {
dropTableAndIndex(handle, type, indexDef.getName());
return null;
};
});
}
}
}
private void createTable(IDBI database, final String type, final String indexName,
final Schema schema) {
String tableDefinition =
tableHelper
.getTableDefinition(type, indexName, schema, groupName, storeName, partitionName);
log.debug("create table: {}" + tableDefinition);
JDBIHelper.createTable(database,
tableHelper.getTableDrop(type, indexName, groupName, storeName, partitionName),
tableDefinition);
}
private void createIndex(IDBI database, final String type, final String indexName,
final Schema schemaDefinition) {
database.inTransaction(new TransactionCallback<Void>() {
@Override
public Void inTransaction(Handle handle, TransactionStatus status) throws Exception {
try {
handle
.createStatement(tableHelper.getPrefix() + "drop_index")
.define("table_name",
tableHelper.getTableName(type, indexName, groupName, storeName, partitionName))
.define("index_name",
tableHelper.getIndexName(type, indexName, groupName, storeName, partitionName))
.execute();
} catch (UnableToExecuteStatementException ok) {
// expected case in mysql - this is just best-effort anyway
}
String indexDefinition =
tableHelper.getIndexDefinition(type, indexName, schemaDefinition, groupName, storeName,
partitionName);
log.debug("create index: {}" + indexDefinition);
handle.createStatement(indexDefinition).execute();
return null;
}
});
}
private void dropTableAndIndex(Handle handle, final String type, final String indexName) {
handle.createStatement(
tableHelper.getTableDrop(type, indexName, groupName, storeName, partitionName)).execute();
try {
handle
.createStatement(tableHelper.getPrefix() + "drop_index")
.define("table_name",
tableHelper.getTableName(type, indexName, groupName, storeName, partitionName))
.define("index_name",
tableHelper.getIndexName(type, indexName, groupName, storeName, partitionName))
.execute();
} catch (UnableToExecuteStatementException ok) {
// expected case in mysql - this is just best-effort anyway
}
}
private void insertEntity(Handle handle, final Long id, final Map<String, Object> value,
final String type, final String indexName, final Schema schema) throws KazukiException {
SqlParamBindings bindings = new SqlParamBindings(true);
Update insert =
handle.createStatement(tableHelper.getInsertStatement(type, indexName, schema, bindings,
groupName, storeName, partitionName));
IndexDefinition indexDefinition = schema.getIndex(indexName);
bindings.bind("id", id, Attribute.Type.U64);
for (IndexAttribute attr : indexDefinition.getIndexAttributes()) {
String attrName = attr.getName();
if ("id".equals(attrName)) {
continue;
} else {
Object v = value.get(attrName) != null ? value.get(attrName).toString() : null;
bindings.bind(attrName, tableHelper.transformAttributeValue(v, attr),
schema.getAttribute(attrName).getType());
}
}
bindings.bindToStatement(insert);
try {
insert.execute();
} catch (UnableToExecuteStatementException e) {
if (tableHelper.isConstraintViolation(e)) {
throw new KazukiException("unique index constraint violation");
} else {
throw e;
}
}
}
private void updateEntity(Handle handle, final Long id, final Map<String, Object> value,
final Map<String, Object> prev, final String type, final String indexName, final Schema schema)
throws KazukiException {
IndexDefinition indexDefinition = schema.getIndex(indexName);
String origKey = tableHelper.computeIndexKey(type, indexName, indexDefinition, prev);
String newKey = tableHelper.computeIndexKey(type, indexName, indexDefinition, value);
if (origKey.equals(newKey)) {
return;
}
SqlParamBindings bindings = new SqlParamBindings(true);
Update update =
handle.createStatement(tableHelper.getUpdateStatement(type, indexName, schema, bindings,
groupName, storeName, partitionName));
for (IndexAttribute attr : indexDefinition.getIndexAttributes()) {
String attrName = attr.getName();
if ("id".equals(attrName)) {
continue;
} else {
Object v = value.get(attrName) != null ? value.get(attrName).toString() : null;
bindings.bind(attrName, tableHelper.transformAttributeValue(v, attr),
schema.getAttribute(attrName).getType());
}
}
bindings.bind("id", id, Attribute.Type.U64);
bindings.bindToStatement(update);
try {
update.execute();
} catch (UnableToExecuteStatementException e) {
if (tableHelper.isConstraintViolation(e)) {
throw new KazukiException("unique index constraint violation");
} else {
throw e;
}
}
}
private void deleteEntity(Handle handle, final Long id, final String type,
final Map<String, Object> value, final String indexName, final Schema schema)
throws KazukiException {
IndexDefinition indexDefinition = schema.getIndex(indexName);
if (indexDefinition == null) {
throw new KazukiException("schema or index not found " + type + "." + indexName);
}
SqlParamBindings bindings = new SqlParamBindings(true);
Update delete =
handle.createStatement(tableHelper.getDeleteStatement(type, indexName, bindings, groupName,
storeName, partitionName));
bindings.bind("id", id, Attribute.Type.U64);
bindings.bindToStatement(delete);
delete.execute();
}
/*
* private void setEntityQuarantine(Handle handle, final Long id, final String type, final String
* indexName, boolean isQuarantined, final Map<String, Object> original, final Schema schema) {
* SqlParamBindings bindings = new SqlParamBindings(true);
*
* Update quarantine = handle.createStatement(tableHelper.getQuarantineStatement(type, indexName,
* bindings, isQuarantined, groupName, storeName, partitionName));
*
* bindings.bind("id", id, Attribute.Type.U64); bindings.bindToStatement(quarantine);
*
* quarantine.execute(); }
*/
private KeyValueIterable<Key> doIndexQuery(IDBI database, final String type, String indexName,
List<QueryTerm> queryTerms, final SortDirection sortDirection, Long offset, Long pageSize,
boolean includeQuarantine, final Schema schema) throws Exception {
SecondaryIndexQueryValidation.validateQuery(indexName, queryTerms, schema);
IndexDefinition indexDefinition = schema.getIndex(indexName);
final FieldTransform transform = new FieldTransform(schema);
Map<String, List<QueryTerm>> termMap = tableHelper.sortTerms(indexDefinition, queryTerms);
final SqlParamBindings bindings = new SqlParamBindings(true);
final String querySql =
tableHelper.getIndexQuery(type, indexName, termMap, sortDirection, offset, pageSize,
includeQuarantine, indexDefinition, schema, transform, bindings, groupName, storeName,
partitionName);
log.debug("non-unique index query : {} : bindings : {}", querySql, bindings.asMap());
final Handle handle = database.open();
final Query<Map<String, Object>> select = handle.createQuery(querySql);
bindings.bindToStatement(select);
return new KeyValueIterable<Key>() {
private volatile KeyValueIterableJdbiImpl<LinkedHashMap> inner =
new KeyValueIterableJdbiImpl(availability, sequence, kvStore, schema, handle,
tableHelper.getPrefix(), "_id", select, type, LinkedHashMap.class, sortDirection,
null, null, false, false);
@Override
public KeyValueIterator<Key> iterator() {
return new KeyValueIterator<Key>() {
volatile KeyValueIterator<KeyValuePair<LinkedHashMap>> innerIter = inner.iterator();
@Override
public boolean hasNext() {
if (innerIter == null) {
return false;
}
return innerIter.hasNext();
}
@Override
public Key next() {
return innerIter.next().getKey();
}
@Override
public void remove() {
throw new UnsupportedOperationException("not supported - yet");
// innerIter.remove();
}
@Override
public void close() {
IoHelper.closeQuietly(innerIter, log);
innerIter = null;
}
};
}
@Override
public void close() {
IoHelper.closeQuietly(inner, log);
inner = null;
}
};
}
private void truncateTable(Handle handle, final String type, final String indexName,
final String groupName, String storeName, String partitionName) {
String indexTableName =
tableHelper.getTableName(type, indexName, groupName, storeName, partitionName);
handle.createStatement(tableHelper.getPrefix() + "truncate_table")
.define("table_name", indexTableName).execute();
}
private IndexDefinition getUniqueIndexDef(Schema schema) {
for (IndexDefinition indexDef : schema.getIndexes()) {
if (indexDef.isUnique()) {
return indexDef;
}
}
return null;
}
}