/**
* Copyright (C) 2009-2013 FoundationDB, LLC
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package com.foundationdb.server.store;
import com.foundationdb.ais.model.AkibanInformationSchema;
import com.foundationdb.ais.model.CacheValueGenerator;
import com.foundationdb.ais.model.Column;
import com.foundationdb.ais.model.Group;
import com.foundationdb.ais.model.GroupIndex;
import com.foundationdb.ais.model.Index;
import com.foundationdb.ais.model.Index.IndexType;
import com.foundationdb.ais.model.Table;
import com.foundationdb.ais.model.TableIndex;
import com.foundationdb.ais.util.TableChange.ChangeType;
import com.foundationdb.ais.util.TableChangeValidator.ChangeLevel;
import com.foundationdb.qp.exec.Plannable;
import com.foundationdb.qp.operator.API;
import com.foundationdb.qp.operator.ChainedCursor;
import com.foundationdb.qp.operator.Cursor;
import com.foundationdb.qp.operator.Operator;
import com.foundationdb.qp.operator.QueryBindings;
import com.foundationdb.qp.operator.QueryContext;
import com.foundationdb.qp.operator.Rebindable;
import com.foundationdb.qp.operator.SimpleQueryContext;
import com.foundationdb.qp.operator.StoreAdapter;
import com.foundationdb.qp.operator.Delete_Returning;
import com.foundationdb.qp.row.OverlayingRow;
import com.foundationdb.qp.row.ProjectedRow;
import com.foundationdb.qp.row.Row;
import com.foundationdb.qp.row.WriteIndexRow;
import com.foundationdb.qp.rowtype.ProjectedTableRowType;
import com.foundationdb.qp.rowtype.RowType;
import com.foundationdb.qp.rowtype.Schema;
import com.foundationdb.qp.rowtype.TableRowType;
import com.foundationdb.qp.storeadapter.indexrow.SpatialColumnHandler;
import com.foundationdb.qp.util.SchemaCache;
import com.foundationdb.server.error.ConcurrentViolationException;
import com.foundationdb.server.error.ConstraintViolationException;
import com.foundationdb.server.error.InvalidOperationException;
import com.foundationdb.server.error.NoSuchRowException;
import com.foundationdb.server.error.NotAllowedByConfigException;
import com.foundationdb.server.error.SQLParserInternalException;
import com.foundationdb.server.service.blob.BlobRef;
import com.foundationdb.server.types.aksql.aktypes.AkBlob;
import com.foundationdb.server.types.common.types.TypesTranslator;
import com.foundationdb.server.types.service.TypesRegistryService;
import com.foundationdb.server.service.dxl.DelegatingContext;
import com.foundationdb.server.service.listener.RowListener;
import com.foundationdb.server.service.session.Session;
import com.foundationdb.server.service.transaction.TransactionService;
import com.foundationdb.server.store.SchemaManager.OnlineChangeState;
import com.foundationdb.server.store.TableChanges.Change;
import com.foundationdb.server.store.TableChanges.ChangeSet;
import com.foundationdb.server.store.TableChanges.IndexChange;
import com.foundationdb.server.types.TCast;
import com.foundationdb.server.types.TInstance;
import com.foundationdb.server.types.texpressions.TCastExpression;
import com.foundationdb.server.types.texpressions.TPreparedExpression;
import com.foundationdb.server.types.texpressions.TPreparedField;
import com.foundationdb.server.types.value.ValueSource;
import com.foundationdb.sql.StandardException;
import com.foundationdb.sql.optimizer.CreateAsCompiler;
import com.foundationdb.sql.optimizer.plan.BasePlannable;
import com.foundationdb.sql.optimizer.rule.OperatorAssembler;
import com.foundationdb.sql.optimizer.rule.PlanContext;
import com.foundationdb.sql.optimizer.rule.PlanGenerator;
import com.foundationdb.ais.model.TableName;
import com.foundationdb.sql.parser.DMLStatementNode;
import com.foundationdb.sql.parser.SQLParser;
import com.foundationdb.sql.parser.StatementNode;
import com.foundationdb.sql.server.ServerSession;
import com.google.common.collect.HashMultimap;
import com.google.common.collect.Multimap;
import com.persistit.Key;
import com.persistit.KeyState;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.UUID;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
public class OnlineHelper implements RowListener
{
private static final Logger LOG = LoggerFactory.getLogger(OnlineHelper.class);
private static final Object TRANSFORM_CACHE_KEY = new Object();
private final TransactionService txnService;
private final SchemaManager schemaManager;
private final Store store;
private final TypesRegistryService typesRegistry;
private final ConstraintHandler constraintHandler;
private final boolean withConcurrentDML;
public OnlineHelper(TransactionService txnService,
SchemaManager schemaManager,
Store store,
TypesRegistryService typesRegistry,
ConstraintHandler constraintHandler,
boolean withConcurrentDML) {
this.txnService = txnService;
this.schemaManager = schemaManager;
this.store = store;
this.typesRegistry = typesRegistry;
this.constraintHandler = constraintHandler;
this.withConcurrentDML = withConcurrentDML;
}
public void buildIndexes(Session session, QueryContext context) {
LOG.debug("Building indexes");
txnService.beginTransaction(session);
try {
buildIndexesInternal(session, context);
txnService.commitTransaction(session);
} finally {
txnService.rollbackTransactionIfOpen(session);
}
}
public void checkTableConstraints(final Session session, QueryContext context) {
LOG.debug("Checking constraints");
txnService.beginTransaction(session);
try {
Collection<ChangeSet> changeSets = schemaManager.getOnlineChangeSets(session);
assert (commonChangeLevel(changeSets) == ChangeLevel.METADATA_CONSTRAINT) : changeSets;
// Gather all tables that need scanned, keyed by group
AkibanInformationSchema oldAIS = schemaManager.getAis(session);
Schema oldSchema = SchemaCache.globalSchema(oldAIS);
Multimap<Group,RowType> groupMap = HashMultimap.create();
for(ChangeSet cs : changeSets) {
RowType rowType = oldSchema.tableRowType(cs.getTableId());
groupMap.put(rowType.table().getGroup(), rowType);
}
// Scan all affected groups
StoreAdapter adapter = store.createAdapter(session);
final TransformCache transformCache = getTransformCache(session, null);
for(Entry<Group, Collection<RowType>> entry : groupMap.asMap().entrySet()) {
Operator plan = API.filter_Default(API.groupScan_Default(entry.getKey()), entry.getValue());
runPlan(session, contextIfNull(context, adapter), schemaManager, txnService, plan, new RowHandler() {
@Override
public void handleRow(Row row) {
simpleCheckConstraints(session, transformCache, row);
}
});
}
} finally {
txnService.rollbackTransactionIfOpen(session);
}
}
public void alterTable(Session session, QueryContext context) {
LOG.debug("Altering table");
txnService.beginTransaction(session);
try {
alterInternal(session, context);
} finally {
txnService.rollbackTransactionIfOpen(session);
}
}
//
// RowListener
//
public void onInsertPost(Session session, Table table, Key hKey, Row row) {
TableTransform transform = getConcurrentDMLTransform(session, table);
if(transform == null) {
return;
}
try {
concurrentDML(session, transform, hKey, null, row);
} catch(ConstraintViolationException e) {
setOnlineError(session, table, e);
}
}
@Override
public void onUpdatePre(Session session, Table table, Key hKey, Row oldRow, Row newRow) {
TableTransform transform = getConcurrentDMLTransform(session, table);
if(transform == null) {
return;
}
try {
concurrentDML(session, transform, hKey, oldRow, null);
} catch(ConstraintViolationException e) {
setOnlineError(session, table, e);
}
}
@Override
public void onUpdatePost(Session session, Table table, Key hKey, Row oldRow, Row newRow) {
TableTransform transform = getConcurrentDMLTransform(session, table);
if(transform == null) {
return;
}
try {
concurrentDML(session, transform, hKey, null, newRow);
} catch(ConstraintViolationException e) {
setOnlineError(session, table, e);
}
}
@Override
public void onDeletePre(Session session, Table table, Key hKey, Row row) {
TableTransform transform = getConcurrentDMLTransform(session, table);
if(transform == null) {
return;
}
try {
concurrentDML(session, transform, hKey, row, null);
} catch(ConstraintViolationException e) {
setOnlineError(session, table, e);
}
}
//
// ConstraintHandler.Handler-ish
//
public void handleInsert(Session session, Table table, Row row) {
TableTransform transform = getConcurrentDMLTransform(session, table);
if(transform == null) {
return;
}
if(transform.checkConstraints) {
boolean orig = txnService.setForceImmediateForeignKeyCheck(session, true);
try {
constraintHandler.handleInsert(session, transform.rowType.table(), row);
} catch(ConstraintViolationException e) {
setOnlineError(session, table, e);
} finally {
txnService.setForceImmediateForeignKeyCheck(session, orig);
}
}
}
public void handleUpdatePre(Session session, Table table, Row oldRow, Row newRow) {
TableTransform transform = getConcurrentDMLTransform(session, table);
if(transform == null) {
return;
}
if(transform.checkConstraints) {
boolean orig = txnService.setForceImmediateForeignKeyCheck(session, true);
try {
constraintHandler.handleUpdatePre(session, transform.rowType.table(), oldRow, newRow);
} catch(ConstraintViolationException e) {
setOnlineError(session, table, e);
} finally {
txnService.setForceImmediateForeignKeyCheck(session, orig);
}
}
}
public void handleUpdatePost(Session session, Table table, Row oldRow, Row newRow) {
TableTransform transform = getConcurrentDMLTransform(session, table);
if(transform == null) {
return;
}
if(transform.checkConstraints) {
boolean orig = txnService.setForceImmediateForeignKeyCheck(session, true);
try {
constraintHandler.handleUpdatePost(session, transform.rowType.table(), oldRow, newRow);
} catch(ConstraintViolationException e) {
setOnlineError(session, table, e);
} finally {
txnService.setForceImmediateForeignKeyCheck(session, orig);
}
}
}
public void handleDelete(Session session, Table table, Row row) {
TableTransform transform = getConcurrentDMLTransform(session, table);
if(transform == null) {
return;
}
if(transform.checkConstraints) {
boolean orig = txnService.setForceImmediateForeignKeyCheck(session, true);
try {
constraintHandler.handleDelete(session, transform.rowType.table(), row);
} catch(ConstraintViolationException e) {
setOnlineError(session, table, e);
} finally {
txnService.setForceImmediateForeignKeyCheck(session, orig);
}
}
}
public void handleTruncate(Session session, Table table) {
TableTransform transform = getConcurrentDMLTransform(session, table);
if(transform == null) {
return;
}
if(transform.checkConstraints) {
boolean orig = txnService.setForceImmediateForeignKeyCheck(session, true);
try {
constraintHandler.handleTruncate(session, transform.rowType.table());
} catch(ConstraintViolationException e) {
setOnlineError(session, table, e);
} finally {
txnService.setForceImmediateForeignKeyCheck(session, orig);
}
}
}
//
// Internal
//
private void setOnlineError(Session session, Table t, ConstraintViolationException e) {
// Note: Written in the same transaction executing DML, checked in session executing DDL
schemaManager.setOnlineDMLError(session, t.getTableId(), e.getMessage());
}
private void buildIndexesInternal(Session session, QueryContext context) {
Collection<ChangeSet> changeSets = schemaManager.getOnlineChangeSets(session);
ChangeLevel changeLevel = commonChangeLevel(changeSets);
assert (changeLevel == ChangeLevel.INDEX || changeLevel == ChangeLevel.INDEX_CONSTRAINT) : changeSets;
TransformCache transformCache = getTransformCache(session, null);
Multimap<Group,RowType> tableIndexes = HashMultimap.create();
Set<GroupIndex> groupIndexes = new HashSet<>();
for(ChangeSet cs : changeSets) {
TableTransform transform = transformCache.get(cs.getTableId());
tableIndexes.put(transform.rowType.table().getGroup(), transform.rowType);
groupIndexes.addAll(transform.groupIndexes);
}
StoreAdapter adapter = store.createAdapter(session);
if(!tableIndexes.isEmpty()) {
buildTableIndexes(session, context, adapter, transformCache, tableIndexes);
}
if(!groupIndexes.isEmpty()) {
if(changeLevel == ChangeLevel.INDEX_CONSTRAINT) {
throw new IllegalStateException("Constraint and group indexes");
}
buildGroupIndexes(session, context, adapter, groupIndexes);
}
}
public void createAsSelect(final Session session,
QueryContext context,
final ServerSession server,
String queryExpression,
TableName tableName) {
LOG.debug("Creating Table As Select Online");
txnService.beginTransaction(session);
try {
SQLParser parser = server.getParser();
StatementNode stmt;
String statement = "insert into " + tableName.toStringEscaped() + " " + queryExpression;
try {
stmt = parser.parseStatement(statement);
} catch (StandardException e) {
throw new SQLParserInternalException(e);//make specific runtime error unexpectedException
}
AkibanInformationSchema onlineAIS = schemaManager.getOnlineAIS(session);
StoreAdapter adapter = store.createAdapter(session);
CreateAsCompiler compiler = new CreateAsCompiler(server, adapter, false, onlineAIS);
DMLStatementNode dmlStmt = (DMLStatementNode) stmt;
PlanContext planContext = new PlanContext(compiler);
BasePlannable result = compiler.compile(dmlStmt, null, planContext);
Plannable plannable = result.getPlannable();
QueryContext newContext = contextIfNull(context, adapter);
getTransformCache(session, server);
runPlan(session, newContext, schemaManager, txnService, (Operator) plannable, null);
}finally{
txnService.commitTransaction(session);
}
}
private void alterInternal(final Session session, QueryContext context) {
final Collection<ChangeSet> changeSets = schemaManager.getOnlineChangeSets(session);
final ChangeLevel changeLevel = commonChangeLevel(changeSets);
assert (changeLevel == ChangeLevel.TABLE || changeLevel == ChangeLevel.GROUP) : changeSets;
final AkibanInformationSchema origAIS = schemaManager.getAis(session);
final AkibanInformationSchema newAIS = schemaManager.getOnlineAIS(session);
final StoreAdapter origAdapter = store.createAdapter(session);
final QueryContext origContext = new DelegatingContext(origAdapter, context);
final QueryBindings origBindings = origContext.createBindings();
final TransformCache transformCache = getTransformCache(session, null);
Set<Table> origRoots = findOldRoots(changeSets, origAIS, newAIS);
for(Table root : origRoots) {
final LobCheck lobCheckRes = checkForDropLob(root, changeSets, origAIS);
Operator plan = API.groupScan_Default(root.getGroup());
runPlan(session, contextIfNull(context, origAdapter), schemaManager, txnService, plan, new RowHandler() {
@Override
public void handleRow(Row oldRow) {
TableTransform transform = transformCache.get(oldRow.rowType().typeId());
Row newRow = transformRow(origContext, origBindings, transform, oldRow);
if (lobCheckRes != null ) {
registerLob(session, oldRow, lobCheckRes.tableId, lobCheckRes.columnPos);
}
origAdapter.writeRow(newRow, transform.tableIndexes, transform.groupIndexes);
}
});
}
}
private LobCheck checkForDropLob(Table root, Collection<ChangeSet> changeSets, AkibanInformationSchema oldAIS) {
LobCheck lc = null;
for (ChangeSet cs : changeSets) {
if (cs.hasChangeLevel() && cs.getChangeLevel().equals(ChangeLevel.TABLE.name())) {
Table oldTable = oldAIS.getTable(cs.getOldSchema(), cs.getOldName());
if (oldTable.getGroup().getRoot() == root) {
for( int i =0; i < cs.getColumnChangeCount(); i++) {
Change change = cs.getColumnChange(i);
if (change.hasChangeType() && change.getChangeType().equals("DROP")) {
String oldColName = change.getOldName();
Column col = oldTable.getColumn(oldColName);
if (AkBlob.isBlob(col.getType().typeClass())){
assert lc == null; // change set can only contain a single column drop
lc = new LobCheck();
lc. tableId = oldTable.getTableId();
lc.columnPos = col.getPosition();
}
}
}
}
}
}
return lc;
}
private class LobCheck {
public int tableId;
public int columnPos;
}
private void registerLob(Session session, Row oldRow, int tableId, int dropField) {
if (oldRow.rowType().typeId() == tableId) {
ValueSource val = oldRow.value(dropField);
Object blob = val.getObject();
if (blob instanceof BlobRef) {
if (((BlobRef) blob).isLongLob()) {
UUID blobId = ((BlobRef) blob).getId();
if (store instanceof FDBStore) {
TableName rootTable = oldRow.rowType().table().getGroup().getName();
((FDBStore) store).registerLobForOnlineDelete(session, rootTable, blobId);
}
}
}
}
}
private void buildTableIndexes(final Session session,
QueryContext context,
StoreAdapter adapter,
final TransformCache transformCache,
Multimap<Group,RowType> tableIndexes) {
final WriteIndexRow buffer = new WriteIndexRow();
for(Entry<Group, Collection<RowType>> entry : tableIndexes.asMap().entrySet()) {
if(entry.getValue().isEmpty()) {
continue;
}
Operator plan = API.filter_Default(
API.groupScan_Default(entry.getKey()),
entry.getValue()
);
runPlan(session, contextIfNull(context, adapter), schemaManager, txnService, plan, new RowHandler() {
@Override
public void handleRow(final Row row) {
TableTransform transform = transformCache.get(row.rowType().typeId());
simpleCheckConstraints(session, transformCache, row);
for(final TableIndex index : transform.tableIndexes) {
final Key hKey = store.createKey();
row.hKey().copyTo(hKey);
if (index.isSpatial()) {
final SpatialColumnHandler spatialColumnHandler = new SpatialColumnHandler(index);
spatialColumnHandler.processSpatialObject(
row,
new SpatialColumnHandler.Operation()
{
@Override
public void handleZValue(long z)
{
store.writeIndexRow(session, index, row, hKey, buffer,
spatialColumnHandler, z, true);
}
});
} else {
store.writeIndexRow(session, index, row, hKey, buffer, null, -1L, true);
}
}
}
});
}
}
@SuppressWarnings("unchecked")
private void buildGroupIndexes(final Session session,
QueryContext context,
StoreAdapter adapter,
Collection<GroupIndex> groupIndexes) {
if(groupIndexes.isEmpty()) {
return;
}
for(final GroupIndex groupIndex : groupIndexes) {
Schema schema = SchemaCache.globalSchema(groupIndex.getAIS());
final Operator plan = StoreGIMaintenancePlans.groupIndexCreationPlan(schema, groupIndex);
final StoreGIHandler giHandler = StoreGIHandler.forBuilding((AbstractStore)store, session, schema, groupIndex);
runPlan(session, contextIfNull(context, adapter), schemaManager, txnService, plan, new RowHandler() {
@Override
public void handleRow(Row row) {
giHandler.handleRow(groupIndex, row, StoreGIHandler.Action.STORE);
}
});
}
}
private void simpleCheckConstraints(Session session, TransformCache transformCache, Row row) {
TableTransform transform = transformCache.get(row.rowType().typeId());
if(transform == null || !transform.checkConstraints) {
return;
}
constraintHandler.handleInsert(session, transform.rowType.table(), row);
}
private void concurrentDML(final Session session,
final TableTransform transform,
final Key hKey,
final Row oldRow,
final Row newRow) {
final boolean doDelete = (oldRow != null);
final boolean doWrite = (newRow != null);
QueryContext context = null;
switch(transform.changeLevel) {
case INDEX:
if(!transform.tableIndexes.isEmpty()) {
final WriteIndexRow buffer = new WriteIndexRow ();
for(final TableIndex index : transform.tableIndexes) {
long oldZValue = -1;
long newZValue = -1;
if (index.isSpatial()) {
final SpatialColumnHandler spatialColumnHandler = new SpatialColumnHandler(index);
if(doDelete) {
spatialColumnHandler.processSpatialObject
(oldRow,
new SpatialColumnHandler.Operation()
{
@Override
public void handleZValue(long z)
{
store.deleteIndexRow(session,
index,
oldRow,
hKey,
buffer,
spatialColumnHandler,
z,
false);
}
});
}
if(doWrite) {
spatialColumnHandler.processSpatialObject
(oldRow,
new SpatialColumnHandler.Operation()
{
@Override
public void handleZValue(long z)
{
Row outputRow = new OverlayingRow(newRow, transform.rowType);
store.writeIndexRow(session,
index,
outputRow,
hKey,
buffer,
spatialColumnHandler,
z,
false);
}
});
}
} else {
if(doDelete) {
store.deleteIndexRow(session, index, oldRow, hKey, buffer, null, -1L, false);
}
if(doWrite) {
Row outputRow = new OverlayingRow(newRow, transform.rowType);
store.writeIndexRow(session, index, outputRow, hKey, buffer, null, -1L, false);
}
}
}
}
if(!transform.groupIndexes.isEmpty()) {
if(doDelete) {
Row deleteRow = new OverlayingRow (oldRow, transform.rowType);
store.deleteIndexRows(session, transform.rowType.table(), deleteRow, transform.groupIndexes);
}
if(doWrite) {
Row outputRow = new OverlayingRow(newRow, transform.rowType);
store.writeIndexRows(session, transform.rowType.table(), outputRow, transform.groupIndexes);
}
}
break;
case TABLE:
if(transform.deleteOperator != null && transform.insertOperator != null) {
StoreAdapter adapter = store.createAdapter(session);
context = new SimpleQueryContext(adapter);
QueryBindings bindings = context.createBindings();
if (doDelete) {
bindings.setRow(OperatorAssembler.CREATE_AS_BINDING_POSITION, oldRow);
try {
runPlan(context, transform.deleteOperator, bindings);
} catch (NoSuchRowException e) {
LOG.debug("row not present: {}", oldRow);
}
}
if (doWrite) {
bindings.setRow(OperatorAssembler.CREATE_AS_BINDING_POSITION,
transformRow(context, bindings, transform, newRow));
try {
runPlan(context, transform.insertOperator, bindings);
} catch (NoSuchRowException e) {
LOG.debug("row not present: {}", newRow);
}
}
break;
}
case GROUP:
StoreAdapter adapter = store.createAdapter(session);
context = new SimpleQueryContext(adapter);
QueryBindings bindings = context.createBindings();
if(doDelete) {
Row newOldRow = transformRow(context, bindings, transform, oldRow);
try {
adapter.deleteRow(newOldRow, false);
} catch(NoSuchRowException e) {
LOG.debug("row not present: {}", newOldRow);
}
}
if(doWrite) {
Row newNewRow = transformRow(context, bindings, transform, newRow);
adapter.writeRow(newNewRow, transform.tableIndexes, transform.groupIndexes);
}
break;
}
transform.hKeySaver.save(schemaManager, session, hKey);
}
private TransformCache getTransformCache(final Session session, final ServerSession server) {
AkibanInformationSchema ais = schemaManager.getAis(session);
TransformCache cache = ais.getCachedValue(TRANSFORM_CACHE_KEY, null);
if(cache == null) {
cache = ais.getCachedValue(TRANSFORM_CACHE_KEY, new CacheValueGenerator<TransformCache>() {
@Override
public TransformCache valueFor(AkibanInformationSchema ais) {
TransformCache cache = new TransformCache();
TypesTranslator typesTranslator = schemaManager.getTypesTranslator();
Collection<OnlineChangeState> states = schemaManager.getOnlineChangeStates(session);
for(OnlineChangeState s : states) {
buildTransformCache(cache, s.getChangeSets(), ais, s.getAIS(), typesRegistry, typesTranslator, session, server, store);
}
return cache;
}
});
}
return cache;
}
private TableTransform getConcurrentDMLTransform(Session session, Table table) {
if(!schemaManager.isOnlineActive(session, table.getTableId())) {
return null;
}
if(!withConcurrentDML) {
throw new NotAllowedByConfigException("DML during online DDL");
}
TableTransform transform = getTransformCache(session, null).get(table.getTableId());
if(isTransformedTable(transform, table)) {
return null;
}
return transform;
}
//
// Static
//
private static void buildTransformCache(TransformCache cache,
Collection<ChangeSet> changeSets,
AkibanInformationSchema oldAIS,
AkibanInformationSchema newAIS,
TypesRegistryService typesRegistry,
TypesTranslator typesTranslator,
Session session,
ServerSession server,
Store givenStore) {
final ChangeLevel changeLevel = commonChangeLevel(changeSets);
final Schema newSchema = SchemaCache.globalSchema(newAIS);
Plannable deletePlan = null;
Plannable insertPlan = null;
for(ChangeSet cs : changeSets) {
if(cs.hasSelectStatement()) {
SQLParser parser = server.getParser();
StatementNode insertStmt;
try {
insertStmt = parser.parseStatement("insert into " + newAIS.getTable(cs.getToTableId()).getName().toStringEscaped() + " " + cs.getSelectStatement());
} catch (StandardException e) {
throw new SQLParserInternalException(e);
}
StoreAdapter adapter = givenStore.createAdapter(session);
CreateAsCompiler compiler = new CreateAsCompiler(server, adapter, true, newAIS);
PlanContext planContext = new PlanContext(compiler);
BasePlannable insertResult = compiler.compile((DMLStatementNode) insertStmt, null, planContext);
insertPlan = insertResult.getPlannable();
deletePlan = new Delete_Returning(insertPlan.getInputOperators().iterator().next(), false);
}
int tableID = cs.getTableId();
TableRowType newType = newSchema.tableRowType(tableID);
TableTransform transform = buildTableTransform(cs, changeLevel, oldAIS, newType, typesRegistry,
typesTranslator, (Operator)deletePlan, (Operator)insertPlan);
TableTransform prev = cache.put(tableID, transform);
assert (prev == null) : tableID;
}
}
private static void runPlan(Session session,
QueryContext context,
SchemaManager schemaManager,
TransactionService txnService,
Operator plan,
RowHandler handler) {
LOG.debug("Running online plan: {}", plan);
Map<RowType,HKeyChecker> checkers = new HashMap<>();
QueryBindings bindings = context.createBindings();
Cursor cursor = API.cursor(plan, context, bindings);
Rebindable rebindable = getRebindable(cursor);
cursor.openTopLevel();
try {
boolean done = false;
Row lastCommitted = null;
boolean checkOnlineError = true;
long rowCount = 0;
while(!done) {
Row row = cursor.next();
boolean didCommit = false;
boolean didRollback = false;
if(checkOnlineError) {
// Checked once per transaction here and in final phase in DDLFunctions
checkOnlineError(session, schemaManager);
checkOnlineError = false;
}
if(row != null) {
rowCount++;
RowType rowType = row.rowType();
// No way to pre-populate this map as Operator#rowType() is optional and insufficient.
HKeyChecker checker = checkers.get(rowType);
if(checker == null) {
if(rowType.hasTable()) {
checker = new SchemaManagerChecker(rowType.table().getTableId());
} else {
checker = new FalseChecker();
}
checkers.put(row.rowType(), checker);
}
try {
if(handler != null) {
//TODO: Not correct but only option for createAs due to hidden PK
Key hKey = new Key (null, 2047);
row.hKey().copyTo(hKey);
if (!checker.contains(schemaManager, session, hKey)) {
handler.handleRow(row);
} else {
LOG.trace("skipped row: {}", row);
}
}
didCommit = txnService.periodicallyCommit(session);
} catch(InvalidOperationException e) {
if(!e.getCode().isRollbackClass()) {
throw e;
}
didRollback = true;
}
} else {
// Cursor exhausted, completely finished
didRollback = txnService.commitOrRetryTransaction(session);
done = didCommit = !didRollback;
if(didCommit) {
txnService.beginTransaction(session);
}
}
if(didCommit) {
LOG.debug("Committed up to row: {}: {} rows", row, rowCount);
checkOnlineError = true;
lastCommitted = row;
checkers.clear();
} else if(didRollback) {
LOG.debug("Rolling back to row: {}", lastCommitted);
checkOnlineError = true;
checkers.clear();
txnService.rollbackTransactionIfOpen(session);
txnService.beginTransaction(session);
cursor.closeTopLevel();
rebindable.rebind((lastCommitted == null) ? null : lastCommitted.hKey(), true);
cursor.openTopLevel();
}
}
} finally {
cursor.closeTopLevel();
}
}
private static void runPlan(QueryContext context,
Operator plan,
QueryBindings bindings) {
LOG.debug("Running online DML plan: {}", plan);
Map<RowType,HKeyChecker> checkers = new HashMap<>();
Cursor cursor = API.cursor(plan, context, bindings);
cursor.openTopLevel();//open up top cursor
try {
boolean done = false;
while(!done) {
Row row = cursor.next();
if(row != null) {
RowType rowType = row.rowType();
HKeyChecker checker = checkers.get(rowType);
if (checker == null) {
if (rowType.hasTable()) {
checker = new SchemaManagerChecker(rowType.table().getTableId());
} else {
checker = new FalseChecker();
}
checkers.put(row.rowType(), checker);
}
} else {
done = true;
}
}
} finally {
cursor.closeTopLevel();
}
}
private static Set<Table> findOldRoots(Collection<ChangeSet> changeSets,
AkibanInformationSchema oldAIS,
AkibanInformationSchema newAIS) {
Set<Table> oldRoots = new HashSet<>();
for(ChangeSet cs : changeSets) {
Table oldTable = oldAIS.getTable(cs.getTableId());
Table newTable = newAIS.getTable(cs.getTableId());
Table oldNewTable = oldAIS.getTable(newTable.getTableId());
oldRoots.add(oldTable.getGroup().getRoot());
oldRoots.add(oldNewTable.getGroup().getRoot());
}
return oldRoots;
}
private static void checkOnlineError(Session session, SchemaManager sm) {
String msg = sm.getOnlineDMLError(session);
if(msg != null) {
throw new ConcurrentViolationException(msg);
}
}
/** Find all {@code ADD} or {@code MODIFY} group indexes referenced by {@code changeSets}. */
public static Collection<Index> findIndexesToBuild(Collection<ChangeSet> changeSets, AkibanInformationSchema ais) {
// There may be duplicates (e.g. every table has a GI it participates in)
Collection<Index> newIndexes = new HashSet<>();
for(ChangeSet cs : changeSets) {
Table table = ais.getTable(cs.getTableId());
for(IndexChange ic : cs.getIndexChangeList()) {
ChangeType changeType = ChangeType.valueOf(ic.getChange().getChangeType());
if(changeType == ChangeType.ADD || changeType == ChangeType.MODIFY) {
String name = ic.getChange().getNewName();
final Index index;
switch(IndexType.valueOf(ic.getIndexType())) {
case TABLE:
index = table.getIndexIncludingInternal(name);
break;
case FULL_TEXT:
index = table.getFullTextIndex(name);
break;
case GROUP:
index = table.getGroup().getIndex(name);
break;
default:
throw new IllegalStateException(ic.getIndexType());
}
assert index != null : ic;
newIndexes.add(index);
}
}
}
return newIndexes;
}
/** Find all {@code ADD} or {@code MODIFY} table indexes from {@code changeSet}. */
private static Collection<TableIndex> findTableIndexesToBuild(ChangeSet changeSet, Table newTable) {
if(changeSet == null) {
return Collections.emptyList();
}
List<TableIndex> tableIndexes = new ArrayList<>();
for(IndexChange ic : changeSet.getIndexChangeList()) {
if(IndexType.TABLE.name().equals(ic.getIndexType())) {
switch(ChangeType.valueOf(ic.getChange().getChangeType())) {
case ADD:
case MODIFY:
TableIndex index = newTable.getIndexIncludingInternal(ic.getChange().getNewName());
assert (index != null) : newTable.toString() + "," + ic;
tableIndexes.add(index);
break;
}
}
}
return tableIndexes;
}
/** Find all {@code ADD} or {@code MODIFY} group indexes from {@code changeSet}. */
private static Collection<GroupIndex> findGroupIndexesToBuild(ChangeSet changeSet, Table newTable) {
if(changeSet == null) {
return Collections.emptyList();
}
List<GroupIndex> groupIndexes = new ArrayList<>();
Group group = newTable.getGroup();
for(IndexChange ic : changeSet.getIndexChangeList()) {
if(IndexType.GROUP.name().equals(ic.getIndexType())) {
switch(ChangeType.valueOf(ic.getChange().getChangeType())) {
case ADD:
case MODIFY:
GroupIndex index = group.getIndex(ic.getChange().getNewName());
assert index != null : ic;
groupIndexes.add(index);
break;
}
}
}
return groupIndexes;
}
/** Find {@code newColumn}'s position in {@code oldTable} or {@code null} if it wasn't present */
private static Integer findOldPosition(List<Change> columnChanges, Table oldTable, Column newColumn) {
String newName = newColumn.getName();
for(Change change : columnChanges) {
if(newName.equals(change.getNewName())) {
switch(ChangeType.valueOf(change.getChangeType())) {
case ADD:
return null;
case MODIFY:
Column oldColumn = oldTable.getColumn(change.getOldName());
assert oldColumn != null : newColumn;
return oldColumn.getPosition();
case DROP:
throw new IllegalStateException("Dropped new column: " + newName);
}
}
}
Column oldColumn = oldTable.getColumn(newName);
if((oldColumn == null) && newColumn.isAkibanPKColumn()) {
return null;
}
// Not in change list, must be an original column
assert oldColumn != null : newColumn;
return oldColumn.getPosition();
}
private static ProjectedTableRowType buildProjectedRowType(ChangeSet changeSet,
Table origTable,
RowType newRowType,
boolean isGroupChange,
TypesRegistryService typesRegistry,
TypesTranslator typesTranslator,
QueryContext origContext) {
Table newTable = newRowType.table();
final List<Column> newColumns = newTable.getColumnsIncludingInternal();
final List<TPreparedExpression> projections = new ArrayList<>(newColumns.size());
for(Column newCol : newColumns) {
Integer oldPosition = findOldPosition(changeSet.getColumnChangeList(), origTable, newCol);
TInstance newInst = newCol.getType();
if(oldPosition == null) {
projections.add(buildColumnDefault(newCol, typesRegistry, typesTranslator, origContext));
} else {
Column oldCol = origTable.getColumnsIncludingInternal().get(oldPosition);
TInstance oldInst = oldCol.getType();
TPreparedExpression pExp = new TPreparedField(oldInst, oldPosition);
if(!oldInst.equalsExcludingNullable(newInst)) {
TCast cast = typesRegistry.getCastsResolver().cast(oldInst.typeClass(), newInst.typeClass());
pExp = new TCastExpression(pExp, cast, newInst);
}
projections.add(pExp);
}
}
return new ProjectedTableRowType(newRowType.schema(), newTable, projections, true);
}
// This should be quite similar to ExpressionAssembler#assembleColumnDefault()
private static TPreparedExpression buildColumnDefault(Column newCol,
TypesRegistryService typesRegistry,
TypesTranslator typesTranslator,
QueryContext origContext) {
return PlanGenerator.generateDefaultExpression(newCol,
null,
typesRegistry,
typesTranslator,
origContext);
}
private static TableTransform buildTableTransform(ChangeSet changeSet,
ChangeLevel changeLevel,
AkibanInformationSchema oldAIS,
TableRowType newRowType,
TypesRegistryService typesRegistry,
TypesTranslator typesTranslator,
Operator deleteOperator,
Operator insertOperator) {
Table newTable = newRowType.table();
Collection<TableIndex> tableIndexes = findTableIndexesToBuild(changeSet, newTable);
Collection<GroupIndex> groupIndexes = findGroupIndexesToBuild(changeSet, newTable);
ProjectedTableRowType projectedRowType = null;
boolean checkConstraints = false;
switch(changeLevel) {
case METADATA_CONSTRAINT:
case INDEX_CONSTRAINT:
checkConstraints = true;
assert groupIndexes.isEmpty() : groupIndexes;
break;
case TABLE:
if(deleteOperator != null && insertOperator != null) break;
case GROUP:
Table oldTable = oldAIS.getTable(newTable.getTableId());
if((changeSet.getColumnChangeCount() > 0) ||
(newRowType.nFields() != oldTable.getColumnsIncludingInternal().size())) {
projectedRowType = buildProjectedRowType(changeSet,
oldTable,
newRowType,
changeLevel == ChangeLevel.GROUP,
typesRegistry,
typesTranslator,
new SimpleQueryContext());
}
break;
}
return new TableTransform(changeLevel,
new SchemaManagerSaver(changeSet.getTableId()),
newRowType,
projectedRowType,
checkConstraints,
tableIndexes,
groupIndexes,
deleteOperator,
insertOperator);
}
/**
* NB: Current usage is *only* with plans that have GroupScan at the bottom. Use this fact to find the bottom,
* which can rebind(), for when periodicCommit() fails.
*/
private static Rebindable getRebindable(Cursor cursor) {
Cursor toRebind = cursor;
while(toRebind instanceof ChainedCursor) {
toRebind = ((ChainedCursor)toRebind).getInput();
}
if(!(toRebind instanceof Rebindable))
return null;
return (Rebindable)toRebind;
}
private static QueryContext contextIfNull(QueryContext context, StoreAdapter adapter) {
if(context == null) {
return new SimpleQueryContext(adapter);
}
assert(context.getSession() != null);
return new DelegatingContext(adapter, context);
}
public static ChangeLevel commonChangeLevel(Collection<ChangeSet> changeSets) {
ChangeLevel level = null;
for(ChangeSet cs : changeSets) {
if(level == null) {
level = ChangeLevel.valueOf(cs.getChangeLevel());
} else if(!level.name().equals(cs.getChangeLevel())) {
throw new IllegalStateException("Mixed ChangeLevels: " + changeSets);
}
}
assert (level != null);
return level;
}
/** Check if {@code table} is the post-transform/online DDL state. Use to avoid skip double-handling a row. */
private static boolean isTransformedTable(TableTransform transform, Table table) {
return (transform.rowType.table() == table);
}
private static Row transformRow(QueryContext context,
QueryBindings bindings,
TableTransform transform,
Row origRow) {
final Row newRow;
if(transform.projectedRowType != null) {
List<? extends TPreparedExpression> pProjections = transform.projectedRowType.getProjections();
newRow = new ProjectedRow(transform.projectedRowType,
origRow,
context,
bindings,
ProjectedRow.createTEvaluatableExpressions(pProjections)
);
} else {
newRow = new OverlayingRow(origRow, transform.rowType);
}
return newRow;
}
//
// Classes
//
private interface RowHandler {
void handleRow(Row row);
}
/**
* Helper for saving concurrently handled rows.
* Concrete implementations *must* be thread safe.
*/
private interface HKeySaver
{
void save(SchemaManager sm, Session session, Key hKey);
}
/**
* Helper for checking for concurrently handled rows.
* Must *only* be called with increasing hKeys and thrown away when the transaction closes.
*/
private interface HKeyChecker
{
boolean contains(SchemaManager sm, Session session, Key hKey);
}
private static class SchemaManagerSaver implements HKeySaver
{
private final int tableID;
private SchemaManagerSaver(int tableID) {
this.tableID = tableID;
}
@Override
public void save(SchemaManager sm, Session session, Key hKey) {
sm.addOnlineHandledHKey(session, tableID, hKey);
}
}
private static class SchemaManagerChecker implements HKeyChecker
{
private final int tableID;
private Iterator<byte[]> iter;
private KeyState last;
private SchemaManagerChecker(int tableID) {
this.tableID = tableID;
}
private void advance() {
byte[] bytes = iter.next();
last = (bytes != null) ? new KeyState(bytes) : null;
}
@Override
public boolean contains(SchemaManager sm, Session session, Key hKey) {
if(iter == null) {
iter = sm.getOnlineHandledHKeyIterator(session, tableID, hKey);
advance();
}
// Can scan until we reach, or go past, hKey. If past, can't skip.
while(last != null) {
int ret = last.compareTo(hKey);
if(ret == 0) {
return true; // Match
}
if(ret > 0) {
return false; // last from iterator is ahead of hKey
}
advance();
}
// Iterator exhausted: no more to skip
return false;
}
}
private static class FalseChecker implements HKeyChecker
{
@Override
public boolean contains(SchemaManager sm, Session session, Key hKey) {
return false;
}
}
/** Holds information about how to maintain/populate the new/modified instance of a table. */
private static class TableTransform {
public final ChangeLevel changeLevel;
/** Target for concurrently handled DML. */
public final HKeySaver hKeySaver;
/** New row type for the table. */
public final TableRowType rowType;
/** Not {@code null} *iff* new rows need projected. */
public final ProjectedTableRowType projectedRowType;
/** Not {@code null} *iff* new rows need only be verified. */
public final boolean checkConstraints;
/** Contains table indexes to build (can be empty) */
public final Collection<TableIndex> tableIndexes;
/** Populated with group indexes to build (can be empty) */
public final Collection<GroupIndex> groupIndexes;
/** Used for CreateTableAs */
public Operator deleteOperator;
public Operator insertOperator;
public TableTransform(ChangeLevel changeLevel,
HKeySaver hKeySaver,
TableRowType rowType,
ProjectedTableRowType projectedRowType,
boolean checkConstraints,
Collection<TableIndex> tableIndexes,
Collection<GroupIndex> groupIndexes,
Operator deleteOperator ,
Operator insertOperator) {
this.changeLevel = changeLevel;
this.hKeySaver = hKeySaver;
this.rowType = rowType;
this.projectedRowType = projectedRowType;
this.checkConstraints = checkConstraints;
this.tableIndexes = tableIndexes;
this.groupIndexes = groupIndexes;
this.deleteOperator = deleteOperator;
this.insertOperator = insertOperator;
}
}
/** Table ID -> TableTransform */
private static class TransformCache extends HashMap<Integer,TableTransform>
{
}
}