/**
* diqube: Distributed Query Base.
*
* Copyright (C) 2015 Bastian Gloeckle
*
* This file is part of diqube.
*
* diqube is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
package org.diqube.execution.steps;
import java.util.Arrays;
import java.util.Comparator;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.NavigableSet;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.ConcurrentLinkedDeque;
import java.util.concurrent.ConcurrentSkipListSet;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.function.Consumer;
import java.util.stream.Collectors;
import org.diqube.data.column.StandardColumnShard;
import org.diqube.execution.ColumnVersionBuiltHelper;
import org.diqube.execution.consumers.AbstractThreadedColumnBuiltConsumer;
import org.diqube.execution.consumers.AbstractThreadedColumnVersionBuiltConsumer;
import org.diqube.execution.consumers.AbstractThreadedRowIdConsumer;
import org.diqube.execution.consumers.ColumnBuiltConsumer;
import org.diqube.execution.consumers.ColumnVersionBuiltConsumer;
import org.diqube.execution.consumers.DoneConsumer;
import org.diqube.execution.consumers.GenericConsumer;
import org.diqube.execution.consumers.OrderedRowIdConsumer;
import org.diqube.execution.consumers.RowIdConsumer;
import org.diqube.execution.exception.ExecutablePlanBuildException;
import org.diqube.executionenv.ExecutionEnvironment;
import org.diqube.executionenv.VersionedExecutionEnvironment;
import org.diqube.executionenv.querystats.QueryableColumnShard;
import org.diqube.queries.QueryRegistry;
import org.diqube.util.ArrayViewLongList;
import org.diqube.util.Pair;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.base.Function;
import com.google.common.collect.Iterables;
import com.google.common.collect.Sets;
/**
* Orders the incoming row IDs by the values of potentially multiple columns, ascending or descending.
*
* <p>
* Any order step can have a LIMIT set which will cut off the results after the amount of rows. If a limit is specified,
* a limitStart may be specified, denoting the first index of the row IDs to return. If <b>both</b> these values are
* unset, <i>softLimit</i> may be set. A soft limit is needed for cluster nodes which might be able to order only by a
* subset of the columns that actually need to be ordered according to the users request. That might happen if the ORDER
* BY contained a column which is based on a group aggregation function - as remotes do not have the final grouped
* aggregation values available, they obviously cannot order by them. We nevertheless want to faciliate using limits as
* well as possible also on cluster nodes already to reduce the data transfers to the query master and the memory
* consumption on the query master. Therefore there is a soft limit, which will cut off any results, too, following
* these requirements:
*
* <ul>
* <li>If softLimit is set, this step will try to return only softLimit number of rows, just like when LIMIT is set.
* <li>If, though, there are any rows after sorting which would be cut off, but whose values of the ordered columns is
* equal to any of the rows that are not cut off, then these rows will be contained in the result, too. This then
* enables these "equal" rows to be sorted further by the query master and then being cut off correctly.
* <li>As the rowIDs are ordered already before executing the cut-off, this step only inspects the last row that is not
* cut-off and compares that to any rows marked for cut-off - if there are equal rows, they will be included.
* </ul>
*
* <p>
* When this step is executed on the query master, there will usually be (at least) one
* {@link ColumnVersionBuiltConsumer} wired. This means that the ordering will take place on the intermediary column
* values of a {@link VersionedExecutionEnvironment}. In that case, the actual cut-off by any limits is not executed
* while working on those intermediary column, but only when the columns have been built fully. This is because each
* value in each of the interesting columns might change arbitrarily in a intermediary column and we therefore cannot
* guarantee that a rowId that looks like not being cut-off in the first run, might get values that would force us to
* not cut-off that row ID in a later execution.
*
* <p>
* The columns that are ordered by are expected to be {@link StandardColumnShard}s.
*
* <p>
* Input: 1 {@link RowIdConsumer}, optionally multiple {@link ColumnBuiltConsumer}s, optionally multiple
* {@link ColumnVersionBuiltConsumer}s <br>
* Output: {@link RowIdConsumer} and/or {@link OrderedRowIdConsumer}
*
* @author Bastian Gloeckle
*/
public class OrderStep extends AbstractThreadedExecutablePlanStep {
private static final Logger logger = LoggerFactory.getLogger(OrderStep.class);
private AtomicBoolean columnBuiltInputIsDone = new AtomicBoolean(false);
/**
* Interesting only, if ColumnBuiltConsumer is wired. Then it contains the names of the columns we still wait for to
* be fully built.
*/
private Set<String> columnsThatNeedToBeBuilt;
/**
* <code>true</code> when all columns this step is waiting for have been built and are available in
* {@link #defaultEnv} .
*/
private AtomicBoolean allColumnsBuilt = new AtomicBoolean(false);
private AbstractThreadedColumnBuiltConsumer columnBuiltConsumer = new AbstractThreadedColumnBuiltConsumer(this) {
@Override
protected void doColumnBuilt(String colName) {
columnsThatNeedToBeBuilt.remove(colName);
if (columnsThatNeedToBeBuilt.isEmpty())
allColumnsBuilt.set(true);
}
@Override
protected void allSourcesAreDone() {
columnBuiltInputIsDone.set(true);
}
};
/**
* The newest available {@link VersionedExecutionEnvironment} which should be used to query values of columns. This
* will be set only if a {@link ColumnVersionBuiltConsumer} is wired.
*/
private VersionedExecutionEnvironment newestTemporaryEnv = null;
private AbstractThreadedColumnVersionBuiltConsumer columnVersionBuiltConsumer =
new AbstractThreadedColumnVersionBuiltConsumer(this) {
@Override
protected void allSourcesAreDone() {
// we rely on ColumnBuiltConsumer to report the final build.
}
@Override
protected synchronized void doColumnBuilt(VersionedExecutionEnvironment env, String colName,
Set<Long> adjustedRowIds) {
if (newestTemporaryEnv == null)
newestTemporaryEnv = env;
else if (newestTemporaryEnv.getVersion() < env.getVersion())
newestTemporaryEnv = env;
}
};
private AtomicBoolean rowIdSourceIsEmpty = new AtomicBoolean(false);
/** input rowIDs as reported by input {@link RowIdConsumer}. */
private ConcurrentLinkedDeque<Long> rowIds = new ConcurrentLinkedDeque<>();
private AbstractThreadedRowIdConsumer rowIdConsumer = new AbstractThreadedRowIdConsumer(this) {
@Override
public void allSourcesAreDone() {
OrderStep.this.rowIdSourceIsEmpty.set(true);
}
@Override
protected void doConsume(Long[] rowIds) {
for (long rowId : rowIds)
OrderStep.this.rowIds.add(rowId);
}
};
/**
* current version of sorted row IDs. This array may be longer than how many values are actually available. See
* {@link #sortedRowIdsLength}.
*/
private Long[] sortedRowIds = new Long[0];
/** Number of valid entries in {@link #sortedRowIds}. See {@link #resizeArrayForLength(Long[], int)}. */
private int sortedRowIdsLength = 0;
/**
* maximum value of {@link #sortedRowIdsLength} that we should consider when taking into account the LIMIT and
* {@link #limitStart}. Only valid of {@link #softLimit} is <code>null</code>!
*/
private Long sortedRowIdsMaxLength;
private Long limitStart;
private Long softLimit;
/**
* Creates a {@link SortComparator} that sorts according to the requested parameters. This can be called as soon as
* all columns that this {@link OrderStep} is interested have been created (= {@link #columnBuiltConsumer} in the
* given {@link ExecutionEnvironment}
*/
private Function<ExecutionEnvironment, SortComparator> headComparatorProvider;
/**
* List of columns to sort by, just like the user specified it in the query. Pair is col Name to boolean (true if ASC,
* false if DESC).
*/
private List<Pair<String, Boolean>> sortCols;
/** A Set with just the names of the columns to sort by (unordered). */
private Set<String> sortColSet;
private ExecutionEnvironment defaultEnv;
/**
* row IDs that were reported as active by the input {@link RowIdConsumer}, but which have not yet been inspected
* because there are no values available for all columns at these rows. This is only interesting if there are
* {@link ColumnVersionBuiltConsumer}s wired (= only on query master).
*/
private NavigableSet<Long> notYetProcessedRowIds = new TreeSet<>();
/**
*
* @param sortCols
* List of Pairs of which columns should be sorted after (in that order!). Left side of the pair is the
* column name, right side is <code>true</code> if sorted should be ascending, <code>false</code> if
* descending.
* @param limit
* if not <code>null</code>, a limit on the resulting set of row IDs will be applied, that means at most that
* many rows will be returned (taking into account the ordering, of course).
* @param limitStart
* if not <code>null</code>, not the _first_ (limit) row IDs will be returned, but the ones starting at index
* limitStart. This cannot be set, if limit is <code>null</code>.
* @param softLimit
* If both limit and limitStart are null, this field might be set, otherwise it needs to be <code>null</code>
* . For a description, see class comment.
*/
public OrderStep(int stepId, QueryRegistry queryRegistry, ExecutionEnvironment defaultEnv,
List<Pair<String, Boolean>> sortCols, Long limit, Long limitStart, Long softLimit) {
super(stepId, queryRegistry);
this.defaultEnv = defaultEnv;
this.sortCols = sortCols;
this.limitStart = limitStart;
this.softLimit = softLimit;
if (limit != null) {
sortedRowIdsMaxLength = limit;
if (limitStart != null)
sortedRowIdsMaxLength += limitStart;
}
}
@Override
public void initialize() {
sortColSet = sortCols.stream().map(p -> p.getLeft()).collect(Collectors.toSet());
columnsThatNeedToBeBuilt = new ConcurrentSkipListSet<>(sortColSet);
for (Iterator<String> it = columnsThatNeedToBeBuilt.iterator(); it.hasNext();)
if (defaultEnv.getColumnShard(it.next()) != null)
it.remove();
// factory method for comparators based on a specific env.
headComparatorProvider = (executionEnvironment) -> {
SortComparator headComparator = null;
SortComparator lastComparator = null;
for (Pair<String, Boolean> sortCol : sortCols) {
String colName = sortCol.getLeft();
boolean sortAsc = sortCol.getRight();
QueryableColumnShard column = executionEnvironment.getColumnShard(colName);
ColumnValueIdResolver resolver = (rowId) -> column.resolveColumnValueIdForRow(rowId);
SortComparator newComparator = new SortComparator(resolver, sortAsc);
if (lastComparator != null)
lastComparator.setDelegateComparatorOnEqual(newComparator);
else
headComparator = newComparator;
lastComparator = newComparator;
}
return headComparator;
};
}
@Override
protected void validateOutputConsumer(GenericConsumer consumer) throws IllegalArgumentException {
if (!(consumer instanceof DoneConsumer) && !(consumer instanceof OrderedRowIdConsumer)
&& !(consumer instanceof RowIdConsumer))
throw new IllegalArgumentException("Only OrderedRowIdConsumer and RowIdConsumer accepted.");
}
@Override
protected void execute() {
// intermediateRun = true if NOT all final versions of all columns have been built and are available in defaultEnv
// -> we only have intermediary values!
boolean intermediateRun = !(columnBuiltConsumer.getNumberOfTimesWired() == 0 || allColumnsBuilt.get());
if (columnBuiltConsumer.getNumberOfTimesWired() > 0 && columnBuiltInputIsDone.get() && !allColumnsBuilt.get()) {
logger.debug("Ordering needs to wait for a column to be built, but it won't be built. Skipping.");
forEachOutputConsumerOfType(GenericConsumer.class, c -> c.sourceIsDone());
doneProcessing();
return;
}
ExecutionEnvironment env;
if (!intermediateRun) {
env = defaultEnv;
} else {
env = newestTemporaryEnv;
if (env == null)
return;
// we'll be sorting on a version of the Env that contains only intermediate columns. Check if we have at least
// some values for all interesting columns.
boolean allColsAvailable = sortColSet.stream().allMatch(colName -> env.getColumnShard(colName) != null);
if (!allColsAvailable)
return;
}
// new row IDs we ought to order into the result.
NavigableSet<Long> activeRowIdsSet = new TreeSet<>();
Long tmpNextRowId;
while ((tmpNextRowId = rowIds.poll()) != null)
activeRowIdsSet.add(tmpNextRowId);
SortComparator headComparator = headComparatorProvider.apply(env);
if (intermediateRun) {
// Make sure that we only order those rows, where we have values for all columns.
// Please note the following:
// We only make sure that each column contains /any/ value on the row IDs, these might be as well default values
// filled in by SparseColumnShardBuilder! We therefore might order based on "wrong" values here. But this is not
// as important, because as soon as we have correct values and the orderStep is executed again (either still in
// 'intermediary' mode or in 'isLastRun' mode) the row will be ordered correctly.
new ColumnVersionBuiltHelper().publishActiveRowIds(env, sortColSet, activeRowIdsSet, notYetProcessedRowIds);
} else {
if (notYetProcessedRowIds.size() > 0) {
activeRowIdsSet.addAll(notYetProcessedRowIds);
notYetProcessedRowIds.clear();
}
}
logger.trace(
"Starting to order based on Env {}, having active RowIDs (limt) {}, not yet processed (limit) {}. intermediateRun: {}",
env, Iterables.limit(activeRowIdsSet, 100), Iterables.limit(notYetProcessedRowIds, 100), intermediateRun);
if (activeRowIdsSet.size() > 0) {
Long[] activeRowIds = activeRowIdsSet.toArray(new Long[activeRowIdsSet.size()]);
// be sure that we have enough space in the array
sortedRowIds = resizeArrayForLength(sortedRowIds, sortedRowIdsLength + activeRowIds.length);
// add new values to the array and use insertion sort to put them at the right sorted locations
System.arraycopy(activeRowIds, 0, sortedRowIds, sortedRowIdsLength, activeRowIds.length);
sortedRowIdsLength += activeRowIds.length;
}
// Use default JVM sorting, which implements a TimSort at least in OpenJDK - this executes very well even on
// partly sorted arrays (which is [mostly] true in our case if the execute method is executed at least twice).
// -
// Execute this in each run. This is needed, as either there were new rowIds added or some rows changed their values
// (otherwise execute() would not be called). Therefore we always need to sort the array.
// TODO #8 support sorting only /some/ elements in case we're based on intermediary values.
Arrays.sort(sortedRowIds, 0, sortedRowIdsLength, headComparator);
// cutOffPoint = first index in sortedRowIds that would be cut off by a limit/softLimit clause. null otherwise.
Integer cutOffPoint = null;
// Find cut off point according to limit/softLimit.
if (softLimit == null) {
if (sortedRowIdsMaxLength != null && sortedRowIdsLength > sortedRowIdsMaxLength) {
// we have a LIMIT set but our sorted array is already longer. Cut its length to save some memory.
// remember point to cut off at
cutOffPoint = sortedRowIdsMaxLength.intValue();
}
} else {
if (sortedRowIdsLength > softLimit) {
// we are above the soft limit!
Long lastContainedRowId = sortedRowIds[softLimit.intValue() - 1];
int softLength = softLimit.intValue();
// linearily search those rows which should be included in the result although they would have been cut-off.
while (softLength < sortedRowIdsLength
&& headComparator.compare(lastContainedRowId, sortedRowIds[softLength]) == 0)
softLength++;
logger.trace("Hit soft limit {}, using length {}", softLimit.intValue(), softLength);
// remember point to cut off at
cutOffPoint = softLength;
}
}
// inform RowIdConsumers about new row IDs
if (activeRowIdsSet.size() > 0) {
if (intermediateRun) {
// if this is an intermediary run, make sure that we report all rowIds to subsequent steps, as we did not
// actually execute the cut-off.
Long[] rowIds = activeRowIdsSet.stream().toArray(l -> new Long[l]);
forEachOutputConsumerOfType(RowIdConsumer.class, c -> c.consume(rowIds));
} else {
// This is a last run. This means that sorting is now based on the defaultEnv and that none of the columns might
// change its values anymore.
// If there were new rowIds, we need to report only those that actually are included inside
// the result value (and are not cut-off).
Long[] rowIdsToBeOutput;
if (cutOffPoint != null)
rowIdsToBeOutput = findRowIdsToBeOutputOnCutOff(activeRowIdsSet, cutOffPoint);
else
rowIdsToBeOutput = activeRowIdsSet.stream().toArray(l -> new Long[l]);
// report all row IDs so subsequent RowID consumers can handle them. We might report too much rowIDs here.
forEachOutputConsumerOfType(RowIdConsumer.class, c -> c.consume(rowIdsToBeOutput));
}
}
if (!intermediateRun && cutOffPoint != null)
// execute cut off if we're not in an intermediate run. While in an intermediate run we're based on arbitrary
// versions of the column (see ColumnVersionBuiltConsumer). This means that the values of all interesting rows
// in all columns might change arbitrarily. Therefore we cannot execute a cut-off in that case, as a row that
// we'd cut off now might change its value later on so we'd actually need to include it. On the other hand, not
// only that row might change its value, but all other rows might change their values and force that row to be
// inside the result set - but if we cut it off before, there's no chance to recover it. So we execute no
// cut-off in that case.
sortedRowIdsLength = cutOffPoint;
logger.trace("Ordering result (limit): {}",
Iterables.limit(Arrays.asList(sortedRowIds), Math.min(20, sortedRowIdsLength)));
final Integer finalIntendedCutOffPoint = cutOffPoint;
// output sorted result in each run - it might be that we did not have new row IDs, but the value of specific row
// IDs were changed and therefore we have a now ordering (after sorting above) - we should publicize that.
forEachOutputConsumerOfType(OrderedRowIdConsumer.class, new Consumer<OrderedRowIdConsumer>() {
@Override
public void accept(OrderedRowIdConsumer orderedConsumer) {
int startIdx;
int length;
if (limitStart != null) {
startIdx = limitStart.intValue();
length = sortedRowIdsLength - limitStart.intValue();
} else {
startIdx = 0;
length = sortedRowIdsLength;
}
if (intermediateRun && finalIntendedCutOffPoint != null) {
// we did not execute a cut-off, as we're in an intermediary run. So do at least a 'virtual cut-off' here, by
// only reporting those rows inside the intendedCutOffPoint to the steps interested in the ordering.
length -= sortedRowIdsLength - finalIntendedCutOffPoint;
}
if (length < 0 || startIdx >= sortedRowIds.length) {
// ensure that we do not pass on invalid values to ArrayViewLongList.
startIdx = 0;
length = 0;
}
orderedConsumer.consumeOrderedRowIds(new ArrayViewLongList(sortedRowIds, startIdx, length));
}
});
// check if we're done.
if (!intermediateRun && rowIdSourceIsEmpty.get() && rowIds.isEmpty()) {
forEachOutputConsumerOfType(GenericConsumer.class, c -> c.sourceIsDone());
doneProcessing();
}
}
/**
* Find those rowIDs that were in activeRowIdsSet, but which will be cut-off when using the given cut-off-point.
*
* <p>
* This needs to be executed before executing a cutOff (=adjusting {@link #sortedRowIdsLength}), as this method needs
* that value.
*
* @param activeRowIdsSet
* The source set of row IDs.
* @param cutOffPoint
* The first index in {@link #sortedRowIds} that will be cut-off.
* @return the row IDs to be reported as newly added. These will be all the longs in activeRowIdsSet which are NOT cut
* off.
*/
private Long[] findRowIdsToBeOutputOnCutOff(Set<Long> activeRowIdsSet, int cutOffPoint) {
Set<Long> rowIdsBeingCutOff = new HashSet<>();
for (int i = cutOffPoint; i < sortedRowIdsLength; i++)
rowIdsBeingCutOff.add(sortedRowIds[i]);
logger.trace("Cutting off {} results because of (soft) limit clause: (limt) {}", sortedRowIdsLength - cutOffPoint,
Iterables.limit(rowIdsBeingCutOff, 100));
return Sets.difference(activeRowIdsSet, rowIdsBeingCutOff).stream().toArray(l -> new Long[l]);
}
@Override
protected List<GenericConsumer> inputConsumers() {
return Arrays.asList(new GenericConsumer[] { rowIdConsumer, columnBuiltConsumer, columnVersionBuiltConsumer });
}
@Override
protected void validateWiredStatus() throws ExecutablePlanBuildException {
// there may be an arbitrary number of ColumnBuiltConsumers, but there has to be at least the RowIdConsumer
if (rowIdConsumer.getNumberOfTimesWired() == 0)
throw new ExecutablePlanBuildException("RowIdConsumer is not wired.");
}
private Long[] resizeArrayForLength(Long[] longArray, int requestedNewLength) {
if (longArray != null && requestedNewLength <= longArray.length)
return longArray;
int newLength;
int highestBit = Integer.highestOneBit(requestedNewLength);
if (highestBit == 0)
highestBit = 1;
if ((highestBit << 1) > requestedNewLength)
newLength = highestBit << 1;
else
newLength = Integer.MAX_VALUE;
Long[] resultArray = new Long[newLength];
if (longArray != null)
System.arraycopy(longArray, 0, resultArray, 0, longArray.length);
return resultArray;
}
/**
* Resolves a row ID to a Column value ID.
*/
private interface ColumnValueIdResolver {
public long resolveColumnValueId(long rowId);
}
/**
* A {@link Comparator} that compared the column value IDs of the rowIDs that are expected to be provided to the
* {@link #compare(Long, Long)} method. If the column value IDs are equal, the comparator may forward the decision to
* another {@link SortComparator} - this enables us to implement a detailed sorting using multiple columns after each
* other.
*/
private class SortComparator implements Comparator<Long> {
private ColumnValueIdResolver columnValueIdResolver;
private SortComparator delegateComparatorOnEqual = null;
private int orderFactor;
public SortComparator(ColumnValueIdResolver columnValueIdResolver, boolean sortAscending) {
this.columnValueIdResolver = columnValueIdResolver;
orderFactor = (sortAscending) ? 1 : -1;
}
@Override
public int compare(Long rowId1, Long rowId2) {
long colValue1 = columnValueIdResolver.resolveColumnValueId(rowId1);
long colValue2 = columnValueIdResolver.resolveColumnValueId(rowId2);
if (colValue1 == colValue2) {
if (delegateComparatorOnEqual != null)
return delegateComparatorOnEqual.compare(rowId1, rowId2);
return 0;
}
return orderFactor * Long.compare(colValue1, colValue2);
}
public void setDelegateComparatorOnEqual(SortComparator delegateComparatorOnEqual) {
this.delegateComparatorOnEqual = delegateComparatorOnEqual;
}
}
@Override
protected String getAdditionalToStringDetails() {
return "sortCols=" + sortCols;
}
}