/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.facebook.presto.operator;
import com.facebook.presto.spi.Page;
import com.facebook.presto.spi.type.Type;
import com.facebook.presto.sql.gen.JoinFilterFunctionCompiler.JoinFilterFunctionFactory;
import com.facebook.presto.sql.planner.Symbol;
import com.facebook.presto.sql.planner.plan.PlanNodeId;
import com.google.common.collect.ImmutableList;
import com.google.common.util.concurrent.ListenableFuture;
import javax.annotation.concurrent.ThreadSafe;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkState;
import static com.google.common.collect.ImmutableList.toImmutableList;
import static java.util.Objects.requireNonNull;
@ThreadSafe
public class HashBuilderOperator
implements Operator
{
public static class HashBuilderOperatorFactory
implements OperatorFactory
{
private final int operatorId;
private final PlanNodeId planNodeId;
private final PartitionedLookupSourceFactory lookupSourceFactory;
private final List<Integer> outputChannels;
private final List<Integer> hashChannels;
private final Optional<Integer> preComputedHashChannel;
private final Optional<JoinFilterFunctionFactory> filterFunctionFactory;
private final PagesIndex.Factory pagesIndexFactory;
private final int expectedPositions;
private int partitionIndex;
private boolean closed;
public HashBuilderOperatorFactory(
int operatorId,
PlanNodeId planNodeId,
List<Type> types,
List<Integer> outputChannels,
Map<Symbol, Integer> layout,
List<Integer> hashChannels,
Optional<Integer> preComputedHashChannel,
boolean outer,
Optional<JoinFilterFunctionFactory> filterFunctionFactory,
int expectedPositions,
int partitionCount,
PagesIndex.Factory pagesIndexFactory)
{
this.operatorId = operatorId;
this.planNodeId = requireNonNull(planNodeId, "planNodeId is null");
checkArgument(Integer.bitCount(partitionCount) == 1, "partitionCount must be a power of 2");
lookupSourceFactory = new PartitionedLookupSourceFactory(
types,
outputChannels.stream()
.map(types::get)
.collect(toImmutableList()),
hashChannels,
partitionCount,
requireNonNull(layout, "layout is null"),
outer);
this.outputChannels = ImmutableList.copyOf(requireNonNull(outputChannels, "outputChannels is null"));
this.hashChannels = ImmutableList.copyOf(requireNonNull(hashChannels, "hashChannels is null"));
this.preComputedHashChannel = requireNonNull(preComputedHashChannel, "preComputedHashChannel is null");
this.filterFunctionFactory = requireNonNull(filterFunctionFactory, "filterFunctionFactory is null");
this.pagesIndexFactory = requireNonNull(pagesIndexFactory, "pagesIndexFactory is null");
this.expectedPositions = expectedPositions;
}
public LookupSourceFactory getLookupSourceFactory()
{
return lookupSourceFactory;
}
@Override
public List<Type> getTypes()
{
return lookupSourceFactory.getTypes();
}
@Override
public Operator createOperator(DriverContext driverContext)
{
checkState(!closed, "Factory is already closed");
OperatorContext operatorContext = driverContext.addOperatorContext(operatorId, planNodeId, HashBuilderOperator.class.getSimpleName());
HashBuilderOperator operator = new HashBuilderOperator(
operatorContext,
lookupSourceFactory,
partitionIndex,
outputChannels,
hashChannels,
preComputedHashChannel,
filterFunctionFactory,
expectedPositions,
pagesIndexFactory);
partitionIndex++;
return operator;
}
@Override
public void close()
{
closed = true;
}
@Override
public OperatorFactory duplicate()
{
throw new UnsupportedOperationException("Parallel hash build can not be duplicated");
}
}
private final OperatorContext operatorContext;
private final PartitionedLookupSourceFactory lookupSourceFactory;
private final int partitionIndex;
private final List<Integer> outputChannels;
private final List<Integer> hashChannels;
private final Optional<Integer> preComputedHashChannel;
private final Optional<JoinFilterFunctionFactory> filterFunctionFactory;
private final PagesIndex index;
private boolean finishing;
private final HashCollisionsCounter hashCollisionsCounter;
public HashBuilderOperator(
OperatorContext operatorContext,
PartitionedLookupSourceFactory lookupSourceFactory,
int partitionIndex,
List<Integer> outputChannels,
List<Integer> hashChannels,
Optional<Integer> preComputedHashChannel,
Optional<JoinFilterFunctionFactory> filterFunctionFactory,
int expectedPositions,
PagesIndex.Factory pagesIndexFactory)
{
requireNonNull(pagesIndexFactory, "pagesIndexFactory is null");
this.operatorContext = operatorContext;
this.partitionIndex = partitionIndex;
this.filterFunctionFactory = filterFunctionFactory;
this.index = pagesIndexFactory.newPagesIndex(lookupSourceFactory.getTypes(), expectedPositions);
this.lookupSourceFactory = lookupSourceFactory;
this.outputChannels = outputChannels;
this.hashChannels = hashChannels;
this.preComputedHashChannel = preComputedHashChannel;
this.hashCollisionsCounter = new HashCollisionsCounter(operatorContext);
operatorContext.setInfoSupplier(hashCollisionsCounter);
}
@Override
public OperatorContext getOperatorContext()
{
return operatorContext;
}
@Override
public List<Type> getTypes()
{
return lookupSourceFactory.getTypes();
}
@Override
public void finish()
{
if (finishing) {
return;
}
finishing = true;
LookupSourceSupplier partition = index.createLookupSourceSupplier(operatorContext.getSession(), hashChannels, preComputedHashChannel, filterFunctionFactory, Optional.of(outputChannels));
lookupSourceFactory.setPartitionLookupSourceSupplier(partitionIndex, partition);
operatorContext.setMemoryReservation(partition.get().getInMemorySizeInBytes());
hashCollisionsCounter.recordHashCollision(partition.getHashCollisions(), partition.getExpectedHashCollisions());
}
@Override
public boolean isFinished()
{
return finishing && lookupSourceFactory.isDestroyed().isDone();
}
@Override
public boolean needsInput()
{
return !finishing;
}
@Override
public ListenableFuture<?> isBlocked()
{
if (!finishing) {
return NOT_BLOCKED;
}
return lookupSourceFactory.isDestroyed();
}
@Override
public void addInput(Page page)
{
requireNonNull(page, "page is null");
checkState(!isFinished(), "Operator is already finished");
index.addPage(page);
if (!operatorContext.trySetMemoryReservation(index.getEstimatedSize().toBytes())) {
index.compact();
}
operatorContext.setMemoryReservation(index.getEstimatedSize().toBytes());
operatorContext.recordGeneratedOutput(page.getSizeInBytes(), page.getPositionCount());
}
@Override
public Page getOutput()
{
return null;
}
}