/*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.facebook.presto.operator;
import com.facebook.presto.connector.ConnectorId;
import com.facebook.presto.execution.SystemMemoryUsageListener;
import com.facebook.presto.execution.buffer.PagesSerde;
import com.facebook.presto.execution.buffer.PagesSerdeFactory;
import com.facebook.presto.execution.buffer.SerializedPage;
import com.facebook.presto.metadata.Split;
import com.facebook.presto.spi.Page;
import com.facebook.presto.spi.UpdatablePageSource;
import com.facebook.presto.spi.type.Type;
import com.facebook.presto.split.RemoteSplit;
import com.facebook.presto.sql.planner.plan.PlanNodeId;
import com.google.common.util.concurrent.ListenableFuture;
import javax.annotation.concurrent.NotThreadSafe;
import java.io.Closeable;
import java.net.URI;
import java.util.List;
import java.util.Optional;
import java.util.function.Supplier;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkState;
import static java.util.Objects.requireNonNull;
public class ExchangeOperator
implements SourceOperator, Closeable
{
public static final ConnectorId REMOTE_CONNECTOR_ID = new ConnectorId("$remote");
public static class ExchangeOperatorFactory
implements SourceOperatorFactory
{
private final int operatorId;
private final PlanNodeId sourceId;
private final ExchangeClientSupplier exchangeClientSupplier;
private final PagesSerdeFactory serdeFactory;
private final List<Type> types;
private ExchangeClient exchangeClient = null;
private boolean closed;
public ExchangeOperatorFactory(
int operatorId,
PlanNodeId sourceId,
ExchangeClientSupplier exchangeClientSupplier,
PagesSerdeFactory serdeFactory,
List<Type> types)
{
this.operatorId = operatorId;
this.sourceId = sourceId;
this.exchangeClientSupplier = exchangeClientSupplier;
this.serdeFactory = serdeFactory;
this.types = types;
}
@Override
public PlanNodeId getSourceId()
{
return sourceId;
}
@Override
public List<Type> getTypes()
{
return types;
}
@Override
public SourceOperator createOperator(DriverContext driverContext)
{
checkState(!closed, "Factory is already closed");
OperatorContext operatorContext = driverContext.addOperatorContext(operatorId, sourceId, ExchangeOperator.class.getSimpleName());
if (exchangeClient == null) {
exchangeClient = exchangeClientSupplier.get(new UpdateSystemMemory(driverContext.getPipelineContext()));
}
return new ExchangeOperator(
operatorContext,
types,
sourceId,
serdeFactory.createPagesSerde(),
exchangeClient);
}
@Override
public void close()
{
closed = true;
}
}
@NotThreadSafe
private static final class UpdateSystemMemory
implements SystemMemoryUsageListener
{
private final PipelineContext pipelineContext;
public UpdateSystemMemory(PipelineContext pipelineContext)
{
this.pipelineContext = requireNonNull(pipelineContext, "pipelineContext is null");
}
@Override
public void updateSystemMemoryUsage(long deltaMemoryInBytes)
{
if (deltaMemoryInBytes > 0) {
pipelineContext.reserveSystemMemory(deltaMemoryInBytes);
}
else {
pipelineContext.freeSystemMemory(-deltaMemoryInBytes);
}
}
}
private final OperatorContext operatorContext;
private final PlanNodeId sourceId;
private final ExchangeClient exchangeClient;
private final List<Type> types;
private final PagesSerde serde;
public ExchangeOperator(
OperatorContext operatorContext,
List<Type> types,
PlanNodeId sourceId,
PagesSerde serde,
ExchangeClient exchangeClient)
{
this.operatorContext = requireNonNull(operatorContext, "operatorContext is null");
this.sourceId = requireNonNull(sourceId, "sourceId is null");
this.exchangeClient = requireNonNull(exchangeClient, "exchangeClient is null");
this.serde = requireNonNull(serde, "serde is null");
this.types = requireNonNull(types, "types is null");
operatorContext.setInfoSupplier(exchangeClient::getStatus);
}
@Override
public PlanNodeId getSourceId()
{
return sourceId;
}
@Override
public Supplier<Optional<UpdatablePageSource>> addSplit(Split split)
{
requireNonNull(split, "split is null");
checkArgument(split.getConnectorId().equals(REMOTE_CONNECTOR_ID), "split is not a remote split");
URI location = ((RemoteSplit) split.getConnectorSplit()).getLocation();
exchangeClient.addLocation(location);
return Optional::empty;
}
@Override
public void noMoreSplits()
{
exchangeClient.noMoreLocations();
}
@Override
public OperatorContext getOperatorContext()
{
return operatorContext;
}
@Override
public List<Type> getTypes()
{
return types;
}
@Override
public void finish()
{
close();
}
@Override
public boolean isFinished()
{
return exchangeClient.isFinished();
}
@Override
public ListenableFuture<?> isBlocked()
{
ListenableFuture<?> blocked = exchangeClient.isBlocked();
if (blocked.isDone()) {
return NOT_BLOCKED;
}
return blocked;
}
@Override
public boolean needsInput()
{
return false;
}
@Override
public void addInput(Page page)
{
throw new UnsupportedOperationException(getClass().getName() + " can not take input");
}
@Override
public Page getOutput()
{
SerializedPage page = exchangeClient.pollPage();
if (page == null) {
return null;
}
operatorContext.recordGeneratedInput(page.getSizeInBytes(), page.getPositionCount());
return serde.deserialize(page);
}
@Override
public void close()
{
exchangeClient.close();
}
}