/* * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ package com.facebook.presto.operator; import com.facebook.presto.connector.ConnectorId; import com.facebook.presto.execution.SystemMemoryUsageListener; import com.facebook.presto.execution.buffer.PagesSerde; import com.facebook.presto.execution.buffer.PagesSerdeFactory; import com.facebook.presto.execution.buffer.SerializedPage; import com.facebook.presto.metadata.Split; import com.facebook.presto.spi.Page; import com.facebook.presto.spi.UpdatablePageSource; import com.facebook.presto.spi.type.Type; import com.facebook.presto.split.RemoteSplit; import com.facebook.presto.sql.planner.plan.PlanNodeId; import com.google.common.util.concurrent.ListenableFuture; import javax.annotation.concurrent.NotThreadSafe; import java.io.Closeable; import java.net.URI; import java.util.List; import java.util.Optional; import java.util.function.Supplier; import static com.google.common.base.Preconditions.checkArgument; import static com.google.common.base.Preconditions.checkState; import static java.util.Objects.requireNonNull; public class ExchangeOperator implements SourceOperator, Closeable { public static final ConnectorId REMOTE_CONNECTOR_ID = new ConnectorId("$remote"); public static class ExchangeOperatorFactory implements SourceOperatorFactory { private final int operatorId; private final PlanNodeId sourceId; private final ExchangeClientSupplier exchangeClientSupplier; private final PagesSerdeFactory serdeFactory; private final List<Type> types; private ExchangeClient exchangeClient = null; private boolean closed; public ExchangeOperatorFactory( int operatorId, PlanNodeId sourceId, ExchangeClientSupplier exchangeClientSupplier, PagesSerdeFactory serdeFactory, List<Type> types) { this.operatorId = operatorId; this.sourceId = sourceId; this.exchangeClientSupplier = exchangeClientSupplier; this.serdeFactory = serdeFactory; this.types = types; } @Override public PlanNodeId getSourceId() { return sourceId; } @Override public List<Type> getTypes() { return types; } @Override public SourceOperator createOperator(DriverContext driverContext) { checkState(!closed, "Factory is already closed"); OperatorContext operatorContext = driverContext.addOperatorContext(operatorId, sourceId, ExchangeOperator.class.getSimpleName()); if (exchangeClient == null) { exchangeClient = exchangeClientSupplier.get(new UpdateSystemMemory(driverContext.getPipelineContext())); } return new ExchangeOperator( operatorContext, types, sourceId, serdeFactory.createPagesSerde(), exchangeClient); } @Override public void close() { closed = true; } } @NotThreadSafe private static final class UpdateSystemMemory implements SystemMemoryUsageListener { private final PipelineContext pipelineContext; public UpdateSystemMemory(PipelineContext pipelineContext) { this.pipelineContext = requireNonNull(pipelineContext, "pipelineContext is null"); } @Override public void updateSystemMemoryUsage(long deltaMemoryInBytes) { if (deltaMemoryInBytes > 0) { pipelineContext.reserveSystemMemory(deltaMemoryInBytes); } else { pipelineContext.freeSystemMemory(-deltaMemoryInBytes); } } } private final OperatorContext operatorContext; private final PlanNodeId sourceId; private final ExchangeClient exchangeClient; private final List<Type> types; private final PagesSerde serde; public ExchangeOperator( OperatorContext operatorContext, List<Type> types, PlanNodeId sourceId, PagesSerde serde, ExchangeClient exchangeClient) { this.operatorContext = requireNonNull(operatorContext, "operatorContext is null"); this.sourceId = requireNonNull(sourceId, "sourceId is null"); this.exchangeClient = requireNonNull(exchangeClient, "exchangeClient is null"); this.serde = requireNonNull(serde, "serde is null"); this.types = requireNonNull(types, "types is null"); operatorContext.setInfoSupplier(exchangeClient::getStatus); } @Override public PlanNodeId getSourceId() { return sourceId; } @Override public Supplier<Optional<UpdatablePageSource>> addSplit(Split split) { requireNonNull(split, "split is null"); checkArgument(split.getConnectorId().equals(REMOTE_CONNECTOR_ID), "split is not a remote split"); URI location = ((RemoteSplit) split.getConnectorSplit()).getLocation(); exchangeClient.addLocation(location); return Optional::empty; } @Override public void noMoreSplits() { exchangeClient.noMoreLocations(); } @Override public OperatorContext getOperatorContext() { return operatorContext; } @Override public List<Type> getTypes() { return types; } @Override public void finish() { close(); } @Override public boolean isFinished() { return exchangeClient.isFinished(); } @Override public ListenableFuture<?> isBlocked() { ListenableFuture<?> blocked = exchangeClient.isBlocked(); if (blocked.isDone()) { return NOT_BLOCKED; } return blocked; } @Override public boolean needsInput() { return false; } @Override public void addInput(Page page) { throw new UnsupportedOperationException(getClass().getName() + " can not take input"); } @Override public Page getOutput() { SerializedPage page = exchangeClient.pollPage(); if (page == null) { return null; } operatorContext.recordGeneratedInput(page.getSizeInBytes(), page.getPositionCount()); return serde.deserialize(page); } @Override public void close() { exchangeClient.close(); } }