/*
* Copyright 2016-present Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License. You may obtain
* a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package com.facebook.buck.parser;
import com.facebook.buck.event.BuckEventBus;
import com.facebook.buck.event.PerfEventId;
import com.facebook.buck.event.SimplePerfEvent;
import com.facebook.buck.log.Logger;
import com.facebook.buck.model.BuildTarget;
import com.facebook.buck.model.BuildTargetException;
import com.facebook.buck.parser.PipelineNodeCache.Cache;
import com.facebook.buck.rules.Cell;
import com.facebook.buck.rules.TargetNode;
import com.google.common.base.Function;
import com.google.common.collect.ImmutableSet;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
import java.nio.file.Path;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import javax.annotation.concurrent.ThreadSafe;
/**
* This implements a multithreaded pipeline for parsing BUCK files.
*
* <p>The high-level flow looks like this: (in) BuildTarget -> [getRawNodes] -*> [createTargetNode]
* -> (out) TargetNode (steps in [] have their output cached, -*> means that this step has parallel
* fanout).
*
* <p>The work is simply dumped onto the executor, the {@link ProjectBuildFileParserPool} is used to
* constrain the number of concurrent active parsers. Within a single pipeline instance work is not
* duplicated (the **JobsCache variables) are used to make sure we don't schedule the same work more
* than once), however it's possible for multiple read-only commands to duplicate work.
*/
@ThreadSafe
public class TargetNodeParsePipeline
extends ConvertingPipeline<Map<String, Object>, TargetNode<?, ?>> {
private static final Logger LOG = Logger.get(TargetNodeParsePipeline.class);
private final ParserTargetNodeFactory<TargetNode<?, ?>> delegate;
private final BuckEventBus eventBus;
private final boolean speculativeDepsTraversal;
private final RawNodeParsePipeline rawNodeParsePipeline;
private final SimplePerfEvent.Scope targetNodePipelineLifetimeEventScope;
/**
* Create new pipeline for parsing Buck files.
*
* @param cache where to persist results.
* @param targetNodeDelegate where to farm out the creation of TargetNodes to
* @param executorService executor
* @param eventBus bus to use for parse start/stop events
* @param speculativeDepsTraversal whether to automatically schedule parsing of nodes' deps in the
* @param rawNodeParsePipeline
*/
public TargetNodeParsePipeline(
Cache<BuildTarget, TargetNode<?, ?>> cache,
ParserTargetNodeFactory<TargetNode<?, ?>> targetNodeDelegate,
ListeningExecutorService executorService,
BuckEventBus eventBus,
boolean speculativeDepsTraversal,
RawNodeParsePipeline rawNodeParsePipeline) {
super(executorService, cache);
this.delegate = targetNodeDelegate;
this.eventBus = eventBus;
this.speculativeDepsTraversal = speculativeDepsTraversal;
this.rawNodeParsePipeline = rawNodeParsePipeline;
this.targetNodePipelineLifetimeEventScope =
SimplePerfEvent.scope(eventBus, PerfEventId.of("target_node_parse_pipeline"));
}
@Override
protected BuildTarget getBuildTarget(
Path root, Optional<String> cellName, Path buildFile, Map<String, Object> from) {
return BuildTarget.of(
RawNodeParsePipeline.parseBuildTargetFromRawRule(root, cellName, from, buildFile));
}
@Override
protected TargetNode<?, ?> computeNode(
final Cell cell,
final BuildTarget buildTarget,
final Map<String, Object> rawNode,
AtomicLong processedBytes)
throws BuildTargetException {
try (final SimplePerfEvent.Scope scope =
SimplePerfEvent.scopeIgnoringShortEvents(
eventBus,
PerfEventId.of("GetTargetNode"),
"target",
buildTarget,
targetNodePipelineLifetimeEventScope,
getMinimumPerfEventTimeMs(),
TimeUnit.MILLISECONDS)) {
Function<PerfEventId, SimplePerfEvent.Scope> perfEventScopeFunction =
perfEventId ->
SimplePerfEvent.scopeIgnoringShortEvents(
eventBus, perfEventId, scope, getMinimumPerfEventTimeMs(), TimeUnit.MILLISECONDS);
final TargetNode<?, ?> targetNode =
delegate.createTargetNode(
cell,
cell.getAbsolutePathToBuildFile(buildTarget),
buildTarget,
rawNode,
perfEventScopeFunction);
if (speculativeDepsTraversal) {
executorService.submit(
() -> {
for (BuildTarget depTarget : targetNode.getParseDeps()) {
Cell depCell = cell.getCellIgnoringVisibilityCheck(depTarget.getCellPath());
try {
if (depTarget.isFlavored()) {
getNodeJob(
depCell,
BuildTarget.of(depTarget.getUnflavoredBuildTarget()),
processedBytes);
}
getNodeJob(depCell, depTarget, processedBytes);
} catch (BuildTargetException e) {
// No biggie, we'll hit the error again in the non-speculative path.
LOG.info(e, "Could not schedule speculative parsing for %s", depTarget);
}
}
});
}
return targetNode;
}
}
@Override
protected ListenableFuture<ImmutableSet<Map<String, Object>>> getItemsToConvert(
Cell cell, Path buildFile, AtomicLong processedBytes) throws BuildTargetException {
return rawNodeParsePipeline.getAllNodesJob(cell, buildFile, processedBytes);
}
@Override
protected ListenableFuture<Map<String, Object>> getItemToConvert(
Cell cell, BuildTarget buildTarget, AtomicLong processedBytes) throws BuildTargetException {
return rawNodeParsePipeline.getNodeJob(cell, buildTarget, processedBytes);
}
@Override
public void close() {
targetNodePipelineLifetimeEventScope.close();
super.close();
}
}