/*
* Copyright 2017-present Facebook, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License"); you may
* not use this file except in compliance with the License. You may obtain
* a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
* License for the specific language governing permissions and limitations
* under the License.
*/
package com.facebook.buck.cli;
import com.facebook.buck.distributed.DistBuildService;
import com.facebook.buck.distributed.thrift.BuildJobState;
import com.facebook.buck.distributed.thrift.BuildJobStateFileHashEntry;
import com.facebook.buck.distributed.thrift.BuildJobStateFileHashes;
import com.facebook.buck.distributed.thrift.StampedeId;
import com.facebook.buck.io.ProjectFilesystem;
import com.facebook.buck.util.ThrowingPrintWriter;
import com.google.common.collect.Lists;
import java.io.BufferedOutputStream;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
import java.util.Optional;
import org.kohsuke.args4j.Argument;
import org.kohsuke.args4j.Option;
/**
* Distributed build debug command that prints out all files in the local hard-drive whose contents
* were hashed in order to take part in any rule key computation. This capturing is done at the
* level of the ActionGraph after all transformation and enhancement steps have taken place. Only
* files in this list will take part of the distributed build and will be lazily materialised on the
* remote BuildSlave servers.
*
* <p>This command can be run in two modes: 1. Produce the list of used files for a repository in
* the local machine passing any number of build targets. 2. Fetch and output the list of targets
* used for a distributed build that has already occurred. This is done by passing in the
* [stampede-id] of the build.
*/
public class DistBuildSourceFilesCommand extends AbstractDistBuildCommand {
private static final String OUTPUT_FILENAME = "stampede_build_source_files.txt";
@Option(
name = "--output-file",
usage = "File where stampede source file dependencies will be saved to."
)
private String outputFilename = OUTPUT_FILENAME;
/**
* The distributed build state does not contain any cell absolute IDs or paths. It just contains a
* name hint/alias for the cell. All cells will be rooted off whatever value is provided to this
* option.
*/
@Option(name = "--cells-root-path", usage = "Path where all cells will be rooted from.")
private String cellsRootPath = "/CELLS_ROOT_PATH";
/** List of build targets that will be used if this command is ran locally. */
@Argument private List<String> arguments;
public DistBuildSourceFilesCommand() {
this(Lists.newArrayList());
}
public DistBuildSourceFilesCommand(List<String> arguments) {
this.arguments = arguments;
}
@Override
public boolean isReadOnly() {
return true;
}
@Override
public String getShortDescription() {
return "gets the list of all source files required for a distributed build.";
}
@Override
public int runWithoutHelp(CommandRunnerParams params) throws IOException, InterruptedException {
Optional<StampedeId> stampedeId = getStampedeIdOptional();
if (stampedeId.isPresent()) {
runUsingStampedeId(params, stampedeId.get());
} else {
runLocally(params);
}
return 0;
}
/**
* Runs the first stage of a distributed build locally to compute all required source files for
* remote materialisation.
*/
private void runLocally(CommandRunnerParams params) throws IOException, InterruptedException {
try (CommandThreadManager pool =
new CommandThreadManager(
"DistBuildSourceFiles", getConcurrencyLimit(params.getBuckConfig()))) {
BuildJobState jobState =
BuildCommand.getDistBuildState(arguments, params, pool.getExecutor());
outputResultToTempFile(params, jobState);
}
}
/**
* Fetches the state from a previous distributed build and outputs all source files that were
* deemed required for that build to take place.
*/
private void runUsingStampedeId(CommandRunnerParams params, StampedeId stampedeId)
throws IOException {
try (DistBuildService service = DistBuildFactory.newDistBuildService(params)) {
BuildJobState jobState = service.fetchBuildJobState(stampedeId);
outputResultToTempFile(params, jobState);
}
}
/**
* Print one required source file per line expanding the root of all cells using the command line
* argument --cells-root-path.
*/
private void outputResultToTempFile(CommandRunnerParams params, BuildJobState jobState)
throws IOException {
Path logDir = params.getInvocationInfo().get().getLogDirectoryPath();
Path outputFileAbs = logDir.resolve(outputFilename).normalize();
int writtenLineCount = 0;
try (ThrowingPrintWriter writer =
new ThrowingPrintWriter(new BufferedOutputStream(Files.newOutputStream(outputFileAbs)))) {
ProjectFilesystem fs = params.getCell().getFilesystem();
Path cellsCommonRootPath = fs.resolve(Paths.get(cellsRootPath).normalize());
for (BuildJobStateFileHashes cellHashes : jobState.getFileHashes()) {
String cellName = jobState.getCells().get(cellHashes.cellIndex).getNameHint();
Path cellRoot = cellsCommonRootPath.resolve(cellName);
if (!cellHashes.isSetEntries()) {
continue;
}
for (BuildJobStateFileHashEntry entry : cellHashes.getEntries()) {
Path absPath = cellRoot.resolve(entry.getPath().getPath()).normalize();
writer.println(absPath);
++writtenLineCount;
}
}
}
params
.getConsole()
.printSuccess(
"A total of [%d] source file paths were saved to [%s].",
writtenLineCount, outputFileAbs.toAbsolutePath().toString());
}
}