// Copyright 2014 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.rules.repository;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.ImmutableMap;
import com.google.devtools.build.lib.analysis.BlazeDirectories;
import com.google.devtools.build.lib.cmdline.RepositoryName;
import com.google.devtools.build.lib.events.Event;
import com.google.devtools.build.lib.events.Location;
import com.google.devtools.build.lib.packages.Rule;
import com.google.devtools.build.lib.packages.RuleFormatter;
import com.google.devtools.build.lib.rules.repository.RepositoryFunction.RepositoryFunctionException;
import com.google.devtools.build.lib.skyframe.FileValue;
import com.google.devtools.build.lib.skyframe.PrecomputedValue;
import com.google.devtools.build.lib.skyframe.PrecomputedValue.Precomputed;
import com.google.devtools.build.lib.skyframe.SkyFunctions;
import com.google.devtools.build.lib.syntax.EvalException;
import com.google.devtools.build.lib.util.Fingerprint;
import com.google.devtools.build.lib.vfs.FileSystemUtils;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.build.skyframe.LegacySkyKey;
import com.google.devtools.build.skyframe.SkyFunction;
import com.google.devtools.build.skyframe.SkyFunctionException;
import com.google.devtools.build.skyframe.SkyFunctionException.Transience;
import com.google.devtools.build.skyframe.SkyKey;
import com.google.devtools.build.skyframe.SkyValue;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.Map;
import java.util.TreeMap;
import java.util.concurrent.atomic.AtomicBoolean;
import javax.annotation.Nullable;
/**
* A {@link SkyFunction} that implements delegation to the correct repository fetcher.
*
* <p>
* Each repository in the WORKSPACE file is represented by a {@link SkyValue} that is computed by
* this function.
*/
public final class RepositoryDelegatorFunction implements SkyFunction {
public static final Precomputed<Map<RepositoryName, PathFragment>> REPOSITORY_OVERRIDES =
new Precomputed<>(LegacySkyKey.create(SkyFunctions.PRECOMPUTED, "repository_overrides"));
// The marker file version is inject in the rule key digest so the rule key is always different
// when we decide to update the format.
private static final int MARKER_FILE_VERSION = 2;
// A special repository delegate used to handle Skylark remote repositories if present.
public static final String SKYLARK_DELEGATE_NAME = "$skylark";
// Mapping of rule class name to RepositoryFunction.
private final ImmutableMap<String, RepositoryFunction> handlers;
// Delegate function to handle skylark remote repositories
private final RepositoryFunction skylarkHandler;
// This is a reference to isFetch in BazelRepositoryModule, which tracks whether the current
// command is a fetch. Remote repository lookups are only allowed during fetches.
private final AtomicBoolean isFetch;
private Map<String, String> clientEnvironment;
public RepositoryDelegatorFunction(ImmutableMap<String, RepositoryFunction> handlers,
@Nullable RepositoryFunction skylarkHandler, AtomicBoolean isFetch) {
this.handlers = handlers;
this.skylarkHandler = skylarkHandler;
this.isFetch = isFetch;
}
public void setClientEnvironment(Map<String, String> clientEnvironment) {
this.clientEnvironment = clientEnvironment;
}
private void setupRepositoryRoot(Path repoRoot) throws RepositoryFunctionException {
try {
FileSystemUtils.deleteTree(repoRoot);
FileSystemUtils.createDirectoryAndParents(repoRoot.getParentDirectory());
} catch (IOException e) {
throw new RepositoryFunctionException(e, Transience.TRANSIENT);
}
}
@Override
public SkyValue compute(SkyKey skyKey, Environment env)
throws SkyFunctionException, InterruptedException {
RepositoryName repositoryName = (RepositoryName) skyKey.argument();
BlazeDirectories directories = PrecomputedValue.BLAZE_DIRECTORIES.get(env);
Map<RepositoryName, PathFragment> overrides = REPOSITORY_OVERRIDES.get(env);
if (env.valuesMissing()) {
return null;
}
Path repoRoot = RepositoryFunction.getExternalRepositoryDirectory(directories)
.getRelative(repositoryName.strippedName());
Path markerPath = getMarkerPath(directories, repositoryName.strippedName());
if (overrides.containsKey(repositoryName)) {
return setupOverride(
repositoryName, overrides.get(repositoryName), env, repoRoot, markerPath);
}
Rule rule;
try {
rule = RepositoryFunction.getRule(repositoryName, null, env);
} catch (RepositoryFunction.RepositoryNotFoundException e) {
return RepositoryDirectoryValue.NO_SUCH_REPOSITORY_VALUE;
}
if (rule == null) {
return null;
}
RepositoryFunction handler;
if (rule.getRuleClassObject().isSkylark()) {
handler = skylarkHandler;
} else {
handler = handlers.get(rule.getRuleClass());
}
if (handler == null) {
throw new RepositoryFunctionException(
new EvalException(
Location.fromFile(directories.getWorkspace().getRelative("WORKSPACE")),
"Could not find handler for " + rule),
Transience.PERSISTENT);
}
handler.setClientEnvironment(clientEnvironment);
byte[] ruleSpecificData = handler.getRuleSpecificMarkerData(rule, env);
if (ruleSpecificData == null) {
return null;
}
String ruleKey = computeRuleKey(rule, ruleSpecificData);
Map<String, String> markerData = new TreeMap<>();
if (handler.isLocal(rule)) {
// Local repositories are always fetched because the operation is generally fast and they do
// not depend on non-local data, so it does not make much sense to try to cache from across
// server instances.
setupRepositoryRoot(repoRoot);
RepositoryDirectoryValue.Builder localRepo =
handler.fetch(rule, repoRoot, directories, env, markerData);
if (localRepo == null) {
return null;
} else {
// We write the marker file for local repository essentially for getting the digest and
// injecting it in the RepositoryDirectoryValue.
byte[] digest = writeMarkerFile(markerPath, markerData, ruleKey);
return localRepo.setDigest(digest).build();
}
}
// We check the repository root for existence here, but we can't depend on the FileValue,
// because it's possible that we eventually create that directory in which case the FileValue
// and the state of the file system would be inconsistent.
byte[] markerHash = isFilesystemUpToDate(markerPath, rule, ruleKey, handler, env);
if (env.valuesMissing()) {
return null;
}
if (markerHash != null && repoRoot.exists()) {
// Now that we know that it exists, we can declare a Skyframe dependency on the repository
// root.
RepositoryFunction.getRepositoryDirectory(repoRoot, env);
if (env.valuesMissing()) {
return null;
}
return RepositoryDirectoryValue.builder().setPath(repoRoot).setDigest(markerHash).build();
}
if (isFetch.get()) {
// Fetching enabled, go ahead.
setupRepositoryRoot(repoRoot);
RepositoryDirectoryValue.Builder result =
handler.fetch(rule, repoRoot, directories, env, markerData);
if (env.valuesMissing()) {
return null;
}
// No new Skyframe dependencies must be added between calling the repository implementation
// and writing the marker file because if they aren't computed, it would cause a Skyframe
// restart thus calling the possibly very slow (networking, decompression...) fetch()
// operation again. So we write the marker file here immediately.
byte[] digest = writeMarkerFile(markerPath, markerData, ruleKey);
return result.setDigest(digest).build();
}
if (!repoRoot.exists()) {
// The repository isn't on the file system, there is nothing we can do.
throw new RepositoryFunctionException(
new IOException("to fix, run\n\tbazel fetch //...\nExternal repository " + repositoryName
+ " not found and fetching repositories is disabled."),
Transience.TRANSIENT);
}
// Declare a Skyframe dependency so that this is re-evaluated when something happens to the
// directory.
FileValue repoRootValue = RepositoryFunction.getRepositoryDirectory(repoRoot, env);
if (env.valuesMissing()) {
return null;
}
// Try to build with whatever is on the file system and emit a warning.
env.getListener()
.handle(Event.warn(rule.getLocation(),
String.format(
"External repository '%s' is not up-to-date and fetching is disabled. To update, "
+ "run the build without the '--nofetch' command line option.",
rule.getName())));
return RepositoryDirectoryValue.builder().setPath(repoRootValue.realRootedPath().asPath())
.setFetchingDelayed().build();
}
private String computeRuleKey(Rule rule, byte[] ruleSpecificData) {
return new Fingerprint().addBytes(RuleFormatter.serializeRule(rule).build().toByteArray())
.addBytes(ruleSpecificData)
.addInt(MARKER_FILE_VERSION).hexDigestAndReset();
}
/**
* Checks if the state of the repository in the file system is consistent with the rule in the
* WORKSPACE file.
*
* <p>
* Deletes the marker file if not so that no matter what happens after, the state of the file
* system stays consistent.
*
* <p>
* Returns null if the file system is not up to date and a hash of the marker file if the file
* system is up to date.
*/
@Nullable
private byte[] isFilesystemUpToDate(Path markerPath, Rule rule, String ruleKey,
RepositoryFunction handler, Environment env)
throws RepositoryFunctionException, InterruptedException {
try {
if (!markerPath.exists()) {
return null;
}
String content = FileSystemUtils.readContent(markerPath, StandardCharsets.UTF_8);
String[] lines = content.split("\n");
Map<String, String> markerData = new TreeMap<>();
String markerRuleKey = "";
boolean firstLine = true;
for (String line : lines) {
if (firstLine) {
markerRuleKey = line;
firstLine = false;
} else {
int sChar = line.indexOf(' ');
String key = line;
String value = "";
if (sChar > 0) {
key = unescape(line.substring(0, sChar));
value = unescape(line.substring(sChar + 1));
}
markerData.put(key, value);
}
}
boolean result = false;
if (markerRuleKey.equals(ruleKey)) {
result = handler.verifyMarkerData(rule, markerData, env);
if (env.valuesMissing()) {
return null;
}
}
if (result) {
return new Fingerprint().addString(content).digestAndReset();
} else {
// So that we are in a consistent state if something happens while fetching the repository
markerPath.delete();
return null;
}
} catch (IOException e) {
throw new RepositoryFunctionException(e, Transience.TRANSIENT);
}
}
// Escape a value for the marker file
@VisibleForTesting
static String escape(String str) {
return str == null ? "\\0" : str.replace("\\", "\\\\").replace("\n", "\\n").replace(" ", "\\s");
}
// Unescape a value from the marker file
@VisibleForTesting
static String unescape(String str) {
if (str.equals("\\0")) {
return null; // \0 == null string
}
StringBuffer result = new StringBuffer();
boolean escaped = false;
for (int i = 0; i < str.length(); i++) {
char c = str.charAt(i);
if (escaped) {
if (c == 'n') { // n means new line
result.append("\n");
} else if (c == 's') { // s means space
result.append(" ");
} else { // Any other escaped characters are just un-escaped
result.append(c);
}
escaped = false;
} else if (c == '\\') {
escaped = true;
} else {
result.append(c);
}
}
return result.toString();
}
private byte[] writeMarkerFile(
Path markerPath, Map<String, String> markerData, String ruleKey)
throws RepositoryFunctionException {
try {
StringBuilder builder = new StringBuilder();
builder.append(ruleKey).append("\n");
for (Map.Entry<String, String> data : markerData.entrySet()) {
String key = data.getKey();
String value = data.getValue();
builder.append(escape(key)).append(" ").append(escape(value)).append("\n");
}
String content = builder.toString();
FileSystemUtils.writeContent(markerPath, StandardCharsets.UTF_8, content);
return new Fingerprint().addString(content).digestAndReset();
} catch (IOException e) {
throw new RepositoryFunctionException(e, Transience.TRANSIENT);
}
}
private static Path getMarkerPath(BlazeDirectories directories, String ruleName) {
return RepositoryFunction.getExternalRepositoryDirectory(directories)
.getChild("@" + ruleName + ".marker");
}
@Override
public String extractTag(SkyKey skyKey) {
return null;
}
private RepositoryDirectoryValue setupOverride(
RepositoryName repositoryName, PathFragment sourcePath, Environment env, Path repoRoot,
Path markerPath)
throws RepositoryFunctionException, InterruptedException {
setupRepositoryRoot(repoRoot);
RepositoryDirectoryValue.Builder directoryValue = LocalRepositoryFunction.symlink(
repoRoot, sourcePath, env);
if (directoryValue == null) {
return null;
}
String ruleKey = new Fingerprint().addBytes(repositoryName.strippedName().getBytes())
.addBytes(repoRoot.getFileSystem().getPath(sourcePath).getPathString().getBytes())
.addInt(MARKER_FILE_VERSION).hexDigestAndReset();
byte[] digest = writeMarkerFile(markerPath, new TreeMap<String, String>(), ruleKey);
return directoryValue.setDigest(digest).build();
}
}