// Copyright 2016 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.bazel.repository.skylark;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Optional;
import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.devtools.build.lib.bazel.repository.DecompressorDescriptor;
import com.google.devtools.build.lib.bazel.repository.DecompressorValue;
import com.google.devtools.build.lib.bazel.repository.cache.RepositoryCache.KeyType;
import com.google.devtools.build.lib.bazel.repository.downloader.HttpDownloader;
import com.google.devtools.build.lib.bazel.repository.downloader.HttpUtils;
import com.google.devtools.build.lib.cmdline.Label;
import com.google.devtools.build.lib.cmdline.LabelSyntaxException;
import com.google.devtools.build.lib.events.Location;
import com.google.devtools.build.lib.packages.Attribute;
import com.google.devtools.build.lib.packages.NativeClassObjectConstructor;
import com.google.devtools.build.lib.packages.Rule;
import com.google.devtools.build.lib.packages.SkylarkClassObject;
import com.google.devtools.build.lib.rules.repository.RepositoryFunction.RepositoryFunctionException;
import com.google.devtools.build.lib.rules.repository.WorkspaceAttributeMapper;
import com.google.devtools.build.lib.skyframe.FileSymlinkException;
import com.google.devtools.build.lib.skyframe.FileValue;
import com.google.devtools.build.lib.skyframe.InconsistentFilesystemException;
import com.google.devtools.build.lib.skyframe.PackageLookupValue;
import com.google.devtools.build.lib.skylarkinterface.Param;
import com.google.devtools.build.lib.skylarkinterface.ParamType;
import com.google.devtools.build.lib.skylarkinterface.SkylarkCallable;
import com.google.devtools.build.lib.skylarkinterface.SkylarkModule;
import com.google.devtools.build.lib.skylarkinterface.SkylarkModuleCategory;
import com.google.devtools.build.lib.syntax.EvalException;
import com.google.devtools.build.lib.syntax.Runtime;
import com.google.devtools.build.lib.syntax.SkylarkDict;
import com.google.devtools.build.lib.syntax.SkylarkList;
import com.google.devtools.build.lib.syntax.SkylarkType;
import com.google.devtools.build.lib.util.StringUtilities;
import com.google.devtools.build.lib.vfs.FileSystemUtils;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.build.lib.vfs.RootedPath;
import com.google.devtools.build.skyframe.SkyFunction.Environment;
import com.google.devtools.build.skyframe.SkyFunctionException.Transience;
import com.google.devtools.build.skyframe.SkyKey;
import java.io.File;
import java.io.IOException;
import java.io.OutputStream;
import java.net.MalformedURLException;
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
/** Skylark API for the repository_rule's context. */
@SkylarkModule(
name = "repository_ctx",
category = SkylarkModuleCategory.BUILTIN,
doc =
"The context of the repository rule containing"
+ " helper functions and information about attributes. You get a repository_ctx object"
+ " as an argument to the <code>implementation</code> function when you create a"
+ " repository rule."
)
public class SkylarkRepositoryContext {
private final Rule rule;
private final Path outputDirectory;
private final SkylarkClassObject attrObject;
private final SkylarkOS osObject;
private final Environment env;
private final HttpDownloader httpDownloader;
private final Map<String, String> markerData;
/**
* Create a new context (repository_ctx) object for a skylark repository rule ({@code rule}
* argument).
*/
SkylarkRepositoryContext(Rule rule, Path outputDirectory, Environment environment,
Map<String, String> env, HttpDownloader httpDownloader, Map<String, String> markerData)
throws EvalException {
this.rule = rule;
this.outputDirectory = outputDirectory;
this.env = environment;
this.osObject = new SkylarkOS(env);
this.httpDownloader = httpDownloader;
this.markerData = markerData;
WorkspaceAttributeMapper attrs = WorkspaceAttributeMapper.of(rule);
ImmutableMap.Builder<String, Object> attrBuilder = new ImmutableMap.Builder<>();
for (String name : attrs.getAttributeNames()) {
if (!name.equals("$local")) {
Object val = attrs.getObject(name);
attrBuilder.put(
Attribute.getSkylarkName(name),
val == null
? Runtime.NONE
// Attribute values should be type safe
: SkylarkType.convertToSkylark(val, null));
}
}
attrObject = NativeClassObjectConstructor.STRUCT.create(
attrBuilder.build(), "No such attribute '%s'");
}
@SkylarkCallable(
name = "name",
structField = true,
doc = "The name of the external repository created by this rule."
)
public String getName() {
return rule.getName();
}
@SkylarkCallable(
name = "attr",
structField = true,
doc =
"A struct to access the values of the attributes. The values are provided by "
+ "the user (if not, a default value is used)."
)
public SkylarkClassObject getAttr() {
return attrObject;
}
@SkylarkCallable(
name = "path",
doc =
"Returns a path from a string or a label. If the path is relative, it will resolve "
+ "relative to the repository directory. If the path is a label, it will resolve to "
+ "the path of the corresponding file. Note that remote repositories are executed "
+ "during the analysis phase and thus cannot depends on a target result (the "
+ "label should point to a non-generated file)."
)
public SkylarkPath path(Object path) throws EvalException, InterruptedException {
return getPath("path()", path);
}
private SkylarkPath getPath(String method, Object path)
throws EvalException, InterruptedException {
if (path instanceof String) {
PathFragment pathFragment = PathFragment.create(path.toString());
return new SkylarkPath(pathFragment.isAbsolute()
? outputDirectory.getFileSystem().getPath(path.toString())
: outputDirectory.getRelative(pathFragment));
} else if (path instanceof Label) {
return getPathFromLabel((Label) path);
} else if (path instanceof SkylarkPath) {
return (SkylarkPath) path;
} else {
throw new EvalException(Location.BUILTIN, method + " can only take a string or a label.");
}
}
@SkylarkCallable(
name = "symlink",
doc = "Create a symlink on the filesystem.",
parameters = {
@Param(
name = "from",
allowedTypes = {
@ParamType(type = String.class),
@ParamType(type = Label.class),
@ParamType(type = SkylarkPath.class)
},
doc = "path to which the created symlink should point to."
),
@Param(
name = "to",
allowedTypes = {
@ParamType(type = String.class),
@ParamType(type = Label.class),
@ParamType(type = SkylarkPath.class)
},
doc = "path of the symlink to create, relative to the repository directory."
),
}
)
public void symlink(Object from, Object to)
throws RepositoryFunctionException, EvalException, InterruptedException {
SkylarkPath fromPath = getPath("symlink()", from);
SkylarkPath toPath = getPath("symlink()", to);
try {
checkInOutputDirectory(toPath);
makeDirectories(toPath.getPath());
toPath.getPath().createSymbolicLink(fromPath.getPath());
} catch (IOException e) {
throw new RepositoryFunctionException(
new IOException(
"Could not create symlink from " + fromPath + " to " + toPath + ": " + e.getMessage(),
e),
Transience.TRANSIENT);
}
}
private void checkInOutputDirectory(SkylarkPath path) throws RepositoryFunctionException {
if (!path.getPath().getPathString().startsWith(outputDirectory.getPathString())) {
throw new RepositoryFunctionException(
new EvalException(
Location.fromFile(path.getPath()),
"Cannot write outside of the repository directory for path " + path),
Transience.PERSISTENT);
}
}
@SkylarkCallable(
name = "file",
doc = "Generate a file in the repository directory with the provided content.",
parameters = {
@Param(
name = "path",
allowedTypes = {
@ParamType(type = String.class),
@ParamType(type = Label.class),
@ParamType(type = SkylarkPath.class)
},
doc = "path of the file to create, relative to the repository directory."
),
@Param(
name = "content",
type = String.class,
named = true,
defaultValue = "''",
doc = "the content of the file to create, empty by default."
),
@Param(
name = "executable",
named = true,
type = Boolean.class,
defaultValue = "True",
doc = "set the executable flag on the created file, true by default."
),
}
)
public void createFile(Object path, String content, Boolean executable)
throws RepositoryFunctionException, EvalException, InterruptedException {
SkylarkPath p = getPath("file()", path);
try {
checkInOutputDirectory(p);
makeDirectories(p.getPath());
try (OutputStream stream = p.getPath().getOutputStream()) {
stream.write(content.getBytes(StandardCharsets.UTF_8));
}
if (executable) {
p.getPath().setExecutable(true);
}
} catch (IOException e) {
throw new RepositoryFunctionException(e, Transience.TRANSIENT);
}
}
@SkylarkCallable(
name = "template",
doc =
"Generate a new file using a <code>template</code>. Every occurrence in "
+ "<code>template</code> of a key of <code>substitutions</code> will be replaced by "
+ "the corresponding value. The result is written in <code>path</code>. An optional"
+ "<code>executable</code> argument (default to true) can be set to turn on or off"
+ "the executable bit.",
parameters = {
@Param(
name = "path",
allowedTypes = {
@ParamType(type = String.class),
@ParamType(type = Label.class),
@ParamType(type = SkylarkPath.class)
},
doc = "path of the file to create, relative to the repository directory."
),
@Param(
name = "template",
allowedTypes = {
@ParamType(type = String.class),
@ParamType(type = Label.class),
@ParamType(type = SkylarkPath.class)
},
doc = "path to the template file."
),
@Param(
name = "substitutions",
type = SkylarkDict.class,
defaultValue = "{}",
named = true,
doc = "substitutions to make when expanding the template."
),
@Param(
name = "executable",
type = Boolean.class,
defaultValue = "True",
named = true,
doc = "set the executable flag on the created file, true by default."
),
}
)
public void createFileFromTemplate(
Object path, Object template, SkylarkDict<String, String> substitutions, Boolean executable)
throws RepositoryFunctionException, EvalException, InterruptedException {
SkylarkPath p = getPath("template()", path);
SkylarkPath t = getPath("template()", template);
try {
checkInOutputDirectory(p);
makeDirectories(p.getPath());
String tpl = FileSystemUtils.readContent(t.getPath(), StandardCharsets.UTF_8);
for (Map.Entry<String, String> substitution : substitutions.entrySet()) {
tpl =
StringUtilities.replaceAllLiteral(tpl, substitution.getKey(), substitution.getValue());
}
try (OutputStream stream = p.getPath().getOutputStream()) {
stream.write(tpl.getBytes(StandardCharsets.UTF_8));
}
if (executable) {
p.getPath().setExecutable(true);
}
} catch (IOException e) {
throw new RepositoryFunctionException(e, Transience.TRANSIENT);
}
}
// Create parent directories for the given path
private void makeDirectories(Path path) throws IOException {
if (!path.isRootDirectory()) {
Path parent = path.getParentDirectory();
if (!parent.exists()) {
makeDirectories(path.getParentDirectory());
parent.createDirectory();
}
}
}
@SkylarkCallable(
name = "os",
structField = true,
doc = "A struct to access information from the system."
)
public SkylarkOS getOS() {
return osObject;
}
private void createDirectory(Path directory) throws RepositoryFunctionException {
try {
if (!directory.exists()) {
makeDirectories(directory);
directory.createDirectory();
}
} catch (IOException e) {
throw new RepositoryFunctionException(e, Transience.TRANSIENT);
}
}
@SkylarkCallable(
name = "execute",
doc =
"Executes the command given by the list of arguments. The execution time of the command"
+ " is limited by <code>timeout</code> (in seconds, default 600 seconds). This method"
+ " returns an <code>exec_result</code> structure containing the output of the"
+ " command. The <code>environment</code> map can be used to override some"
+ " environment variables to be passed to the process.",
parameters = {
@Param(
name = "arguments",
type = SkylarkList.class,
doc =
"List of arguments, the first element should be the path to the program to "
+ "execute."
),
@Param(
name = "timeout",
type = Integer.class,
named = true,
defaultValue = "600",
doc = "maximum duration of the command in seconds (default is 600 seconds)."
),
@Param(
name = "environment",
type = SkylarkDict.class,
defaultValue = "{}",
named = true,
doc = "force some environment variables to be set to be passed to the process."
),
@Param(
name = "quiet",
type = Boolean.class,
defaultValue = "True",
named = true,
doc = "If stdout and stderr should be printed to the terminal."
),
}
)
public SkylarkExecutionResult execute(
SkylarkList<Object> arguments, Integer timeout, SkylarkDict<String, String> environment,
boolean quiet)
throws EvalException, RepositoryFunctionException {
createDirectory(outputDirectory);
return SkylarkExecutionResult.builder(osObject.getEnvironmentVariables())
.addArguments(arguments)
.setDirectory(outputDirectory.getPathFile())
.addEnvironmentVariables(environment)
.setTimeout(timeout.longValue() * 1000)
.setQuiet(quiet)
.execute();
}
@SkylarkCallable(
name = "which",
doc =
"Returns the path of the corresponding program or None "
+ "if there is no such program in the path",
allowReturnNones = true
)
public SkylarkPath which(String program) throws EvalException {
if (program.contains("/") || program.contains("\\")) {
throw new EvalException(
Location.BUILTIN,
"Program argument of which() may not contains a / or a \\ ('" + program + "' given)");
}
for (String p : getPathEnvironment()) {
PathFragment fragment = PathFragment.create(p);
if (fragment.isAbsolute()) {
// We ignore relative path as they don't mean much here (relative to where? the workspace
// root?).
Path path = outputDirectory.getFileSystem().getPath(fragment).getChild(program);
try {
if (path.exists() && path.isExecutable()) {
return new SkylarkPath(path);
}
} catch (IOException e) {
// IOException when checking executable file means we cannot read the file data so
// we cannot executes it, swallow the exception.
}
}
}
return null;
}
@SkylarkCallable(
name = "download",
doc = "Download a file to the output path for the provided url.",
parameters = {
@Param(
name = "url",
allowedTypes = {
@ParamType(type = String.class),
@ParamType(type = SkylarkList.class, generic1 = String.class),
},
named = true,
doc = "List of mirror URLs referencing the same file."
),
@Param(
name = "output",
allowedTypes = {
@ParamType(type = String.class),
@ParamType(type = Label.class),
@ParamType(type = SkylarkPath.class)
},
defaultValue = "''",
named = true,
doc = "path to the output file, relative to the repository directory."
),
@Param(
name = "sha256",
type = String.class,
defaultValue = "''",
named = true,
doc =
"the expected SHA-256 hash of the file downloaded."
+ " This must match the SHA-256 hash of the file downloaded. It is a security risk"
+ " to omit the SHA-256 as remote files can change. At best omitting this field"
+ " will make your build non-hermetic. It is optional to make development easier"
+ " but should be set before shipping."
),
@Param(
name = "executable",
type = Boolean.class,
defaultValue = "False",
named = true,
doc = "set the executable flag on the created file, false by default."
),
}
)
public void download(
Object url, Object output, String sha256, Boolean executable)
throws RepositoryFunctionException, EvalException, InterruptedException {
validateSha256(sha256);
List<URL> urls = getUrls(url);
SkylarkPath outputPath = getPath("download()", output);
try {
checkInOutputDirectory(outputPath);
makeDirectories(outputPath.getPath());
httpDownloader.download(
urls,
sha256,
Optional.<String>absent(),
outputPath.getPath(),
env.getListener(),
osObject.getEnvironmentVariables());
if (executable) {
outputPath.getPath().setExecutable(true);
}
} catch (InterruptedException e) {
throw new RepositoryFunctionException(
new IOException("thread interrupted"), Transience.TRANSIENT);
} catch (IOException e) {
throw new RepositoryFunctionException(e, Transience.TRANSIENT);
}
}
@SkylarkCallable(
name = "download_and_extract",
doc = "Download a file to the output path for the provided url, and extract it.",
parameters = {
@Param(
name = "url",
allowedTypes = {
@ParamType(type = String.class),
@ParamType(type = SkylarkList.class, generic1 = String.class),
},
named = true,
doc = "List of mirror URLs referencing the same file."
),
@Param(
name = "output",
allowedTypes = {
@ParamType(type = String.class),
@ParamType(type = Label.class),
@ParamType(type = SkylarkPath.class)
},
defaultValue = "''",
named = true,
doc =
"path to the directory where the archive will be unpacked,"
+ " relative to the repository directory."
),
@Param(
name = "sha256",
type = String.class,
defaultValue = "''",
named = true,
doc =
"the expected SHA-256 hash of the file downloaded."
+ " This must match the SHA-256 hash of the file downloaded. It is a security risk"
+ " to omit the SHA-256 as remote files can change. At best omitting this field"
+ " will make your build non-hermetic. It is optional to make development easier"
+ " but should be set before shipping."
),
@Param(
name = "type",
type = String.class,
defaultValue = "''",
named = true,
doc =
"the archive type of the downloaded file."
+ " By default, the archive type is determined from the file extension of the URL."
+ " If the file has no extension, you can explicitly specify either \"zip\","
+ " \"jar\", \"war\", \"tar.gz\", \"tgz\", \"tar.bz2\", or \"tar.xz\" here."
),
@Param(
name = "stripPrefix",
type = String.class,
defaultValue = "''",
named = true,
doc =
"a directory prefix to strip from the extracted files."
+ "\nMany archives contain a top-level directory that contains all files in the"
+ " archive. Instead of needing to specify this prefix over and over in the"
+ " <code>build_file</code>, this field can be used to strip it from extracted"
+ " files."
),
}
)
public void downloadAndExtract(
Object url, Object output, String sha256, String type, String stripPrefix)
throws RepositoryFunctionException, InterruptedException, EvalException {
validateSha256(sha256);
List<URL> urls = getUrls(url);
// Download to outputDirectory and delete it after extraction
SkylarkPath outputPath = getPath("download_and_extract()", output);
checkInOutputDirectory(outputPath);
createDirectory(outputPath.getPath());
Path downloadedPath;
try {
downloadedPath =
httpDownloader.download(
urls,
sha256,
Optional.of(type),
outputPath.getPath(),
env.getListener(),
osObject.getEnvironmentVariables());
} catch (InterruptedException e) {
throw new RepositoryFunctionException(
new IOException("thread interrupted"), Transience.TRANSIENT);
} catch (IOException e) {
throw new RepositoryFunctionException(e, Transience.TRANSIENT);
}
DecompressorValue.decompress(
DecompressorDescriptor.builder()
.setTargetKind(rule.getTargetKind())
.setTargetName(rule.getName())
.setArchivePath(downloadedPath)
.setRepositoryPath(outputPath.getPath())
.setPrefix(stripPrefix)
.build());
try {
if (downloadedPath.exists()) {
downloadedPath.delete();
}
} catch (IOException e) {
throw new RepositoryFunctionException(
new IOException(
"Couldn't delete temporary file (" + downloadedPath.getPathString() + ")", e),
Transience.TRANSIENT);
}
}
private static void validateSha256(String sha256) throws RepositoryFunctionException {
if (!sha256.isEmpty() && !KeyType.SHA256.isValid(sha256)) {
throw new RepositoryFunctionException(
new IOException("Invalid SHA256 checksum"), Transience.TRANSIENT);
}
}
private static List<URL> getUrls(Object urlOrList) throws RepositoryFunctionException {
List<String> urlStrings;
if (urlOrList instanceof String) {
urlStrings = ImmutableList.of((String) urlOrList);
} else {
@SuppressWarnings("unchecked")
List<String> list = (List<String>) urlOrList;
urlStrings = list;
}
if (urlStrings.isEmpty()) {
throw new RepositoryFunctionException(new IOException("urls not set"), Transience.PERSISTENT);
}
List<URL> urls = new ArrayList<>();
for (String urlString : urlStrings) {
URL url;
try {
url = new URL(urlString);
} catch (MalformedURLException e) {
throw new RepositoryFunctionException(
new IOException("Bad URL: " + urlString), Transience.PERSISTENT);
}
if (!HttpUtils.isUrlSupportedByDownloader(url)) {
throw new RepositoryFunctionException(
new IOException("Unsupported protocol: " + url.getProtocol()), Transience.PERSISTENT);
}
urls.add(url);
}
return urls;
}
// This is just for test to overwrite the path environment
private static ImmutableList<String> pathEnv = null;
@VisibleForTesting
static void setPathEnvironment(String... pathEnv) {
SkylarkRepositoryContext.pathEnv = ImmutableList.<String>copyOf(pathEnv);
}
private ImmutableList<String> getPathEnvironment() {
if (pathEnv != null) {
return pathEnv;
}
String pathEnviron = osObject.getEnvironmentVariables().get("PATH");
if (pathEnviron == null) {
return ImmutableList.of();
}
return ImmutableList.copyOf(pathEnviron.split(File.pathSeparator));
}
@Override
public String toString() {
return "repository_ctx[" + rule.getLabel() + "]";
}
private static RootedPath getRootedPathFromLabel(Label label, Environment env)
throws InterruptedException, EvalException {
// Look for package.
if (label.getPackageIdentifier().getRepository().isDefault()) {
try {
label = Label.create(label.getPackageIdentifier().makeAbsolute(), label.getName());
} catch (LabelSyntaxException e) {
throw new AssertionError(e); // Can't happen because the input label is valid
}
}
SkyKey pkgSkyKey = PackageLookupValue.key(label.getPackageIdentifier());
PackageLookupValue pkgLookupValue = (PackageLookupValue) env.getValue(pkgSkyKey);
if (pkgLookupValue == null) {
throw SkylarkRepositoryFunction.restart();
}
if (!pkgLookupValue.packageExists()) {
throw new EvalException(Location.BUILTIN,
"Unable to load package for " + label + ": not found.");
}
// And now for the file
Path packageRoot = pkgLookupValue.getRoot();
return RootedPath.toRootedPath(packageRoot, label.toPathFragment());
}
// Resolve the label given by value into a file path.
private SkylarkPath getPathFromLabel(Label label) throws EvalException, InterruptedException {
RootedPath rootedPath = getRootedPathFromLabel(label, env);
SkyKey fileSkyKey = FileValue.key(rootedPath);
FileValue fileValue = null;
try {
fileValue = (FileValue) env.getValueOrThrow(fileSkyKey, IOException.class,
FileSymlinkException.class, InconsistentFilesystemException.class);
} catch (IOException | FileSymlinkException | InconsistentFilesystemException e) {
throw new EvalException(Location.BUILTIN, e);
}
if (fileValue == null) {
throw SkylarkRepositoryFunction.restart();
}
if (!fileValue.isFile()) {
throw new EvalException(Location.BUILTIN,
"Not a file: " + rootedPath.asPath().getPathString());
}
// A label do not contains space so it safe to use as a key.
markerData.put("FILE:" + label, Integer.toString(fileValue.realFileStateValue().hashCode()));
return new SkylarkPath(rootedPath.asPath());
}
private static boolean verifyLabelMarkerData(String key, String value, Environment env)
throws InterruptedException {
Preconditions.checkArgument(key.startsWith("FILE:"));
try {
Label label = Label.parseAbsolute(key.substring(5));
RootedPath rootedPath = getRootedPathFromLabel(label, env);
SkyKey fileSkyKey = FileValue.key(rootedPath);
FileValue fileValue = (FileValue) env.getValueOrThrow(fileSkyKey, IOException.class,
FileSymlinkException.class, InconsistentFilesystemException.class);
if (fileValue == null || !fileValue.isFile()) {
return false;
}
return Objects.equals(value, Integer.toString(fileValue.realFileStateValue().hashCode()));
} catch (LabelSyntaxException e) {
throw new IllegalStateException(
"Key " + key + " is not a correct file key (should be in form FILE:label)", e);
} catch (IOException | FileSymlinkException | InconsistentFilesystemException
| EvalException e) {
// Consider those exception to be a cause for invalidation
return false;
}
}
static boolean verifyMarkerDataForFiles(Map<String, String> markerData, Environment env)
throws InterruptedException {
for (Map.Entry<String, String> entry : markerData.entrySet()) {
if (entry.getKey().startsWith("FILE:")) {
if (!verifyLabelMarkerData(entry.getKey(), entry.getValue(), env)) {
return false;
}
}
}
return true;
}
}