// Copyright 2014 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.packages;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Function;
import com.google.common.base.Joiner;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import com.google.devtools.build.lib.cmdline.Label;
import com.google.devtools.build.lib.cmdline.LabelSyntaxException;
import com.google.devtools.build.lib.cmdline.LabelValidator;
import com.google.devtools.build.lib.cmdline.PackageIdentifier;
import com.google.devtools.build.lib.events.Event;
import com.google.devtools.build.lib.events.EventHandler;
import com.google.devtools.build.lib.events.Location;
import com.google.devtools.build.lib.events.NullEventHandler;
import com.google.devtools.build.lib.events.StoredEventHandler;
import com.google.devtools.build.lib.packages.Globber.BadGlobException;
import com.google.devtools.build.lib.packages.License.DistributionType;
import com.google.devtools.build.lib.packages.Preprocessor.AstAfterPreprocessing;
import com.google.devtools.build.lib.packages.RuleFactory.BuildLangTypedAttributeValuesMap;
import com.google.devtools.build.lib.skylarkinterface.Param;
import com.google.devtools.build.lib.skylarkinterface.SkylarkSignature;
import com.google.devtools.build.lib.skylarkinterface.SkylarkValue;
import com.google.devtools.build.lib.syntax.AssignmentStatement;
import com.google.devtools.build.lib.syntax.BaseFunction;
import com.google.devtools.build.lib.syntax.BazelLibrary;
import com.google.devtools.build.lib.syntax.BuildFileAST;
import com.google.devtools.build.lib.syntax.BuiltinFunction;
import com.google.devtools.build.lib.syntax.ClassObject;
import com.google.devtools.build.lib.syntax.Environment;
import com.google.devtools.build.lib.syntax.Environment.Extension;
import com.google.devtools.build.lib.syntax.Environment.Phase;
import com.google.devtools.build.lib.syntax.EvalException;
import com.google.devtools.build.lib.syntax.EvalUtils;
import com.google.devtools.build.lib.syntax.Expression;
import com.google.devtools.build.lib.syntax.FuncallExpression;
import com.google.devtools.build.lib.syntax.FunctionSignature;
import com.google.devtools.build.lib.syntax.GlobList;
import com.google.devtools.build.lib.syntax.Identifier;
import com.google.devtools.build.lib.syntax.Mutability;
import com.google.devtools.build.lib.syntax.ParserInputSource;
import com.google.devtools.build.lib.syntax.Runtime;
import com.google.devtools.build.lib.syntax.SkylarkDict;
import com.google.devtools.build.lib.syntax.SkylarkList;
import com.google.devtools.build.lib.syntax.SkylarkList.MutableList;
import com.google.devtools.build.lib.syntax.SkylarkSemanticsOptions;
import com.google.devtools.build.lib.syntax.SkylarkSignatureProcessor;
import com.google.devtools.build.lib.syntax.SkylarkUtils;
import com.google.devtools.build.lib.syntax.Statement;
import com.google.devtools.build.lib.syntax.Type;
import com.google.devtools.build.lib.syntax.Type.ConversionException;
import com.google.devtools.build.lib.util.Preconditions;
import com.google.devtools.build.lib.vfs.FileSystem;
import com.google.devtools.build.lib.vfs.FileSystemUtils;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.UnixGlob;
import com.google.devtools.common.options.Options;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.Future;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import java.util.logging.Logger;
import javax.annotation.Nullable;
/**
* The package factory is responsible for constructing Package instances
* from a BUILD file's abstract syntax tree (AST).
*
* <p>A PackageFactory is a heavy-weight object; create them sparingly.
* Typically only one is needed per client application.
*/
public final class PackageFactory {
/**
* An argument to the {@code package()} function.
*/
public abstract static class PackageArgument<T> {
private final String name;
private final Type<T> type;
protected PackageArgument(String name, Type<T> type) {
this.name = name;
this.type = type;
}
public String getName() {
return name;
}
private void convertAndProcess(
Package.Builder pkgBuilder, Location location, Object value)
throws EvalException {
T typedValue = type.convert(value, "'package' argument", pkgBuilder.getBuildFileLabel());
process(pkgBuilder, location, typedValue);
}
/**
* Process an argument.
*
* @param pkgBuilder the package builder to be mutated
* @param location the location of the {@code package} function for error reporting
* @param value the value of the argument. Typically passed to {@link Type#convert}
*/
protected abstract void process(
Package.Builder pkgBuilder, Location location, T value)
throws EvalException;
}
/**
* An extension to the global namespace of the BUILD language.
*/
// TODO(bazel-team): this is largely unrelated to syntax.Environment.Extension,
// and should probably be renamed PackageFactory.RuntimeExtension, since really,
// we're extending the Runtime with more classes.
public interface EnvironmentExtension {
/**
* Update the global environment with the identifiers this extension contributes.
*/
void update(Environment environment);
/**
* Update the global environment of WORKSPACE files.
*/
void updateWorkspace(Environment environment);
/**
* Returns the extra functions needed to be added to the Skylark native module.
*/
ImmutableList<BaseFunction> nativeModuleFunctions();
/**
* Returns the extra arguments to the {@code package()} statement.
*/
Iterable<PackageArgument<?>> getPackageArguments();
}
private static class DefaultVisibility extends PackageArgument<List<Label>> {
private DefaultVisibility() {
super("default_visibility", BuildType.LABEL_LIST);
}
@Override
protected void process(Package.Builder pkgBuilder, Location location,
List<Label> value) throws EvalException{
try {
pkgBuilder.setDefaultVisibility(getVisibility(pkgBuilder.getBuildFileLabel(), value));
} catch (EvalException e) {
throw new EvalException(location, e.getMessage());
}
}
}
private static class DefaultTestOnly extends PackageArgument<Boolean> {
private DefaultTestOnly() {
super("default_testonly", Type.BOOLEAN);
}
@Override
protected void process(Package.Builder pkgBuilder, Location location,
Boolean value) {
pkgBuilder.setDefaultTestonly(value);
}
}
private static class DefaultDeprecation extends PackageArgument<String> {
private DefaultDeprecation() {
super("default_deprecation", Type.STRING);
}
@Override
protected void process(Package.Builder pkgBuilder, Location location,
String value) {
pkgBuilder.setDefaultDeprecation(value);
}
}
private static class Features extends PackageArgument<List<String>> {
private Features() {
super("features", Type.STRING_LIST);
}
@Override
protected void process(Package.Builder pkgBuilder, Location location,
List<String> value) {
pkgBuilder.addFeatures(value);
}
}
private static class DefaultLicenses extends PackageArgument<License> {
private DefaultLicenses() {
super("licenses", BuildType.LICENSE);
}
@Override
protected void process(Package.Builder pkgBuilder, Location location,
License value) {
pkgBuilder.setDefaultLicense(value);
}
}
private static class DefaultDistribs extends PackageArgument<Set<DistributionType>> {
private DefaultDistribs() {
super("distribs", BuildType.DISTRIBUTIONS);
}
@Override
protected void process(Package.Builder pkgBuilder, Location location,
Set<DistributionType> value) {
pkgBuilder.setDefaultDistribs(value);
}
}
/**
* Declares the package() attribute specifying the default value for
* {@link RuleClass#COMPATIBLE_ENVIRONMENT_ATTR} when not explicitly specified.
*/
private static class DefaultCompatibleWith extends PackageArgument<List<Label>> {
private DefaultCompatibleWith() {
super(Package.DEFAULT_COMPATIBLE_WITH_ATTRIBUTE, BuildType.LABEL_LIST);
}
@Override
protected void process(Package.Builder pkgBuilder, Location location,
List<Label> value) {
pkgBuilder.setDefaultCompatibleWith(value, Package.DEFAULT_COMPATIBLE_WITH_ATTRIBUTE,
location);
}
}
/**
* Declares the package() attribute specifying the default value for
* {@link RuleClass#RESTRICTED_ENVIRONMENT_ATTR} when not explicitly specified.
*/
private static class DefaultRestrictedTo extends PackageArgument<List<Label>> {
private DefaultRestrictedTo() {
super(Package.DEFAULT_RESTRICTED_TO_ATTRIBUTE, BuildType.LABEL_LIST);
}
@Override
protected void process(Package.Builder pkgBuilder, Location location,
List<Label> value) {
pkgBuilder.setDefaultRestrictedTo(value, Package.DEFAULT_RESTRICTED_TO_ATTRIBUTE, location);
}
}
public static final String PKG_CONTEXT = "$pkg_context";
// Used outside of Bazel!
/** {@link Globber} that uses the legacy GlobCache. */
public static class LegacyGlobber implements Globber {
private final GlobCache globCache;
private final boolean sort;
private LegacyGlobber(GlobCache globCache, boolean sort) {
this.globCache = globCache;
this.sort = sort;
}
private static class Token extends Globber.Token {
public final List<String> includes;
public final List<String> excludes;
public final boolean excludeDirs;
public Token(List<String> includes, List<String> excludes, boolean excludeDirs) {
this.includes = includes;
this.excludes = excludes;
this.excludeDirs = excludeDirs;
}
}
@Override
public Token runAsync(List<String> includes, List<String> excludes, boolean excludeDirs)
throws BadGlobException {
for (String pattern : Iterables.concat(includes, excludes)) {
@SuppressWarnings("unused")
Future<?> possiblyIgnoredError = globCache.getGlobUnsortedAsync(pattern, excludeDirs);
}
return new Token(includes, excludes, excludeDirs);
}
@Override
public List<String> fetch(Globber.Token token) throws IOException, InterruptedException {
List<String> result;
Token legacyToken = (Token) token;
try {
result = globCache.globUnsorted(legacyToken.includes, legacyToken.excludes,
legacyToken.excludeDirs);
} catch (BadGlobException e) {
throw new IllegalStateException(e);
}
if (sort) {
Collections.sort(result);
}
return result;
}
@Override
public void onInterrupt() {
globCache.cancelBackgroundTasks();
}
@Override
public void onCompletion() {
globCache.finishBackgroundTasks();
}
}
private static final Logger LOG = Logger.getLogger(PackageFactory.class.getName());
private final RuleFactory ruleFactory;
private final RuleClassProvider ruleClassProvider;
private AtomicReference<? extends UnixGlob.FilesystemCalls> syscalls;
private final ThreadPoolExecutor threadPool;
private Map<String, String> platformSetRegexps;
private int maxDirectoriesToEagerlyVisitInGlobbing;
private final ImmutableList<EnvironmentExtension> environmentExtensions;
private final ImmutableMap<String, PackageArgument<?>> packageArguments;
private final Package.Builder.Helper packageBuilderHelper;
/** Builder for {@link PackageFactory} instances. Intended to only be used by unit tests. */
@VisibleForTesting
public abstract static class BuilderForTesting {
protected final String version = "test";
protected Iterable<EnvironmentExtension> environmentExtensions = ImmutableList.of();
protected Map<String, String> platformSetRegexps = null;
protected Function<RuleClass, AttributeContainer> attributeContainerFactory =
AttributeContainer.ATTRIBUTE_CONTAINER_FACTORY;
protected boolean doChecksForTesting = true;
public BuilderForTesting setEnvironmentExtensions(
Iterable<EnvironmentExtension> environmentExtensions) {
this.environmentExtensions = environmentExtensions;
return this;
}
public BuilderForTesting setPlatformSetRegexps(Map<String, String> platformSetRegexps) {
this.platformSetRegexps = platformSetRegexps;
return this;
}
public BuilderForTesting disableChecks() {
this.doChecksForTesting = false;
return this;
}
public abstract PackageFactory build(RuleClassProvider ruleClassProvider, FileSystem fs);
}
/**
* Factory for {@link PackageFactory.BuilderForTesting} instances. Intended to only be used by
* unit tests.
*/
@VisibleForTesting
public abstract static class BuilderFactoryForTesting {
public abstract BuilderForTesting builder();
}
/**
* Constructs a {@code PackageFactory} instance with a specific glob path translator
* and rule factory.
*
* <p>Only intended to be called by BlazeRuntime or {@link FactoryForTesting#create}.
*
* <p>Do not call this constructor directly in tests; please use
* TestConstants#PACKAGE_FACTORY_FACTORY_FOR_TESTING instead.
*/
public PackageFactory(
RuleClassProvider ruleClassProvider,
Map<String, String> platformSetRegexps,
Function<RuleClass, AttributeContainer> attributeContainerFactory,
Iterable<EnvironmentExtension> environmentExtensions,
String version,
Package.Builder.Helper packageBuilderHelper) {
this.platformSetRegexps = platformSetRegexps;
this.ruleFactory = new RuleFactory(ruleClassProvider, attributeContainerFactory);
this.ruleClassProvider = ruleClassProvider;
threadPool = new ThreadPoolExecutor(100, 100, 15L, TimeUnit.SECONDS,
new LinkedBlockingQueue<Runnable>(),
new ThreadFactoryBuilder().setNameFormat("Legacy globber %d").build());
// Do not consume threads when not in use.
threadPool.allowCoreThreadTimeOut(true);
this.environmentExtensions = ImmutableList.copyOf(environmentExtensions);
this.packageArguments = createPackageArguments();
this.nativeModule = newNativeModule();
this.workspaceNativeModule = WorkspaceFactory.newNativeModule(ruleClassProvider, version);
this.packageBuilderHelper = packageBuilderHelper;
}
/**
* Sets the syscalls cache used in globbing.
*/
public void setSyscalls(AtomicReference<? extends UnixGlob.FilesystemCalls> syscalls) {
this.syscalls = Preconditions.checkNotNull(syscalls);
}
/**
* Sets the max number of threads to use for globbing.
*/
public void setGlobbingThreads(int globbingThreads) {
threadPool.setCorePoolSize(globbingThreads);
threadPool.setMaximumPoolSize(globbingThreads);
}
public void setMaxDirectoriesToEagerlyVisitInGlobbing(
int maxDirectoriesToEagerlyVisitInGlobbing) {
this.maxDirectoriesToEagerlyVisitInGlobbing = maxDirectoriesToEagerlyVisitInGlobbing;
}
/**
* Returns the immutable, unordered set of names of all the known rule
* classes.
*/
public Set<String> getRuleClassNames() {
return ruleFactory.getRuleClassNames();
}
/**
* Returns the {@link RuleClass} for the specified rule class name.
*/
public RuleClass getRuleClass(String ruleClassName) {
return ruleFactory.getRuleClass(ruleClassName);
}
/**
* Returns the {@link RuleClassProvider} of this {@link PackageFactory}.
*/
public RuleClassProvider getRuleClassProvider() {
return ruleClassProvider;
}
public ImmutableList<EnvironmentExtension> getEnvironmentExtensions() {
return environmentExtensions;
}
/**
* Creates the list of arguments for the 'package' function.
*/
private ImmutableMap<String, PackageArgument<?>> createPackageArguments() {
ImmutableList.Builder<PackageArgument<?>> arguments =
ImmutableList.<PackageArgument<?>>builder()
.add(new DefaultDeprecation())
.add(new DefaultDistribs())
.add(new DefaultLicenses())
.add(new DefaultTestOnly())
.add(new DefaultVisibility())
.add(new Features())
.add(new DefaultCompatibleWith())
.add(new DefaultRestrictedTo());
for (EnvironmentExtension extension : environmentExtensions) {
arguments.addAll(extension.getPackageArguments());
}
ImmutableMap.Builder<String, PackageArgument<?>> packageArguments = ImmutableMap.builder();
for (PackageArgument<?> argument : arguments.build()) {
packageArguments.put(argument.getName(), argument);
}
return packageArguments.build();
}
/**
* ************************************************************************** Environment function
* factories.
*/
/**
* Returns a function-value implementing "glob" in the specified package context.
*
* @param async if true, start globs in the background but don't block on their completion. Only
* use this for heuristic preloading.
*/
@SkylarkSignature(
name = "glob",
objectType = Object.class,
returnType = SkylarkList.class,
doc = "Returns a list of files that match glob search pattern.",
parameters = {
@Param(
name = "include",
type = SkylarkList.class,
generic1 = String.class,
doc = "a list of strings specifying patterns of files to include."
),
@Param(
name = "exclude",
type = SkylarkList.class,
generic1 = String.class,
defaultValue = "[]",
positional = false,
named = true,
doc = "a list of strings specifying patterns of files to exclude."
),
// TODO(bazel-team): migrate all existing code to use boolean instead?
@Param(
name = "exclude_directories",
type = Integer.class,
defaultValue = "1",
positional = false,
named = true,
doc = "a integer that if non-zero indicates directories should not be matched."
)
},
documented = false,
useAst = true,
useEnvironment = true
)
private static final BuiltinFunction.Factory newGlobFunction =
new BuiltinFunction.Factory("glob") {
public BuiltinFunction create(final PackageContext originalContext, final boolean async) {
return new BuiltinFunction("glob", this) {
public SkylarkList invoke(
SkylarkList include,
SkylarkList exclude,
Integer excludeDirectories,
FuncallExpression ast,
Environment env)
throws EvalException, ConversionException, InterruptedException {
return callGlob(
originalContext, async, include, exclude, excludeDirectories != 0, ast, env);
}
};
}
};
static SkylarkList callGlob(@Nullable PackageContext originalContext,
boolean async, Object include, Object exclude, boolean excludeDirs,
FuncallExpression ast, Environment env)
throws EvalException, ConversionException, InterruptedException {
// Skylark build extensions need to get the PackageContext from the Environment;
// async glob functions cannot do the same because the Environment is not thread safe.
PackageContext context;
if (originalContext == null) {
Preconditions.checkArgument(!async);
context = getContext(env, ast);
} else {
context = originalContext;
}
List<String> includes = Type.STRING_LIST.convert(include, "'glob' argument");
List<String> excludes = Type.STRING_LIST.convert(exclude, "'glob' argument");
GlobList<String> globList;
if (async) {
try {
context.globber.runAsync(includes, excludes, excludeDirs);
} catch (BadGlobException e) {
// Ignore: errors will appear during the actual evaluation of the package.
}
globList = GlobList.captureResults(includes, excludes, ImmutableList.<String>of());
} else {
globList = handleGlob(includes, excludes, excludeDirs, context, ast);
}
return new MutableList(globList, env);
}
/**
* Adds a glob to the package, reporting any errors it finds.
*
* @param includes the list of includes which must be non-null
* @param excludes the list of excludes which must be non-null
* @param context the package context
* @param ast the AST
* @return the list of matches
* @throws EvalException if globbing failed
*/
private static GlobList<String> handleGlob(List<String> includes, List<String> excludes,
boolean excludeDirs, PackageContext context, FuncallExpression ast)
throws EvalException, InterruptedException {
try {
Globber.Token globToken = context.globber.runAsync(includes, excludes, excludeDirs);
List<String> matches = context.globber.fetch(globToken);
return GlobList.captureResults(includes, excludes, matches);
} catch (IOException expected) {
context.eventHandler.handle(Event.error(ast.getLocation(),
"error globbing [" + Joiner.on(", ").join(includes) + "]: " + expected.getMessage()));
context.pkgBuilder.setContainsErrors();
return GlobList.captureResults(includes, excludes, ImmutableList.<String>of());
} catch (BadGlobException e) {
throw new EvalException(ast.getLocation(), e.getMessage());
}
}
/**
* Returns a function value implementing the "mocksubinclude" function, emitted by the
* PythonPreprocessor. We annotate the package with additional dependencies. (A 'real' subinclude
* will never be seen by the parser, because the presence of "subinclude" triggers preprocessing.)
*/
@SkylarkSignature(
name = "mocksubinclude",
returnType = Runtime.NoneType.class,
doc = "implement the mocksubinclude function emitted by the PythonPreprocessor.",
parameters = {
@Param(name = "label", type = Object.class, doc = "a label designator."),
@Param(name = "path", type = String.class, doc = "a path.")
},
documented = false,
useLocation = true
)
private static final BuiltinFunction.Factory newMockSubincludeFunction =
new BuiltinFunction.Factory("mocksubinclude") {
public BuiltinFunction create(final PackageContext context) {
return new BuiltinFunction("mocksubinclude", this) {
public Runtime.NoneType invoke(Object labelO, String pathString, Location loc)
throws ConversionException {
Label label =
BuildType.LABEL.convert(
labelO, "'mocksubinclude' argument", context.pkgBuilder.getBuildFileLabel());
Path path =
pathString.isEmpty()
? null
: context.pkgBuilder.getFilename().getRelative(pathString);
// A subinclude within a package counts as a file declaration.
if (label.getPackageIdentifier().equals(context.pkgBuilder.getPackageIdentifier())) {
if (loc == null) {
loc = Location.fromFile(context.pkgBuilder.getFilename());
}
context.pkgBuilder.createInputFileMaybe(label, loc);
}
context.pkgBuilder.addSubinclude(label, path);
return Runtime.NONE;
}
};
}
};
/**
* Returns a function value implementing "environment_group" in the specified package context.
* Syntax is as follows:
*
* <pre>{@code
* environment_group(
* name = "sample_group",
* environments = [":env1", ":env2", ...],
* defaults = [":env1", ...]
* )
* }</pre>
*
* <p>Where ":env1", "env2", ... are all environment rules declared in the same package. All
* parameters are mandatory.
*/
@SkylarkSignature(name = "environment_group", returnType = Runtime.NoneType.class,
doc = "Defines a set of related environments that can be tagged onto rules to prevent"
+ "incompatible rules from depending on each other.",
parameters = {
@Param(name = "name", type = String.class, positional = false, named = true,
doc = "The name of the rule."),
// Both parameter below are lists of label designators
@Param(name = "environments", type = SkylarkList.class, generic1 = Object.class,
positional = false, named = true,
doc = "A list of Labels for the environments to be grouped, from the same package."),
@Param(name = "defaults", type = SkylarkList.class, generic1 = Object.class,
positional = false, named = true,
doc = "A list of Labels.")}, // TODO(bazel-team): document what that is
documented = false, useLocation = true)
private static final BuiltinFunction.Factory newEnvironmentGroupFunction =
new BuiltinFunction.Factory("environment_group") {
public BuiltinFunction create(final PackageContext context) {
return new BuiltinFunction("environment_group", this) {
public Runtime.NoneType invoke(
String name, SkylarkList environmentsList, SkylarkList defaultsList,
Location loc) throws EvalException, ConversionException {
List<Label> environments = BuildType.LABEL_LIST.convert(environmentsList,
"'environment_group argument'", context.pkgBuilder.getBuildFileLabel());
List<Label> defaults = BuildType.LABEL_LIST.convert(defaultsList,
"'environment_group argument'", context.pkgBuilder.getBuildFileLabel());
try {
context.pkgBuilder.addEnvironmentGroup(
name, environments, defaults, context.eventHandler, loc);
return Runtime.NONE;
} catch (LabelSyntaxException e) {
throw new EvalException(loc,
"environment group has invalid name: " + name + ": " + e.getMessage());
} catch (Package.NameConflictException e) {
throw new EvalException(loc, e.getMessage());
}
}
};
}
};
/** Returns a function-value implementing "exports_files" in the specified package context. */
@SkylarkSignature(
name = "exports_files",
returnType = Runtime.NoneType.class,
doc = "Declare a set of files as exported",
parameters = {
@Param(
name = "srcs",
type = SkylarkList.class,
generic1 = String.class,
doc = "A list of strings, the names of the files to export."
),
// TODO(blaze-team): make it possible to express a list of label designators,
// i.e. a java List or Skylark list of Label or String.
@Param(
name = "visibility",
type = SkylarkList.class,
noneable = true,
defaultValue = "None",
doc =
"A list of Labels specifying the visibility of the exported files "
+ "(defaults to public)."
),
@Param(
name = "licenses",
type = SkylarkList.class,
generic1 = String.class,
noneable = true,
defaultValue = "None",
doc = "A list of strings specifying the licenses used in the exported code."
)
},
documented = false,
useAst = true,
useEnvironment = true
)
private static final BuiltinFunction.Factory newExportsFilesFunction =
new BuiltinFunction.Factory("exports_files") {
public BuiltinFunction create() {
return new BuiltinFunction("exports_files", this) {
public Runtime.NoneType invoke(
SkylarkList srcs,
Object visibility,
Object licenses,
FuncallExpression ast,
Environment env)
throws EvalException, ConversionException {
return callExportsFiles(srcs, visibility, licenses, ast, env);
}
};
}
};
static Runtime.NoneType callExportsFiles(Object srcs, Object visibilityO, Object licensesO,
FuncallExpression ast, Environment env) throws EvalException, ConversionException {
Package.Builder pkgBuilder = getContext(env, ast).pkgBuilder;
List<String> files = Type.STRING_LIST.convert(srcs, "'exports_files' operand");
RuleVisibility visibility;
try {
visibility = EvalUtils.isNullOrNone(visibilityO)
? ConstantRuleVisibility.PUBLIC
: getVisibility(pkgBuilder.getBuildFileLabel(), BuildType.LABEL_LIST.convert(
visibilityO,
"'exports_files' operand",
pkgBuilder.getBuildFileLabel()));
} catch (EvalException e) {
throw new EvalException(ast.getLocation(), e.getMessage());
}
// TODO(bazel-team): is licenses plural or singular?
License license = BuildType.LICENSE.convertOptional(licensesO, "'exports_files' operand");
for (String file : files) {
String errorMessage = LabelValidator.validateTargetName(file);
if (errorMessage != null) {
throw new EvalException(ast.getLocation(), errorMessage);
}
try {
InputFile inputFile = pkgBuilder.createInputFile(file, ast.getLocation());
if (inputFile.isVisibilitySpecified()
&& inputFile.getVisibility() != visibility) {
throw new EvalException(ast.getLocation(),
String.format("visibility for exported file '%s' declared twice",
inputFile.getName()));
}
if (license != null && inputFile.isLicenseSpecified()) {
throw new EvalException(ast.getLocation(),
String.format("licenses for exported file '%s' declared twice",
inputFile.getName()));
}
if (license == null && pkgBuilder.getDefaultLicense() == License.NO_LICENSE
&& RuleClass.isThirdPartyPackage(pkgBuilder.getPackageIdentifier())) {
throw new EvalException(ast.getLocation(),
"third-party file '" + inputFile.getName() + "' lacks a license declaration "
+ "with one of the following types: notice, reciprocal, permissive, "
+ "restricted, unencumbered, by_exception_only");
}
pkgBuilder.setVisibilityAndLicense(inputFile, visibility, license);
} catch (Package.Builder.GeneratedLabelConflict e) {
throw new EvalException(ast.getLocation(), e.getMessage());
}
}
return Runtime.NONE;
}
@SkylarkSignature(
name = "package_name",
objectType = SkylarkNativeModule.class,
returnType = String.class,
doc =
"The name of the package being evaluated. "
+ "For example, in the BUILD file <code>some/package/BUILD</code>, its value "
+ "will be <code>some/package</code>. "
+ "If the BUILD file calls a function defined in a .bzl file, "
+ "<code>package_name()</code> will match the caller BUILD file package. "
+ "This function is equivalent to the deprecated variable <code>PACKAGE_NAME</code>.",
parameters = {},
useEnvironment = true
)
private static final BuiltinFunction packageNameFunction =
new BuiltinFunction("package_name") {
public String invoke(Environment env) throws EvalException {
return (String) env.lookup("PACKAGE_NAME");
}
};
@SkylarkSignature(
name = "repository_name",
objectType = SkylarkNativeModule.class,
returnType = String.class,
doc =
"The name of the repository the rule or build extension is called from. "
+ "For example, in packages that are called into existence by the WORKSPACE stanza "
+ "<code>local_repository(name='local', path=...)</code> it will be set to "
+ "<code>@local</code>. In packages in the main repository, it will be empty. This "
+ "function is equivalent to the deprecated variable <code>REPOSITORY_NAME</code>.",
parameters = {},
useEnvironment = true
)
private static final BuiltinFunction repositoryNameFunction =
new BuiltinFunction("repository_name") {
public String invoke(Environment env) throws EvalException, ConversionException {
return (String) env.lookup("REPOSITORY_NAME");
}
};
/**
* Returns a function-value implementing "licenses" in the specified package
* context.
* TODO(bazel-team): Remove in favor of package.licenses.
*/
@SkylarkSignature(name = "licenses", returnType = Runtime.NoneType.class,
doc = "Declare the license(s) for the code in the current package.",
parameters = {
@Param(name = "license_strings", type = SkylarkList.class, generic1 = String.class,
doc = "A list of strings, the names of the licenses used.")},
documented = false, useLocation = true)
private static final BuiltinFunction.Factory newLicensesFunction =
new BuiltinFunction.Factory("licenses") {
public BuiltinFunction create(final PackageContext context) {
return new BuiltinFunction("licenses", this) {
public Runtime.NoneType invoke(SkylarkList licensesList, Location loc) {
try {
License license = BuildType.LICENSE.convert(licensesList, "'licenses' operand");
context.pkgBuilder.setDefaultLicense(license);
} catch (ConversionException e) {
context.eventHandler.handle(Event.error(loc, e.getMessage()));
context.pkgBuilder.setContainsErrors();
}
return Runtime.NONE;
}
};
}
};
/**
* Returns a function-value implementing "distribs" in the specified package
* context.
*/
// TODO(bazel-team): Remove in favor of package.distribs.
// TODO(bazel-team): Remove all these new*Function-s and/or have static functions
// that consult the context dynamically via getContext(env, ast) since we have that,
// and share the functions with the native package... which requires unifying the List types.
@SkylarkSignature(name = "distribs", returnType = Runtime.NoneType.class,
doc = "Declare the distribution(s) for the code in the current package.",
parameters = {
@Param(name = "distribution_strings", type = Object.class,
doc = "The distributions.")},
documented = false, useLocation = true)
private static final BuiltinFunction.Factory newDistribsFunction =
new BuiltinFunction.Factory("distribs") {
public BuiltinFunction create(final PackageContext context) {
return new BuiltinFunction("distribs", this) {
public Runtime.NoneType invoke(Object object, Location loc) {
try {
Set<DistributionType> distribs = BuildType.DISTRIBUTIONS.convert(object,
"'distribs' operand");
context.pkgBuilder.setDefaultDistribs(distribs);
} catch (ConversionException e) {
context.eventHandler.handle(Event.error(loc, e.getMessage()));
context.pkgBuilder.setContainsErrors();
}
return Runtime.NONE;
}
};
}
};
@SkylarkSignature(
name = "package_group",
returnType = Runtime.NoneType.class,
doc = "Declare a set of files as exported.",
parameters = {
@Param(
name = "name",
type = String.class,
named = true,
positional = false,
doc = "The name of the rule."
),
@Param(
name = "packages",
type = SkylarkList.class,
generic1 = String.class,
defaultValue = "[]",
named = true,
positional = false,
doc = "A list of Strings specifying the packages grouped."
),
// java list or list of label designators: Label or String
@Param(
name = "includes",
type = SkylarkList.class,
generic1 = Object.class,
defaultValue = "[]",
named = true,
positional = false,
doc = "A list of Label specifiers for the files to include."
)
},
documented = false,
useAst = true,
useEnvironment = true
)
private static final BuiltinFunction.Factory newPackageGroupFunction =
new BuiltinFunction.Factory("package_group") {
public BuiltinFunction create() {
return new BuiltinFunction("package_group", this) {
public Runtime.NoneType invoke(
String name,
SkylarkList packages,
SkylarkList includes,
FuncallExpression ast,
Environment env)
throws EvalException, ConversionException {
return callPackageFunction(name, packages, includes, ast, env);
}
};
}
};
@Nullable
static SkylarkDict<String, Object> callGetRuleFunction(
String name, FuncallExpression ast, Environment env)
throws EvalException, ConversionException {
PackageContext context = getContext(env, ast);
Target target = context.pkgBuilder.getTarget(name);
return targetDict(target, ast.getLocation(), env);
}
@Nullable
private static SkylarkDict<String, Object> targetDict(
Target target, Location loc, Environment env)
throws NotRepresentableException, EvalException {
if (target == null || !(target instanceof Rule)) {
return null;
}
SkylarkDict<String, Object> values = SkylarkDict.<String, Object>of(env);
Rule rule = (Rule) target;
AttributeContainer cont = rule.getAttributeContainer();
for (Attribute attr : rule.getAttributes()) {
if (!Character.isAlphabetic(attr.getName().charAt(0))) {
continue;
}
if (attr.getName().equals("distribs")) {
// attribute distribs: cannot represent type class java.util.Collections$SingletonSet
// in Skylark: [INTERNAL].
continue;
}
try {
Object val = skylarkifyValue(cont.getAttr(attr.getName()), target.getPackage());
if (val == null) {
continue;
}
values.put(attr.getName(), val, loc, env);
} catch (NotRepresentableException e) {
throw new NotRepresentableException(
String.format(
"target %s, attribute %s: %s", target.getName(), attr.getName(), e.getMessage()));
}
}
values.put("name", rule.getName(), loc, env);
values.put("kind", rule.getRuleClass(), loc, env);
return values;
}
static class NotRepresentableException extends EvalException {
NotRepresentableException(String msg) {
super(null, msg);
}
};
/**
* Converts back to type that will work in BUILD and skylark,
* such as string instead of label, SkylarkList instead of List,
* Returns null if we don't want to export the value.
*
* <p>All of the types returned are immutable. If we want, we can change this to
* immutable in the future, but this is the safe choice for now.
*/
@Nullable
private static Object skylarkifyValue(Object val, Package pkg) throws NotRepresentableException {
// TODO(bazel-team): the location of this function is ad-hoc. Arguably, the conversion
// from Java native types to Skylark types should be part of the Type class hierarchy,
if (val == null) {
return null;
}
if (val instanceof Boolean) {
return val;
}
if (val instanceof Integer) {
return val;
}
if (val instanceof String) {
return val;
}
if (val instanceof TriState) {
switch ((TriState) val) {
case AUTO:
return Integer.valueOf(-1);
case YES:
return Integer.valueOf(1);
case NO:
return Integer.valueOf(0);
}
}
if (val instanceof Label) {
Label l = (Label) val;
if (l.getPackageName().equals(pkg.getName())) {
return ":" + l.getName();
}
return l.getCanonicalForm();
}
if (val instanceof List) {
List<Object> l = new ArrayList<>();
for (Object o : (List) val) {
Object elt = skylarkifyValue(o, pkg);
if (elt == null) {
continue;
}
l.add(elt);
}
return SkylarkList.Tuple.copyOf(l);
}
if (val instanceof Map) {
Map<Object, Object> m = new TreeMap<>();
for (Map.Entry<?, ?> e : ((Map<?, ?>) val).entrySet()) {
Object key = skylarkifyValue(e.getKey(), pkg);
Object mapVal = skylarkifyValue(e.getValue(), pkg);
if (key == null || mapVal == null) {
continue;
}
m.put(key, mapVal);
}
return m;
}
if (val.getClass().isAnonymousClass()) {
// Computed defaults. They will be represented as
// "deprecation": com.google.devtools.build.lib.analysis.BaseRuleClasses$2@6960884a,
// Filter them until we invent something more clever.
return null;
}
if (val instanceof SkylarkValue) {
return val;
}
if (val instanceof License) {
// TODO(bazel-team): convert License.getLicenseTypes() to a list of strings.
return null;
}
if (val instanceof BuildType.SelectorList) {
// This is terrible:
// 1) this value is opaque, and not a BUILD value, so it cannot be used in rule arguments
// 2) its representation has a pointer address, so it breaks hermeticity.
//
// Even though this is clearly imperfect, we return this value because otherwise
// native.rules() fails if there is any rule using a select() in the BUILD file.
//
// To remedy this, we should return a syntax.SelectorList. To do so, we have to
// 1) recurse into the Selector contents of SelectorList, so those values are skylarkified too
// 2) get the right Class<?> value. We could probably get at that by looking at
// ((SelectorList)val).getSelectors().first().getEntries().first().getClass().
return val;
}
// We are explicit about types we don't understand so we minimize changes to existing callers
// if we add more types that we can represent.
throw new NotRepresentableException(
String.format(
"cannot represent %s (%s) in skylark", val.toString(), val.getClass().toString()));
}
static SkylarkDict<String, SkylarkDict<String, Object>> callGetRulesFunction(
FuncallExpression ast, Environment env)
throws EvalException {
PackageContext context = getContext(env, ast);
Collection<Target> targets = context.pkgBuilder.getTargets();
Location loc = ast.getLocation();
SkylarkDict<String, SkylarkDict<String, Object>> rules = SkylarkDict.of(env);
for (Target t : targets) {
if (t instanceof Rule) {
SkylarkDict<String, Object> m = targetDict(t, loc, env);
Preconditions.checkNotNull(m);
rules.put(t.getName(), m, loc, env);
}
}
return rules;
}
static Runtime.NoneType callPackageFunction(String name, Object packagesO, Object includesO,
FuncallExpression ast, Environment env) throws EvalException, ConversionException {
PackageContext context = getContext(env, ast);
List<String> packages = Type.STRING_LIST.convert(
packagesO, "'package_group.packages argument'");
List<Label> includes = BuildType.LABEL_LIST.convert(includesO,
"'package_group.includes argument'", context.pkgBuilder.getBuildFileLabel());
try {
context.pkgBuilder.addPackageGroup(name, packages, includes, context.eventHandler,
ast.getLocation());
return Runtime.NONE;
} catch (LabelSyntaxException e) {
throw new EvalException(ast.getLocation(),
"package group has invalid name: " + name + ": " + e.getMessage());
} catch (Package.NameConflictException e) {
throw new EvalException(ast.getLocation(), e.getMessage());
}
}
public static RuleVisibility getVisibility(Label ruleLabel, List<Label> original)
throws EvalException {
RuleVisibility result;
result = ConstantRuleVisibility.tryParse(original);
if (result != null) {
return result;
}
result = PackageGroupsRuleVisibility.tryParse(ruleLabel, original);
return result;
}
/**
* Returns a function-value implementing "package" in the specified package
* context.
*/
private static BaseFunction newPackageFunction(
final ImmutableMap<String, PackageArgument<?>> packageArguments) {
// Flatten the map of argument name of PackageArgument specifier in two co-indexed arrays:
// one for the argument names, to create a FunctionSignature when we create the function,
// one of the PackageArgument specifiers, over which to iterate at every function invocation
// at the same time that we iterate over the function arguments.
final int numArgs = packageArguments.size();
final String[] argumentNames = new String[numArgs];
final PackageArgument<?>[] argumentSpecifiers = new PackageArgument<?>[numArgs];
int i = 0;
for (Map.Entry<String, PackageArgument<?>> entry : packageArguments.entrySet()) {
argumentNames[i] = entry.getKey();
argumentSpecifiers[i++] = entry.getValue();
}
return new BaseFunction("package", FunctionSignature.namedOnly(0, argumentNames)) {
@Override
public Object call(Object[] arguments, FuncallExpression ast, Environment env)
throws EvalException {
Package.Builder pkgBuilder = getContext(env, ast).pkgBuilder;
// Validate parameter list
if (pkgBuilder.isPackageFunctionUsed()) {
throw new EvalException(ast.getLocation(),
"'package' can only be used once per BUILD file");
}
pkgBuilder.setPackageFunctionUsed();
// Parse params
boolean foundParameter = false;
for (int i = 0; i < numArgs; i++) {
Object value = arguments[i];
if (value != null) {
foundParameter = true;
argumentSpecifiers[i].convertAndProcess(pkgBuilder, ast.getLocation(), value);
}
}
if (!foundParameter) {
throw new EvalException(ast.getLocation(),
"at least one argument must be given to the 'package' function");
}
return Runtime.NONE;
}
};
}
// Helper function for createRuleFunction.
private static void addRule(RuleFactory ruleFactory,
String ruleClassName,
PackageContext context,
Map<String, Object> kwargs,
FuncallExpression ast,
Environment env)
throws RuleFactory.InvalidRuleException, Package.NameConflictException, InterruptedException {
RuleClass ruleClass = getBuiltInRuleClass(ruleClassName, ruleFactory);
BuildLangTypedAttributeValuesMap attributeValues = new BuildLangTypedAttributeValuesMap(kwargs);
AttributeContainer attributeContainer = ruleFactory.getAttributeContainer(ruleClass);
RuleFactory.createAndAddRule(context, ruleClass, attributeValues, ast, env, attributeContainer);
}
private static RuleClass getBuiltInRuleClass(String ruleClassName, RuleFactory ruleFactory) {
if (ruleFactory.getRuleClassNames().contains(ruleClassName)) {
return ruleFactory.getRuleClass(ruleClassName);
}
throw new IllegalArgumentException("no such rule class: " + ruleClassName);
}
/**
* Get the PackageContext by looking up in the environment.
*/
public static PackageContext getContext(Environment env, FuncallExpression ast)
throws EvalException {
PackageContext value = (PackageContext) env.lookup(PKG_CONTEXT);
if (value == null) {
// if PKG_CONTEXT is missing, we're not called from a BUILD file. This happens if someone
// uses native.some_func() in the wrong place.
throw new EvalException(ast.getLocation(),
"The native module cannot be accessed from here. "
+ "Wrap the function in a macro and call it from a BUILD file");
}
return value;
}
/**
* Returns a function-value implementing the build rule "ruleClass" (e.g. cc_library) in the
* specified package context.
*/
private static BuiltinFunction newRuleFunction(
final RuleFactory ruleFactory, final String ruleClass) {
return new BuiltinFunction(ruleClass, FunctionSignature.KWARGS, BuiltinFunction.USE_AST_ENV) {
@SuppressWarnings({"unchecked", "unused"})
public Runtime.NoneType invoke(Map<String, Object> kwargs,
FuncallExpression ast, Environment env)
throws EvalException, InterruptedException {
env.checkLoadingOrWorkspacePhase(ruleClass, ast.getLocation());
try {
addRule(ruleFactory, ruleClass, getContext(env, ast), kwargs, ast, env);
} catch (RuleFactory.InvalidRuleException | Package.NameConflictException e) {
throw new EvalException(ast.getLocation(), e.getMessage());
}
return Runtime.NONE;
}
};
}
/**
* Package creation.
*/
/**
* Loads, scans parses and evaluates the build file at "buildFile", and creates and returns a
* Package builder instance capable of building a package identified by "packageId".
*
* <p>This method returns a builder to allow the caller to do additional work, if necessary.
*
* <p>This method assumes "packageId" is a valid package name according to the {@link
* LabelValidator#validatePackageName} heuristic.
*
* <p>See {@link #evaluateBuildFile} for information on AST retention.
*
* <p>Executes {@code globber.onCompletion()} on completion and executes {@code
* globber.onInterrupt()} on an {@link InterruptedException}.
*/
// Used outside of bazel!
public Package.Builder createPackage(
String workspaceName,
PackageIdentifier packageId,
Path buildFile,
ParserInputSource input,
List<Statement> preludeStatements,
Map<String, Extension> imports,
ImmutableList<Label> skylarkFileDependencies,
RuleVisibility defaultVisibility,
SkylarkSemanticsOptions skylarkSemantics,
Globber globber)
throws InterruptedException {
StoredEventHandler localReporterForParsing = new StoredEventHandler();
// Run the lexer and parser with a local reporter, so that errors from other threads do not
// show up below.
BuildFileAST buildFileAST =
parseBuildFile(packageId, input, preludeStatements, localReporterForParsing);
AstAfterPreprocessing astAfterPreprocessing =
new AstAfterPreprocessing(buildFileAST, localReporterForParsing);
return createPackageFromPreprocessingAst(
workspaceName,
packageId,
buildFile,
astAfterPreprocessing,
imports,
skylarkFileDependencies,
defaultVisibility,
skylarkSemantics,
globber);
}
public static BuildFileAST parseBuildFile(PackageIdentifier packageId, ParserInputSource in,
List<Statement> preludeStatements, EventHandler eventHandler) {
// Logged messages are used as a testability hook tracing the parsing progress
LOG.fine("Starting to parse " + packageId);
BuildFileAST buildFileAST = BuildFileAST.parseBuildFile(in, preludeStatements, eventHandler);
LOG.fine("Finished parsing of " + packageId);
return buildFileAST;
}
public Package.Builder createPackageFromPreprocessingAst(
String workspaceName,
PackageIdentifier packageId,
Path buildFile,
Preprocessor.AstAfterPreprocessing astAfterPreprocessing,
Map<String, Extension> imports,
ImmutableList<Label> skylarkFileDependencies,
RuleVisibility defaultVisibility,
SkylarkSemanticsOptions skylarkSemantics,
Globber globber) throws InterruptedException {
MakeEnvironment.Builder makeEnv = new MakeEnvironment.Builder();
if (platformSetRegexps != null) {
makeEnv.setPlatformSetRegexps(platformSetRegexps);
}
try {
// At this point the package is guaranteed to exist. It may have parse or
// evaluation errors, resulting in a diminished number of rules.
prefetchGlobs(
packageId,
astAfterPreprocessing.ast,
buildFile,
globber,
defaultVisibility,
skylarkSemantics,
makeEnv,
imports);
return evaluateBuildFile(
workspaceName,
packageId,
astAfterPreprocessing.ast,
buildFile,
globber,
astAfterPreprocessing.allEvents,
defaultVisibility,
skylarkSemantics,
false /* containsError */,
makeEnv,
imports,
skylarkFileDependencies);
} catch (InterruptedException e) {
globber.onInterrupt();
throw e;
} finally {
globber.onCompletion();
}
}
@VisibleForTesting
public Package.Builder newExternalPackageBuilder(Path workspacePath, String runfilesPrefix) {
return Package.newExternalPackageBuilder(packageBuilderHelper, workspacePath, runfilesPrefix);
}
@VisibleForTesting
public Package.Builder newPackageBuilder(PackageIdentifier packageId, String runfilesPrefix) {
return new Package.Builder(packageBuilderHelper, packageId, runfilesPrefix);
}
@VisibleForTesting
public Package createPackageForTesting(
PackageIdentifier packageId,
Path buildFile,
CachingPackageLocator locator,
EventHandler eventHandler)
throws NoSuchPackageException, InterruptedException {
Package externalPkg = newExternalPackageBuilder(
buildFile.getRelative("WORKSPACE"), "TESTING").build();
return createPackageForTesting(packageId, externalPkg, buildFile, locator, eventHandler);
}
/**
* Same as createPackage, but does the required validation of "packageName" first,
* throwing a {@link NoSuchPackageException} if the name is invalid.
*/
@VisibleForTesting
public Package createPackageForTesting(
PackageIdentifier packageId,
Package externalPkg,
Path buildFile,
CachingPackageLocator locator,
EventHandler eventHandler)
throws NoSuchPackageException, InterruptedException {
String error =
LabelValidator.validatePackageName(packageId.getPackageFragment().getPathString());
if (error != null) {
throw new BuildFileNotFoundException(
packageId, "illegal package name: '" + packageId + "' (" + error + ")");
}
byte[] buildFileBytes = maybeGetBuildFileBytes(buildFile, eventHandler);
if (buildFileBytes == null) {
throw new BuildFileContainsErrorsException(packageId, "IOException occured");
}
Globber globber = createLegacyGlobber(buildFile.getParentDirectory(), packageId, locator);
ParserInputSource input =
ParserInputSource.create(
FileSystemUtils.convertFromLatin1(buildFileBytes), buildFile.asFragment());
Package result =
createPackage(
externalPkg.getWorkspaceName(),
packageId,
buildFile,
input,
/*preludeStatements=*/ ImmutableList.<Statement>of(),
/*imports=*/ ImmutableMap.<String, Extension>of(),
/*skylarkFileDependencies=*/ ImmutableList.<Label>of(),
/*defaultVisibility=*/ ConstantRuleVisibility.PUBLIC,
Options.getDefaults(SkylarkSemanticsOptions.class),
globber)
.build();
Event.replayEventsOn(eventHandler, result.getEvents());
return result;
}
/** Returns a new {@link LegacyGlobber}. */
public LegacyGlobber createLegacyGlobber(
Path packageDirectory,
PackageIdentifier packageId,
CachingPackageLocator locator) {
return createLegacyGlobber(
new GlobCache(
packageDirectory,
packageId,
locator,
syscalls,
threadPool,
maxDirectoriesToEagerlyVisitInGlobbing));
}
/** Returns a new {@link LegacyGlobber}. */
public static LegacyGlobber createLegacyGlobber(GlobCache globCache) {
return new LegacyGlobber(globCache, /*sort=*/ true);
}
/**
* Returns a new {@link LegacyGlobber}, the same as in {@link #createLegacyGlobber}, except that
* the implementation of {@link Globber#fetch} intentionally breaks the contract and doesn't
* return sorted results.
*/
public LegacyGlobber createLegacyGlobberThatDoesntSort(
Path packageDirectory,
PackageIdentifier packageId,
CachingPackageLocator locator) {
return new LegacyGlobber(
new GlobCache(
packageDirectory,
packageId,
locator,
syscalls,
threadPool,
maxDirectoriesToEagerlyVisitInGlobbing),
/*sort=*/ false);
}
@Nullable
private byte[] maybeGetBuildFileBytes(Path buildFile, EventHandler eventHandler) {
try {
return FileSystemUtils.readWithKnownFileSize(buildFile, buildFile.getFileSize());
} catch (IOException e) {
eventHandler.handle(Event.error(Location.fromFile(buildFile), e.getMessage()));
return null;
}
}
/**
* This tuple holds the current package builder, current lexer, etc, for the
* duration of the evaluation of one BUILD file. (We use a PackageContext
* object in preference to storing these values in mutable fields of the
* PackageFactory.)
*
* <p>PLEASE NOTE: references to PackageContext objects are held by many
* BaseFunction closures, but should become unreachable once the Environment is
* discarded at the end of evaluation. Please be aware of your memory
* footprint when making changes here!
*/
public static class PackageContext {
final Package.Builder pkgBuilder;
final Globber globber;
final EventHandler eventHandler;
private final Function<RuleClass, AttributeContainer> attributeContainerFactory;
@VisibleForTesting
public PackageContext(
Package.Builder pkgBuilder,
Globber globber,
EventHandler eventHandler,
Function<RuleClass, AttributeContainer> attributeContainerFactory) {
this.pkgBuilder = pkgBuilder;
this.eventHandler = eventHandler;
this.globber = globber;
this.attributeContainerFactory = attributeContainerFactory;
}
/**
* Returns the Label of this Package.
*/
public Label getLabel() {
return pkgBuilder.getBuildFileLabel();
}
/**
* Returns the MakeEnvironment Builder of this Package.
*/
public MakeEnvironment.Builder getMakeEnvironment() {
return pkgBuilder.getMakeEnvironment();
}
/**
* Returns the builder of this Package.
*/
public Package.Builder getBuilder() {
return pkgBuilder;
}
public Function<RuleClass, AttributeContainer> getAttributeContainerFactory() {
return attributeContainerFactory;
}
}
private final ClassObject nativeModule;
private final ClassObject workspaceNativeModule;
/** @return the Skylark struct to bind to "native" */
public ClassObject getNativeModule(boolean workspace) {
return workspace ? workspaceNativeModule : nativeModule;
}
/**
* Returns a native module with the functions created using the {@link RuleClassProvider}
* of this {@link PackageFactory}.
*/
private ClassObject newNativeModule() {
ImmutableMap.Builder<String, Object> builder = new ImmutableMap.Builder<>();
for (String nativeFunction : Runtime.getFunctionNames(SkylarkNativeModule.class)) {
builder.put(nativeFunction, Runtime.getFunction(SkylarkNativeModule.class, nativeFunction));
}
for (String ruleClass : ruleFactory.getRuleClassNames()) {
builder.put(ruleClass, newRuleFunction(ruleFactory, ruleClass));
}
builder.put("package", newPackageFunction(packageArguments));
for (EnvironmentExtension extension : environmentExtensions) {
for (BaseFunction function : extension.nativeModuleFunctions()) {
builder.put(function.getName(), function);
}
}
return NativeClassObjectConstructor.STRUCT.create(
builder.build(), "no native function or rule '%s'");
}
/** A function that does nothing and ignores the arguments. */
private final BaseFunction noopFunction =
new BaseFunction("noop", FunctionSignature.KWARGS) {
@Override
public Object call(Object[] arguments, FuncallExpression ast, Environment env)
throws EvalException {
return Runtime.NONE;
}
};
/** @param fakeEnv specify if we declare no-op functions, or real functions. */
private void buildPkgEnv(
Environment pkgEnv,
PackageContext context,
RuleFactory ruleFactory,
PackageIdentifier packageId,
boolean fakeEnv) {
// TODO(bazel-team): remove the naked functions that are redundant with the nativeModule,
// or if not possible, at least make them straight copies from the native module variant.
// or better, use a common Environment.Frame for these common bindings
// (that shares a backing ImmutableMap for the bindings?)
pkgEnv
.setup("native", nativeModule)
.setup("distribs", newDistribsFunction.apply(context))
.setup("glob", newGlobFunction.apply(context, /*async=*/ false))
.setup("licenses", newLicensesFunction.apply(context))
.setup("mocksubinclude", newMockSubincludeFunction.apply(context))
.setup("exports_files", newExportsFilesFunction.apply())
.setup("package_group", newPackageGroupFunction.apply())
.setup("package", newPackageFunction(packageArguments))
.setup("package_name", packageNameFunction)
.setup("repository_name", repositoryNameFunction)
.setup("environment_group", newEnvironmentGroupFunction.apply(context));
for (String ruleClass : ruleFactory.getRuleClassNames()) {
BaseFunction ruleFunction = newRuleFunction(ruleFactory, ruleClass);
if (fakeEnv) {
pkgEnv.setup(ruleClass, ruleFunction);
} else {
pkgEnv.setup(ruleClass, noopFunction);
}
}
for (EnvironmentExtension extension : environmentExtensions) {
extension.update(pkgEnv);
}
pkgEnv.setupDynamic(PKG_CONTEXT, context);
pkgEnv.setupDynamic(Runtime.PKG_NAME, packageId.getPackageFragment().getPathString());
pkgEnv.setupDynamic(Runtime.REPOSITORY_NAME, packageId.getRepository().toString());
}
/**
* Called by a caller of {@link #createPackageFromPreprocessingAst} after this caller has fully
* loaded the package.
*/
public void afterDoneLoadingPackage(Package pkg) {
packageBuilderHelper.onLoadingComplete(pkg);
}
/**
* Constructs a Package instance, evaluates the BUILD-file AST inside the
* build environment, and populates the package with Rule instances as it
* goes. As with most programming languages, evaluation stops when an
* exception is encountered: no further rules after the point of failure will
* be constructed. We assume that rules constructed before the point of
* failure are valid; this assumption is not entirely correct, since a
* "vardef" after a rule declaration can affect the behavior of that rule.
*
* <p>Rule attribute checking is performed during evaluation. Each attribute
* must conform to the type specified for that <i>(rule class, attribute
* name)</i> pair. Errors reported at this stage include: missing value for
* mandatory attribute, value of wrong type. Such error cause Rule
* construction to be aborted, so the resulting package will have missing
* members.
*
* @see PackageFactory#PackageFactory
*/
@VisibleForTesting // used by PackageFactoryApparatus
public Package.Builder evaluateBuildFile(
String workspaceName,
PackageIdentifier packageId,
BuildFileAST buildFileAST,
Path buildFilePath,
Globber globber,
Iterable<Event> pastEvents,
RuleVisibility defaultVisibility,
SkylarkSemanticsOptions skylarkSemantics,
boolean containsError,
MakeEnvironment.Builder pkgMakeEnv,
Map<String, Extension> imports,
ImmutableList<Label> skylarkFileDependencies)
throws InterruptedException {
Package.Builder pkgBuilder = new Package.Builder(packageBuilderHelper.createFreshPackage(
packageId, ruleClassProvider.getRunfilesPrefix()));
StoredEventHandler eventHandler = new StoredEventHandler();
try (Mutability mutability = Mutability.create("package %s", packageId)) {
Environment pkgEnv =
Environment.builder(mutability)
.setGlobals(BazelLibrary.GLOBALS)
.setSemantics(skylarkSemantics)
.setEventHandler(eventHandler)
.setImportedExtensions(imports)
.setPhase(Phase.LOADING)
.build();
SkylarkUtils.setToolsRepository(pkgEnv, ruleClassProvider.getToolsRepository());
pkgBuilder.setFilename(buildFilePath)
.setMakeEnv(pkgMakeEnv)
.setDefaultVisibility(defaultVisibility)
// "defaultVisibility" comes from the command line. Let's give the BUILD file a chance to
// set default_visibility once, be reseting the PackageBuilder.defaultVisibilitySet flag.
.setDefaultVisibilitySet(false)
.setSkylarkFileDependencies(skylarkFileDependencies)
.setWorkspaceName(workspaceName);
Event.replayEventsOn(eventHandler, pastEvents);
// Stuff that closes over the package context:
PackageContext context =
new PackageContext(
pkgBuilder, globber, eventHandler, ruleFactory.getAttributeContainerFactory());
buildPkgEnv(pkgEnv, context, ruleFactory, packageId, true);
if (containsError) {
pkgBuilder.setContainsErrors();
}
if (!validatePackageIdentifier(packageId, buildFileAST.getLocation(), eventHandler)) {
pkgBuilder.setContainsErrors();
}
if (!validateAssignmentStatements(pkgEnv, buildFileAST, eventHandler)) {
pkgBuilder.setContainsErrors();
}
if (buildFileAST.containsErrors()) {
pkgBuilder.setContainsErrors();
}
// TODO(bazel-team): (2009) the invariant "if errors are reported, mark the package
// as containing errors" is strewn all over this class. Refactor to use an
// event sensor--and see if we can simplify the calling code in
// createPackage().
if (!buildFileAST.exec(pkgEnv, eventHandler)) {
pkgBuilder.setContainsErrors();
}
}
pkgBuilder.addEvents(eventHandler.getEvents());
return pkgBuilder;
}
/** Visit all targets and expand the globs in parallel. */
private void prefetchGlobs(
PackageIdentifier packageId,
BuildFileAST buildFileAST,
Path buildFilePath,
Globber globber,
RuleVisibility defaultVisibility,
SkylarkSemanticsOptions skylarkSemantics,
MakeEnvironment.Builder pkgMakeEnv,
Map<String, Extension> imports)
throws InterruptedException {
// TODO(bazel-team): It may be wasteful to evaluate the BUILD file here, only to throw away the
// result. It may be better to first scan the ast and see if there are even possibly any globs
// at all. Additionally, it's wasteful to execute Skylark code that cannot invoke globs. So one
// strategy would be to crawl the ast and tag statements whose execution cannot involve globs -
// these can be executed and their impact on the resulting package can be saved.
try (Mutability mutability = Mutability.create("prefetchGlobs for %s", packageId)) {
Environment pkgEnv =
Environment.builder(mutability)
.setGlobals(BazelLibrary.GLOBALS)
.setSemantics(skylarkSemantics)
.setEventHandler(NullEventHandler.INSTANCE)
.setImportedExtensions(imports)
.setPhase(Phase.LOADING)
.build();
SkylarkUtils.setToolsRepository(pkgEnv, ruleClassProvider.getToolsRepository());
Package.Builder pkgBuilder = new Package.Builder(packageBuilderHelper.createFreshPackage(
packageId, ruleClassProvider.getRunfilesPrefix()));
pkgBuilder.setFilename(buildFilePath)
.setMakeEnv(pkgMakeEnv)
.setDefaultVisibility(defaultVisibility)
// "defaultVisibility" comes from the command line. Let's give the BUILD file a chance to
// set default_visibility once, be reseting the PackageBuilder.defaultVisibilitySet flag.
.setDefaultVisibilitySet(false);
// Stuff that closes over the package context:
PackageContext context =
new PackageContext(
pkgBuilder,
globber,
NullEventHandler.INSTANCE,
ruleFactory.getAttributeContainerFactory());
buildPkgEnv(pkgEnv, context, ruleFactory, packageId, false);
try {
pkgEnv.update("glob", newGlobFunction.apply(context, /*async=*/true));
// The Fileset function is heavyweight in that it can run glob(). Avoid this during the
// preloading phase.
pkgEnv.update("FilesetEntry", noopFunction);
pkgEnv.update("vardef", noopFunction);
} catch (EvalException e) {
throw new AssertionError(e);
}
buildFileAST.exec(pkgEnv, NullEventHandler.INSTANCE);
}
}
/**
* Tests a build AST to ensure that it contains no assignment statements that
* redefine built-in build rules.
*
* @param pkgEnv a package environment initialized with all of the built-in
* build rules
* @param ast the build file AST to be tested
* @param eventHandler a eventHandler where any errors should be logged
* @return true if the build file contains no redefinitions of built-in
* functions
*/
// TODO(bazel-team): Remove this check. It should be moved to LValue.assign
private static boolean validateAssignmentStatements(
Environment pkgEnv, BuildFileAST ast, EventHandler eventHandler) {
for (Statement stmt : ast.getStatements()) {
if (stmt instanceof AssignmentStatement) {
Expression lvalue = ((AssignmentStatement) stmt).getLValue().getExpression();
if (!(lvalue instanceof Identifier)) {
continue;
}
String target = ((Identifier) lvalue).getName();
if (pkgEnv.hasVariable(target)) {
eventHandler.handle(Event.error(stmt.getLocation(), "Reassignment of builtin build "
+ "function '" + target + "' not permitted"));
return false;
}
}
}
return true;
}
// Reports an error and returns false iff package identifier was illegal.
private static boolean validatePackageIdentifier(PackageIdentifier packageId, Location location,
EventHandler eventHandler) {
String error = LabelValidator.validatePackageName(packageId.getPackageFragment().toString());
if (error != null) {
eventHandler.handle(Event.error(location, error));
return false; // Invalid package name 'foo'
}
return true;
}
static {
SkylarkSignatureProcessor.configureSkylarkFunctions(PackageFactory.class);
}
/** Empty EnvironmentExtension */
public static class EmptyEnvironmentExtension implements EnvironmentExtension {
@Override
public void update(Environment environment) {}
@Override
public void updateWorkspace(Environment environment) {}
@Override
public Iterable<PackageArgument<?>> getPackageArguments() {
return ImmutableList.of();
}
@Override
public ImmutableList<BaseFunction> nativeModuleFunctions() {
return ImmutableList.<BaseFunction>of();
}
}
}