// Copyright 2015 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.pkgcache;
import static com.google.common.truth.Truth.assertThat;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNotSame;
import static org.junit.Assert.assertSame;
import static org.junit.Assert.fail;
import com.google.common.base.Joiner;
import com.google.common.base.Predicates;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import com.google.common.eventbus.EventBus;
import com.google.devtools.build.lib.analysis.BlazeDirectories;
import com.google.devtools.build.lib.cmdline.Label;
import com.google.devtools.build.lib.events.Reporter;
import com.google.devtools.build.lib.packages.ConstantRuleVisibility;
import com.google.devtools.build.lib.packages.NoSuchPackageException;
import com.google.devtools.build.lib.packages.NoSuchTargetException;
import com.google.devtools.build.lib.packages.NoSuchThingException;
import com.google.devtools.build.lib.packages.Package;
import com.google.devtools.build.lib.packages.Rule;
import com.google.devtools.build.lib.packages.Target;
import com.google.devtools.build.lib.packages.util.LoadingMock;
import com.google.devtools.build.lib.skyframe.DiffAwareness;
import com.google.devtools.build.lib.skyframe.PackageLookupFunction.CrossRepositoryLabelViolationStrategy;
import com.google.devtools.build.lib.skyframe.PackageLookupValue.BuildFileName;
import com.google.devtools.build.lib.skyframe.PrecomputedValue;
import com.google.devtools.build.lib.skyframe.SequencedSkyframeExecutor;
import com.google.devtools.build.lib.skyframe.SkyValueDirtinessChecker;
import com.google.devtools.build.lib.skyframe.SkyframeExecutor;
import com.google.devtools.build.lib.syntax.GlobList;
import com.google.devtools.build.lib.syntax.SkylarkSemanticsOptions;
import com.google.devtools.build.lib.testutil.ManualClock;
import com.google.devtools.build.lib.util.BlazeClock;
import com.google.devtools.build.lib.util.Preconditions;
import com.google.devtools.build.lib.util.io.TimestampGranularityMonitor;
import com.google.devtools.build.lib.vfs.Dirent;
import com.google.devtools.build.lib.vfs.FileStatus;
import com.google.devtools.build.lib.vfs.FileSystem;
import com.google.devtools.build.lib.vfs.FileSystemUtils;
import com.google.devtools.build.lib.vfs.ModifiedFileSet;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
import com.google.devtools.build.lib.vfs.inmemoryfs.InMemoryFileSystem;
import com.google.devtools.build.skyframe.SkyFunction;
import com.google.devtools.build.skyframe.SkyFunctionName;
import com.google.devtools.common.options.Options;
import com.google.devtools.common.options.OptionsClassProvider;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.UUID;
import javax.annotation.Nullable;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.junit.runners.JUnit4;
/**
* Tests for incremental loading; these cover both normal operation and diff awareness, for which a
* list of modified / added / removed files is available.
*/
@RunWith(JUnit4.class)
public class IncrementalLoadingTest {
protected PackageCacheTester tester;
private Path throwOnReaddir = null;
private Path throwOnStat = null;
@Before
public final void createTester() throws Exception {
ManualClock clock = new ManualClock();
FileSystem fs =
new InMemoryFileSystem(clock) {
@Override
public Collection<Dirent> readdir(Path path, boolean followSymlinks) throws IOException {
if (path.equals(throwOnReaddir)) {
throw new FileNotFoundException(path.getPathString());
}
return super.readdir(path, followSymlinks);
}
@Nullable
@Override
public FileStatus stat(Path path, boolean followSymlinks) throws IOException {
if (path.equals(throwOnStat)) {
throw new IOException("bork " + path.getPathString());
}
return super.stat(path, followSymlinks);
}
};
tester = createTester(fs, clock);
}
protected PackageCacheTester createTester(FileSystem fs, ManualClock clock) throws Exception {
return new PackageCacheTester(fs, clock);
}
@Test
public void testNoChange() throws Exception {
tester.addFile("base/BUILD",
"filegroup(name = 'hello', srcs = ['foo.txt'])");
tester.sync();
Target oldTarget = tester.getTarget("//base:hello");
assertNotNull(oldTarget);
tester.sync();
Target newTarget = tester.getTarget("//base:hello");
assertSame(oldTarget, newTarget);
}
@Test
public void testModifyBuildFile() throws Exception {
tester.addFile("base/BUILD", "filegroup(name = 'hello', srcs = ['foo.txt'])");
tester.sync();
Target oldTarget = tester.getTarget("//base:hello");
tester.modifyFile("base/BUILD", "filegroup(name = 'hello', srcs = ['bar.txt'])");
tester.sync();
Target newTarget = tester.getTarget("//base:hello");
assertNotSame(oldTarget, newTarget);
}
@Test
public void testModifyNonBuildFile() throws Exception {
tester.addFile("base/BUILD", "filegroup(name = 'hello', srcs = ['foo.txt'])");
tester.addFile("base/foo.txt", "nothing");
tester.sync();
Target oldTarget = tester.getTarget("//base:hello");
tester.modifyFile("base/foo.txt", "other");
tester.sync();
Target newTarget = tester.getTarget("//base:hello");
assertSame(oldTarget, newTarget);
}
@Test
public void testRemoveNonBuildFile() throws Exception {
tester.addFile("base/BUILD", "filegroup(name = 'hello', srcs = ['foo.txt'])");
tester.addFile("base/foo.txt", "nothing");
tester.sync();
Target oldTarget = tester.getTarget("//base:hello");
tester.removeFile("base/foo.txt");
tester.sync();
Target newTarget = tester.getTarget("//base:hello");
assertSame(oldTarget, newTarget);
}
@Test
public void testModifySymlinkedFileSamePackage() throws Exception {
tester.addSymlink("base/BUILD", "mybuild");
tester.addFile("base/mybuild", "filegroup(name = 'hello', srcs = ['foo.txt'])");
tester.sync();
Target oldTarget = tester.getTarget("//base:hello");
tester.modifyFile("base/mybuild", "filegroup(name = 'hello', srcs = ['bar.txt'])");
tester.sync();
Target newTarget = tester.getTarget("//base:hello");
assertNotSame(oldTarget, newTarget);
}
@Test
public void testModifySymlinkedFileDifferentPackage() throws Exception {
tester.addSymlink("base/BUILD", "../other/BUILD");
tester.addFile("other/BUILD", "filegroup(name = 'hello', srcs = ['foo.txt'])");
tester.sync();
Target oldTarget = tester.getTarget("//base:hello");
tester.modifyFile("other/BUILD", "filegroup(name = 'hello', srcs = ['bar.txt'])");
tester.sync();
Target newTarget = tester.getTarget("//base:hello");
assertNotSame(oldTarget, newTarget);
}
@Test
public void testBUILDSymlinkModifiedThenChanges() throws Exception {
// We need to ensure that the timestamps of "one" and "two" are different, because Blaze
// currently does not recognize changes to symlinks if the timestamps of the old and the new
// file pointed to by the symlink are the same.
tester.addFile("one", "filegroup(name='a', srcs=['1'])");
tester.sync();
tester.addFile("two", "filegroup(name='a', srcs=['2'])");
tester.addSymlink("oldlink", "one");
tester.addSymlink("newlink", "one");
tester.addSymlink("a/BUILD", "../oldlink");
tester.sync();
Target a1 = tester.getTarget("//a:a");
tester.modifySymlink("a/BUILD", "../newlink");
tester.sync();
tester.getTarget("//a:a");
tester.modifySymlink("newlink", "two");
tester.sync();
Target a3 = tester.getTarget("//a:a");
assertNotSame(a1, a3);
}
@Test
public void testBUILDFileIsExternalSymlinkAndChanges() throws Exception {
tester.addFile("/nonroot/file", "filegroup(name='a', srcs=['file'])");
tester.addSymlink("a/BUILD", "/nonroot/file");
tester.sync();
Target a1 = tester.getTarget("//a:a");
tester.modifyFile("/nonroot/file", "filegroup(name='a', srcs=['file2'])");
tester.sync();
Target a2 = tester.getTarget("//a:a");
tester.sync();
assertNotSame(a1, a2);
}
@Test
public void testLabelWithTwoSegmentsAndTotalInvalidation() throws Exception {
tester.addFile("a/BUILD", "filegroup(name='fg', srcs=['b/c'])");
tester.addFile("a/b/BUILD");
tester.sync();
Target fg1 = tester.getTarget("//a:fg");
tester.everythingModified();
tester.sync();
Target fg2 = tester.getTarget("//a:fg");
assertSame(fg1, fg2);
}
@Test
public void testAddGlobFile() throws Exception {
tester.addFile("base/BUILD", "filegroup(name = 'hello', srcs = glob(['*.txt']))");
tester.addFile("base/foo.txt", "nothing");
tester.sync();
Target oldTarget = tester.getTarget("//base:hello");
tester.addFile("base/bar.txt", "also nothing");
tester.sync();
Target newTarget = tester.getTarget("//base:hello");
assertNotSame(oldTarget, newTarget);
}
@Test
public void testRemoveGlobFile() throws Exception {
tester.addFile("base/BUILD", "filegroup(name = 'hello', srcs = glob(['*.txt']))");
tester.addFile("base/foo.txt", "nothing");
tester.addFile("base/bar.txt", "also nothing");
tester.sync();
Target oldTarget = tester.getTarget("//base:hello");
tester.removeFile("base/bar.txt");
tester.sync();
Target newTarget = tester.getTarget("//base:hello");
assertNotSame(oldTarget, newTarget);
}
@Test
public void testPackageNotInLastBuildReplaced() throws Exception {
tester.addFile("a/BUILD", "filegroup(name='a', srcs=['bad.sh'])");
tester.sync();
Target a1 = tester.getTarget("//a:a");
tester.addFile("b/BUILD", "filegroup(name='b', srcs=['b.sh'])");
tester.modifyFile("a/BUILD", "filegroup(name='a', srcs=['good.sh'])");
tester.sync();
tester.getTarget("//b:b");
tester.sync();
Target a2 = tester.getTarget("//a:a");
assertNotSame(a1, a2);
}
@Test
public void testBrokenSymlinkAddedThenFixed() throws Exception {
tester.addFile("a/BUILD", "filegroup(name='a', srcs=glob(['**']))");
tester.sync();
Target a1 = tester.getTarget("//a:a");
tester.addSymlink("a/b", "../c");
tester.sync();
tester.getTarget("//a:a");
tester.addFile("c");
tester.sync();
Target a3 = tester.getTarget("//a:a");
assertNotSame(a1, a3);
}
@Test
public void testBuildFileWithSyntaxError() throws Exception {
tester.addFile("a/BUILD", "sh_library(xyz='a')");
tester.sync();
try {
tester.getTarget("//a:a");
fail();
} catch (NoSuchThingException e) {
// Expected
}
tester.modifyFile("a/BUILD", "sh_library(name='a')");
tester.sync();
tester.getTarget("//a:a");
}
@Test
public void testSymlinkedBuildFileWithSyntaxError() throws Exception {
tester.addFile("a/BUILD.real", "sh_library(xyz='a')");
tester.addSymlink("a/BUILD", "BUILD.real");
tester.sync();
try {
tester.getTarget("//a:a");
fail();
} catch (NoSuchThingException e) {
// Expected
}
tester.modifyFile("a/BUILD.real", "sh_library(name='a')");
tester.sync();
tester.getTarget("//a:a");
}
@Test
public void testTransientErrorsInGlobbing() throws Exception {
Path buildFile = tester.addFile("e/BUILD", "sh_library(name = 'e', data = glob(['*.txt']))");
Path parentDir = buildFile.getParentDirectory();
tester.addFile("e/data.txt");
throwOnReaddir = parentDir;
tester.sync();
Target target = tester.getTarget("//e:e");
assertThat(((Rule) target).containsErrors()).isTrue();
GlobList<?> globList = (GlobList<?>) ((Rule) target).getAttributeContainer().getAttr("data");
assertThat(globList).isEmpty();
throwOnReaddir = null;
tester.sync();
target = tester.getTarget("//e:e");
assertThat(((Rule) target).containsErrors()).isFalse();
globList = (GlobList<?>) ((Rule) target).getAttributeContainer().getAttr("data");
assertThat(globList).containsExactly(Label.parseAbsolute("//e:data.txt"));
}
@Test
public void testIrrelevantFileInSubdirDoesntReloadPackage() throws Exception {
tester.addFile("pkg/BUILD", "sh_library(name = 'pkg', srcs = glob(['**/*.sh']))");
tester.addFile("pkg/pkg.sh", "#!/bin/bash");
tester.addFile("pkg/bar/bar.sh", "#!/bin/bash");
Package pkg = tester.getTarget("//pkg:pkg").getPackage();
// Write file in directory to force reload of top-level glob.
tester.addFile("pkg/irrelevant_file");
tester.addFile("pkg/bar/irrelevant_file"); // Subglob is also reloaded.
assertSame(pkg, tester.getTarget("//pkg:pkg").getPackage());
}
@Test
public void testMissingPackages() throws Exception {
tester.sync();
try {
tester.getTarget("//a:a");
fail();
} catch (NoSuchThingException e) {
// expected
}
tester.addFile("a/BUILD", "sh_library(name='a')");
tester.sync();
tester.getTarget("//a:a");
}
@Test
public void testChangedExternalFile() throws Exception {
tester.addFile("a/BUILD",
"load('/a/b', 'b')",
"b()");
tester.addFile("/b.bzl",
"def b():",
" pass");
tester.addSymlink("a/b.bzl", "/b.bzl");
tester.sync();
tester.getTarget("//a:BUILD");
tester.modifyFile("/b.bzl", "ERROR ERROR");
tester.sync();
try {
tester.getTarget("//a:BUILD");
fail();
} catch (NoSuchThingException e) {
// expected
}
}
static class PackageCacheTester {
private class ManualDiffAwareness implements DiffAwareness {
private View lastView;
private View currentView;
@Override
public View getCurrentView(OptionsClassProvider options) {
lastView = currentView;
currentView = new View() {};
return currentView;
}
@Override
public ModifiedFileSet getDiff(View oldView, View newView) {
if (oldView == lastView && newView == currentView) {
return Preconditions.checkNotNull(modifiedFileSet);
} else {
return ModifiedFileSet.EVERYTHING_MODIFIED;
}
}
@Override
public String name() {
return "PackageCacheTester.DiffAwareness";
}
@Override
public void close() {
}
}
private class ManualDiffAwarenessFactory implements DiffAwareness.Factory {
@Nullable
@Override
public DiffAwareness maybeCreate(Path pathEntry) {
return pathEntry == workspace ? new ManualDiffAwareness() : null;
}
}
private final ManualClock clock;
private final Path workspace;
private final Path outputBase;
private final Reporter reporter = new Reporter(new EventBus());
private final SkyframeExecutor skyframeExecutor;
private final List<Path> changes = new ArrayList<>();
private boolean everythingModified = false;
private ModifiedFileSet modifiedFileSet;
public PackageCacheTester(FileSystem fs, ManualClock clock) throws IOException {
this.clock = clock;
workspace = fs.getPath("/workspace");
workspace.createDirectory();
outputBase = fs.getPath("/output_base");
outputBase.createDirectory();
addFile("WORKSPACE");
LoadingMock loadingMock = LoadingMock.get();
skyframeExecutor =
SequencedSkyframeExecutor.create(
loadingMock
.getPackageFactoryBuilderForTesting()
.build(loadingMock.createRuleClassProvider(), fs),
new BlazeDirectories(
fs.getPath("/install"),
fs.getPath("/output"),
workspace,
loadingMock.getProductName()),
null, /* BinTools */
null, /* workspaceStatusActionFactory */
loadingMock.createRuleClassProvider().getBuildInfoFactories(),
ImmutableList.of(new ManualDiffAwarenessFactory()),
Predicates.<PathFragment>alwaysFalse(),
ImmutableMap.<SkyFunctionName, SkyFunction>of(),
ImmutableList.<PrecomputedValue.Injected>of(),
ImmutableList.<SkyValueDirtinessChecker>of(),
loadingMock.getProductName(),
CrossRepositoryLabelViolationStrategy.ERROR,
ImmutableList.of(BuildFileName.BUILD_DOT_BAZEL, BuildFileName.BUILD));
PackageCacheOptions packageCacheOptions = Options.getDefaults(PackageCacheOptions.class);
packageCacheOptions.defaultVisibility = ConstantRuleVisibility.PUBLIC;
packageCacheOptions.showLoadingProgress = true;
packageCacheOptions.globbingThreads = 7;
skyframeExecutor.preparePackageLoading(
new PathPackageLocator(outputBase, ImmutableList.of(workspace)),
packageCacheOptions,
Options.getDefaults(SkylarkSemanticsOptions.class),
"",
UUID.randomUUID(),
ImmutableMap.<String, String>of(),
ImmutableMap.<String, String>of(),
new TimestampGranularityMonitor(BlazeClock.instance()));
}
Path addFile(String fileName, String... content) throws IOException {
Path buildFile = workspace.getRelative(fileName);
Preconditions.checkState(!buildFile.exists());
Path currentPath = buildFile;
// Add the new file and all the directories that will be created by
// createDirectoryAndParents()
while (!currentPath.exists()) {
changes.add(currentPath);
currentPath = currentPath.getParentDirectory();
}
FileSystemUtils.createDirectoryAndParents(buildFile.getParentDirectory());
FileSystemUtils.writeContentAsLatin1(buildFile, Joiner.on('\n').join(content));
return buildFile;
}
void addSymlink(String fileName, String target) throws IOException {
Path path = workspace.getRelative(fileName);
Preconditions.checkState(!path.exists());
FileSystemUtils.createDirectoryAndParents(path.getParentDirectory());
path.createSymbolicLink(PathFragment.create(target));
changes.add(path);
}
void removeFile(String fileName) throws IOException {
Path path = workspace.getRelative(fileName);
Preconditions.checkState(path.delete());
changes.add(path);
}
void modifyFile(String fileName, String... content) throws IOException {
Path path = workspace.getRelative(fileName);
Preconditions.checkState(path.exists());
Preconditions.checkState(path.delete());
FileSystemUtils.writeContentAsLatin1(path, Joiner.on('\n').join(content));
changes.add(path);
}
void modifySymlink(String fileName, String newTarget) throws IOException {
Path symlink = workspace.getRelative(fileName);
Preconditions.checkState(symlink.exists());
symlink.delete();
symlink.createSymbolicLink(PathFragment.create(newTarget));
changes.add(symlink);
}
void everythingModified() {
everythingModified = true;
}
private ModifiedFileSet getModifiedFileSet() {
if (everythingModified) {
everythingModified = false;
return ModifiedFileSet.EVERYTHING_MODIFIED;
}
ModifiedFileSet.Builder builder = ModifiedFileSet.builder();
for (Path path : changes) {
if (!path.startsWith(workspace)) {
continue;
}
PathFragment workspacePath = path.relativeTo(workspace);
builder.modify(workspacePath);
}
return builder.build();
}
void sync() throws InterruptedException {
clock.advanceMillis(1);
modifiedFileSet = getModifiedFileSet();
PackageCacheOptions packageCacheOptions = Options.getDefaults(PackageCacheOptions.class);
packageCacheOptions.defaultVisibility = ConstantRuleVisibility.PUBLIC;
packageCacheOptions.showLoadingProgress = true;
packageCacheOptions.globbingThreads = 7;
skyframeExecutor.preparePackageLoading(
new PathPackageLocator(outputBase, ImmutableList.of(workspace)),
packageCacheOptions,
Options.getDefaults(SkylarkSemanticsOptions.class),
"",
UUID.randomUUID(),
ImmutableMap.<String, String>of(),
ImmutableMap.<String, String>of(),
new TimestampGranularityMonitor(BlazeClock.instance()));
skyframeExecutor.invalidateFilesUnderPathForTesting(
new Reporter(new EventBus()), modifiedFileSet, workspace);
((SequencedSkyframeExecutor) skyframeExecutor).handleDiffs(new Reporter(new EventBus()));
changes.clear();
}
Target getTarget(String targetName)
throws NoSuchPackageException, NoSuchTargetException, InterruptedException {
Label label = Label.parseAbsoluteUnchecked(targetName);
return skyframeExecutor.getPackageManager().getTarget(reporter, label);
}
}
}