// Copyright 2014 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.skyframe;
import com.google.common.base.Function;
import com.google.common.base.Joiner;
import com.google.common.base.Predicate;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Iterables;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.google.devtools.build.lib.events.Event;
import com.google.devtools.build.lib.events.ExtendedEventHandler;
import com.google.devtools.build.lib.util.Preconditions;
import com.google.devtools.build.skyframe.Differencer.Diff;
import com.google.devtools.build.skyframe.InvalidatingNodeVisitor.DeletingInvalidationState;
import com.google.devtools.build.skyframe.InvalidatingNodeVisitor.DirtyingInvalidationState;
import com.google.devtools.build.skyframe.InvalidatingNodeVisitor.InvalidationState;
import com.google.devtools.build.skyframe.QueryableGraph.Reason;
import java.io.PrintStream;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashSet;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
import javax.annotation.Nullable;
/**
* An inmemory implementation that uses the eager invalidation strategy. This class is, by itself,
* not thread-safe. Neither is it thread-safe to use this class in parallel with any of the
* returned graphs. However, it is allowed to access the graph from multiple threads as long as
* that does not happen in parallel with an {@link #evaluate} call.
*
* <p>This memoizing evaluator requires a sequential versioning scheme. Evaluations
* must pass in a monotonically increasing {@link IntVersion}.
*/
public final class InMemoryMemoizingEvaluator implements MemoizingEvaluator {
private final ImmutableMap<SkyFunctionName, ? extends SkyFunction> skyFunctions;
private final DirtyTrackingProgressReceiver progressReceiver;
// Not final only for testing.
private InMemoryGraph graph;
private IntVersion lastGraphVersion = null;
// State related to invalidation and deletion.
private Set<SkyKey> valuesToDelete = new LinkedHashSet<>();
private Set<SkyKey> valuesToDirty = new LinkedHashSet<>();
private Map<SkyKey, SkyValue> valuesToInject = new HashMap<>();
private final InvalidationState deleterState = new DeletingInvalidationState();
private final Differencer differencer;
// Keep edges in graph. Can be false to save memory, in which case incremental builds are
// not possible.
private final boolean keepEdges;
// Values that the caller explicitly specified are assumed to be changed -- they will be
// re-evaluated even if none of their children are changed.
private final InvalidationState invalidatorState = new DirtyingInvalidationState();
private final EmittedEventState emittedEventState;
private final AtomicBoolean evaluating = new AtomicBoolean(false);
public InMemoryMemoizingEvaluator(
Map<SkyFunctionName, ? extends SkyFunction> skyFunctions, Differencer differencer) {
this(skyFunctions, differencer, null);
}
public InMemoryMemoizingEvaluator(
Map<SkyFunctionName, ? extends SkyFunction> skyFunctions,
Differencer differencer,
@Nullable EvaluationProgressReceiver progressReceiver) {
this(skyFunctions, differencer, progressReceiver, new EmittedEventState(), true);
}
public InMemoryMemoizingEvaluator(
Map<SkyFunctionName, ? extends SkyFunction> skyFunctions,
Differencer differencer,
@Nullable EvaluationProgressReceiver progressReceiver,
EmittedEventState emittedEventState,
boolean keepEdges) {
this.skyFunctions = ImmutableMap.copyOf(skyFunctions);
this.differencer = Preconditions.checkNotNull(differencer);
this.progressReceiver = new DirtyTrackingProgressReceiver(progressReceiver);
this.graph = new InMemoryGraphImpl(keepEdges);
this.emittedEventState = emittedEventState;
this.keepEdges = keepEdges;
}
private void invalidate(Iterable<SkyKey> diff) {
Iterables.addAll(valuesToDirty, diff);
}
@Override
public void delete(final Predicate<SkyKey> deletePredicate) {
valuesToDelete.addAll(
Maps.filterEntries(
graph.getAllValues(),
new Predicate<Entry<SkyKey, ? extends NodeEntry>>() {
@Override
public boolean apply(Entry<SkyKey, ? extends NodeEntry> input) {
Preconditions.checkNotNull(input.getKey(), "Null SkyKey in entry: %s", input);
return input.getValue().isDirty() || deletePredicate.apply(input.getKey());
}
})
.keySet());
}
@Override
public void deleteDirty(long versionAgeLimit) {
Preconditions.checkArgument(versionAgeLimit >= 0);
final Version threshold = IntVersion.of(lastGraphVersion.getVal() - versionAgeLimit);
valuesToDelete.addAll(
Sets.filter(progressReceiver.getUnenqueuedDirtyKeys(), new Predicate<SkyKey>() {
@Override
public boolean apply(SkyKey skyKey) {
NodeEntry entry = graph.get(null, Reason.OTHER, skyKey);
Preconditions.checkNotNull(entry, skyKey);
Preconditions.checkState(entry.isDirty(), skyKey);
return entry.getVersion().atMost(threshold);
}
}));
}
@Override
public <T extends SkyValue> EvaluationResult<T> evaluate(
Iterable<SkyKey> roots,
Version version,
boolean keepGoing,
int numThreads,
ExtendedEventHandler eventHandler)
throws InterruptedException {
// NOTE: Performance critical code. See bug "Null build performance parity".
IntVersion intVersion = (IntVersion) version;
Preconditions.checkState((lastGraphVersion == null && intVersion.getVal() == 0)
|| version.equals(lastGraphVersion.next()),
"InMemoryGraph supports only monotonically increasing Integer versions: %s %s",
lastGraphVersion, version);
setAndCheckEvaluateState(true, roots);
try {
// Mark for removal any inflight nodes from the previous evaluation.
valuesToDelete.addAll(progressReceiver.getAndClearInflightKeys());
// The RecordingDifferencer implementation is not quite working as it should be at this point.
// It clears the internal data structures after getDiff is called and will not return
// diffs for historical versions. This makes the following code sensitive to interrupts.
// Ideally we would simply not update lastGraphVersion if an interrupt occurs.
Diff diff = differencer.getDiff(new DelegatingWalkableGraph(graph), lastGraphVersion,
version);
valuesToInject.putAll(diff.changedKeysWithNewValues());
invalidate(diff.changedKeysWithoutNewValues());
pruneInjectedValues(valuesToInject);
invalidate(valuesToInject.keySet());
performInvalidation();
injectValues(intVersion);
ParallelEvaluator evaluator =
new ParallelEvaluator(
graph,
intVersion,
skyFunctions,
eventHandler,
emittedEventState,
DEFAULT_STORED_EVENT_FILTER,
keepGoing,
numThreads,
progressReceiver);
EvaluationResult<T> result = evaluator.eval(roots);
return EvaluationResult.<T>builder()
.mergeFrom(result)
.setWalkableGraph(new DelegatingWalkableGraph(graph))
.build();
} finally {
lastGraphVersion = intVersion;
setAndCheckEvaluateState(false, roots);
}
}
/**
* Removes entries in {@code valuesToInject} whose values are equal to the present values in the
* graph.
*/
private void pruneInjectedValues(Map<SkyKey, SkyValue> valuesToInject) {
for (Iterator<Entry<SkyKey, SkyValue>> it = valuesToInject.entrySet().iterator();
it.hasNext();) {
Entry<SkyKey, SkyValue> entry = it.next();
SkyKey key = entry.getKey();
SkyValue newValue = entry.getValue();
NodeEntry prevEntry = graph.get(null, Reason.OTHER, key);
if (prevEntry != null && prevEntry.isDone()) {
try {
Iterable<SkyKey> directDeps = prevEntry.getDirectDeps();
Preconditions.checkState(
Iterables.isEmpty(directDeps), "existing entry for %s has deps: %s", key, directDeps);
if (newValue.equals(prevEntry.getValue())
&& !valuesToDirty.contains(key)
&& !valuesToDelete.contains(key)) {
it.remove();
}
} catch (InterruptedException e) {
throw new IllegalStateException(
"InMemoryGraph does not throw: " + entry + ", " + prevEntry, e);
}
}
}
}
/**
* Injects values in {@code valuesToInject} into the graph.
*/
private void injectValues(IntVersion version) {
if (valuesToInject.isEmpty()) {
return;
}
try {
ParallelEvaluator.injectValues(valuesToInject, version, graph, progressReceiver);
} catch (InterruptedException e) {
throw new IllegalStateException("InMemoryGraph doesn't throw interrupts", e);
}
// Start with a new map to avoid bloat since clear() does not downsize the map.
valuesToInject = new HashMap<>();
}
private void performInvalidation() throws InterruptedException {
EagerInvalidator.delete(graph, valuesToDelete, progressReceiver, deleterState, keepEdges);
// Note that clearing the valuesToDelete would not do an internal resizing. Therefore, if any
// build has a large set of dirty values, subsequent operations (even clearing) will be slower.
// Instead, just start afresh with a new LinkedHashSet.
valuesToDelete = new LinkedHashSet<>();
EagerInvalidator.invalidate(graph, valuesToDirty, progressReceiver, invalidatorState);
// Ditto.
valuesToDirty = new LinkedHashSet<>();
}
private void setAndCheckEvaluateState(boolean newValue, Object requestInfo) {
Preconditions.checkState(evaluating.getAndSet(newValue) != newValue,
"Re-entrant evaluation for request: %s", requestInfo);
}
@Override
public Map<SkyKey, SkyValue> getValues() {
return graph.getValues();
}
@Override
public Map<SkyKey, ? extends NodeEntry> getGraphMap() {
return graph.getAllValuesMutable();
}
@Override
public Map<SkyKey, SkyValue> getDoneValues() {
return graph.getDoneValues();
}
private static boolean isDone(@Nullable NodeEntry entry) {
return entry != null && entry.isDone();
}
@Override
@Nullable public SkyValue getExistingValueForTesting(SkyKey key) {
NodeEntry entry = getExistingEntryForTesting(key);
try {
return isDone(entry) ? entry.getValue() : null;
} catch (InterruptedException e) {
throw new IllegalStateException("InMemoryGraph does not throw" + key + ", " + entry, e);
}
}
@Override
@Nullable public ErrorInfo getExistingErrorForTesting(SkyKey key) {
NodeEntry entry = getExistingEntryForTesting(key);
try {
return isDone(entry) ? entry.getErrorInfo() : null;
} catch (InterruptedException e) {
throw new IllegalStateException("InMemoryGraph does not throw" + key + ", " + entry, e);
}
}
@Nullable
@Override
public NodeEntry getExistingEntryForTesting(SkyKey key) {
return graph.get(null, Reason.OTHER, key);
}
@Override
public void injectGraphTransformerForTesting(GraphTransformerForTesting transformer) {
this.graph = transformer.transform(this.graph);
}
public ProcessableGraph getGraphForTesting() {
return graph;
}
@Override
public void dump(boolean summarize, PrintStream out) {
if (summarize) {
long nodes = 0;
long edges = 0;
for (NodeEntry entry : graph.getAllValues().values()) {
nodes++;
if (entry.isDone()) {
try {
edges += Iterables.size(entry.getDirectDeps());
} catch (InterruptedException e) {
throw new IllegalStateException("InMemoryGraph doesn't throw: " + entry, e);
}
}
}
out.println("Node count: " + nodes);
out.println("Edge count: " + edges);
} else {
Function<SkyKey, String> keyFormatter =
new Function<SkyKey, String>() {
@Override
public String apply(SkyKey key) {
return String.format("%s:%s",
key.functionName(), key.argument().toString().replace('\n', '_'));
}
};
for (Entry<SkyKey, ? extends NodeEntry> mapPair : graph.getAllValues().entrySet()) {
SkyKey key = mapPair.getKey();
NodeEntry entry = mapPair.getValue();
if (entry.isDone()) {
out.print(keyFormatter.apply(key));
out.print("|");
try {
out.println(
Joiner.on('|').join(Iterables.transform(entry.getDirectDeps(), keyFormatter)));
} catch (InterruptedException e) {
throw new IllegalStateException("InMemoryGraph doesn't throw: " + entry, e);
}
}
}
}
}
public ImmutableMap<SkyFunctionName, ? extends SkyFunction> getSkyFunctionsForTesting() {
return skyFunctions;
}
public static final EventFilter DEFAULT_STORED_EVENT_FILTER =
new EventFilter() {
@Override
public boolean apply(Event event) {
switch (event.getKind()) {
case INFO:
throw new UnsupportedOperationException(
"SkyFunctions should not display INFO messages: "
+ event.getLocation()
+ ": "
+ event.getMessage());
case PROGRESS:
return false;
default:
return true;
}
}
@Override
public boolean storeEvents() {
return true;
}
};
public static final EvaluatorSupplier SUPPLIER =
new EvaluatorSupplier() {
@Override
public MemoizingEvaluator create(
ImmutableMap<SkyFunctionName, ? extends SkyFunction> skyFunctions,
Differencer differencer,
@Nullable EvaluationProgressReceiver progressReceiver,
EmittedEventState emittedEventState,
boolean keepEdges) {
return new InMemoryMemoizingEvaluator(
skyFunctions, differencer, progressReceiver, emittedEventState, keepEdges);
}
};
}