// Copyright 2017 JanusGraph Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.janusgraph.graphdb.olap.computer;
import com.google.common.collect.ImmutableMap;
import org.janusgraph.core.JanusGraph;
import org.janusgraph.core.JanusGraphVertex;
import org.janusgraph.diskstorage.EntryList;
import org.janusgraph.diskstorage.configuration.Configuration;
import org.janusgraph.diskstorage.keycolumnvalue.SliceQuery;
import org.janusgraph.diskstorage.keycolumnvalue.scan.ScanMetrics;
import org.janusgraph.graphdb.database.StandardJanusGraph;
import org.janusgraph.graphdb.idmanagement.IDManager;
import org.janusgraph.graphdb.olap.QueryContainer;
import org.janusgraph.graphdb.olap.VertexJobConverter;
import org.janusgraph.graphdb.olap.VertexScanJob;
import org.janusgraph.graphdb.vertices.PreloadedVertex;
import org.janusgraph.util.datastructures.Retriever;
import org.apache.tinkerpop.gremlin.process.computer.GraphComputer;
import org.apache.tinkerpop.gremlin.process.computer.MapReduce;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.Closeable;
import java.util.List;
import java.util.Map;
/**
* @author Matthias Broecheler (me@matthiasb.com)
*/
public class VertexMapJob implements VertexScanJob {
private static final Logger log =
LoggerFactory.getLogger(VertexMapJob.class);
public static final PreloadedVertex.AccessCheck MAPREDUCE_CHECK = new PreloadedVertex.AccessCheck() {
@Override
public final void accessEdges() {
throw GraphComputer.Exceptions.incidentAndAdjacentElementsCanNotBeAccessedInMapReduce();
}
@Override
public final void accessProperties() {
return; //Allowed
}
@Override
public void accessSetProperty() {
throw GraphComputer.Exceptions.vertexPropertiesCanNotBeUpdatedInMapReduce();
}
@Override
public Retriever<SliceQuery, EntryList> retrieveSliceQuery() {
return PreloadedVertex.EMPTY_RETRIEVER;
}
};
private final IDManager idManager;
private final Map<MapReduce, FulgoraMapEmitter> mapJobs;
private final FulgoraVertexMemory vertexMemory;
public static final String MAP_JOB_SUCCESS = "map-success";
public static final String MAP_JOB_FAILURE = "map-fail";
private VertexMapJob(IDManager idManager, FulgoraVertexMemory vertexMemory,
Map<MapReduce, FulgoraMapEmitter> mapJobs) {
this.mapJobs = mapJobs;
this.vertexMemory = vertexMemory;
this.idManager = idManager;
}
@Override
public VertexMapJob clone() {
ImmutableMap.Builder<MapReduce, FulgoraMapEmitter> cloneMap = ImmutableMap.builder();
for (Map.Entry<MapReduce, FulgoraMapEmitter> entry : mapJobs.entrySet()) {
cloneMap.put(entry.getKey().clone(), entry.getValue());
}
return new VertexMapJob(idManager, vertexMemory, cloneMap.build());
}
@Override
public void workerIterationStart(JanusGraph graph, Configuration config, ScanMetrics metrics) {
for (Map.Entry<MapReduce, FulgoraMapEmitter> mapJob : mapJobs.entrySet()) {
mapJob.getKey().workerStart(MapReduce.Stage.MAP);
}
}
@Override
public void workerIterationEnd(ScanMetrics metrics) {
for (Map.Entry<MapReduce, FulgoraMapEmitter> mapJob : mapJobs.entrySet()) {
mapJob.getKey().workerEnd(MapReduce.Stage.MAP);
}
}
@Override
public void process(JanusGraphVertex vertex, ScanMetrics metrics) {
PreloadedVertex v = (PreloadedVertex) vertex;
if (vertexMemory != null) {
VertexMemoryHandler vh = new VertexMemoryHandler(vertexMemory, v);
v.setPropertyMixing(vh);
}
v.setAccessCheck(MAPREDUCE_CHECK);
if (idManager.isPartitionedVertex(v.longId()) && !idManager.isCanonicalVertexId(v.longId())) {
return; //Only consider the canonical partition vertex representative
} else {
for (Map.Entry<MapReduce, FulgoraMapEmitter> mapJob : mapJobs.entrySet()) {
MapReduce job = mapJob.getKey();
try {
job.map(v, mapJob.getValue());
metrics.incrementCustom(MAP_JOB_SUCCESS);
} catch (Throwable ex) {
log.error("Encountered exception executing map job [" + job + "] on vertex [" + vertex + "]:", ex);
metrics.incrementCustom(MAP_JOB_FAILURE);
}
}
}
}
@Override
public void getQueries(QueryContainer queries) {
}
public static Executor getVertexMapJob(StandardJanusGraph graph, FulgoraVertexMemory vertexMemory,
Map<MapReduce, FulgoraMapEmitter> mapJobs) {
VertexMapJob job = new VertexMapJob(graph.getIDManager(), vertexMemory, mapJobs);
for (Map.Entry<MapReduce, FulgoraMapEmitter> mapJob : mapJobs.entrySet()) {
mapJob.getKey().workerStart(MapReduce.Stage.MAP);
}
return new Executor(graph, job);
}
public static class Executor extends VertexJobConverter implements Closeable {
private Executor(JanusGraph graph, VertexMapJob job) {
super(graph, job);
open(this.graph.get().getConfiguration().getConfiguration());
}
private Executor(final Executor copy) {
super(copy);
open(this.graph.get().getConfiguration().getConfiguration());
}
@Override
public List<SliceQuery> getQueries() {
List<SliceQuery> queries = super.getQueries();
queries.add(VertexProgramScanJob.SYSTEM_PROPS_QUERY);
return queries;
}
@Override
public void workerIterationStart(Configuration jobConfig, Configuration graphConfig, ScanMetrics metrics) {
job.workerIterationStart(graph.get(), jobConfig, metrics);
}
@Override
public void workerIterationEnd(ScanMetrics metrics) {
job.workerIterationEnd(metrics);
}
@Override
public Executor clone() {
return new Executor(this);
}
@Override
public void close() {
super.close();
}
}
}