// Copyright 2017 JanusGraph Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package org.janusgraph.graphdb.olap;
import com.google.common.base.Preconditions;
import org.janusgraph.core.JanusGraphFactory;
import org.janusgraph.core.JanusGraph;
import org.janusgraph.core.JanusGraphVertex;
import org.janusgraph.diskstorage.EntryList;
import org.janusgraph.diskstorage.StaticBuffer;
import org.janusgraph.diskstorage.configuration.BasicConfiguration;
import org.janusgraph.diskstorage.configuration.Configuration;
import org.janusgraph.diskstorage.keycolumnvalue.SliceQuery;
import org.janusgraph.diskstorage.keycolumnvalue.scan.ScanJob;
import org.janusgraph.diskstorage.keycolumnvalue.scan.ScanMetrics;
import org.janusgraph.diskstorage.util.BufferUtil;
import org.janusgraph.graphdb.database.StandardJanusGraph;
import org.janusgraph.graphdb.idmanagement.IDManager;
import org.janusgraph.graphdb.query.Query;
import org.janusgraph.graphdb.relations.RelationCache;
import org.janusgraph.graphdb.transaction.StandardJanusGraphTx;
import org.janusgraph.graphdb.transaction.StandardTransactionBuilder;
import org.janusgraph.graphdb.types.system.BaseKey;
import org.janusgraph.graphdb.vertices.PreloadedVertex;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.function.Predicate;
/**
* @author Matthias Broecheler (me@matthiasb.com)
*/
public class VertexJobConverter implements ScanJob {
protected static final SliceQuery VERTEX_EXISTS_QUERY = new SliceQuery(BufferUtil.zeroBuffer(1),BufferUtil.oneBuffer(4)).setLimit(1);
public static final String GHOST_VERTEX_COUNT = "ghost-vertices";
/**
* Number of result sets that got (possibly) truncated due to an applied query limit
*/
public static final String TRUNCATED_ENTRY_LISTS = "truncated-results";
protected final GraphProvider graph;
protected final VertexScanJob job;
protected StandardJanusGraphTx tx;
private IDManager idManager;
protected VertexJobConverter(JanusGraph graph, VertexScanJob job) {
Preconditions.checkArgument(job!=null);
this.graph = new GraphProvider();
if (graph!=null) this.graph.setGraph(graph);
this.job = job;
}
protected VertexJobConverter(VertexJobConverter copy) {
this.graph = copy.graph;
this.job = copy.job.clone();
this.tx = copy.tx;
this.idManager = copy.idManager;
}
public static ScanJob convert(JanusGraph graph, VertexScanJob vertexJob) {
return new VertexJobConverter(graph,vertexJob);
}
public static ScanJob convert(VertexScanJob vertexJob) {
return new VertexJobConverter(null,vertexJob);
}
public static StandardJanusGraphTx startTransaction(StandardJanusGraph graph) {
StandardTransactionBuilder txb = graph.buildTransaction().readOnly();
txb.setPreloadedData(true);
txb.checkInternalVertexExistence(false);
txb.dirtyVertexSize(0);
txb.vertexCacheSize(0);
return (StandardJanusGraphTx)txb.start();
}
@Override
public void workerIterationStart(Configuration jobConfig, Configuration graphConfig, ScanMetrics metrics) {
try {
open(graphConfig);
job.workerIterationStart(graph.get(), jobConfig, metrics);
} catch (Throwable e) {
close();
throw e;
}
}
protected void open(Configuration graphConfig) {
graph.initializeGraph(graphConfig);
idManager = graph.get().getIDManager();
tx = startTransaction(graph.get());
}
protected void close() {
if (null != tx && tx.isOpen())
tx.rollback();
graph.close();
}
@Override
public void workerIterationEnd(ScanMetrics metrics) {
job.workerIterationEnd(metrics);
close();
}
@Override
public void process(StaticBuffer key, Map<SliceQuery, EntryList> entries, ScanMetrics metrics) {
long vertexId = getVertexId(key);
assert entries.get(VERTEX_EXISTS_QUERY)!=null;
if (isGhostVertex(vertexId, entries.get(VERTEX_EXISTS_QUERY))) {
metrics.incrementCustom(GHOST_VERTEX_COUNT);
return;
}
JanusGraphVertex vertex = tx.getInternalVertex(vertexId);
Preconditions.checkArgument(vertex instanceof PreloadedVertex,
"The bounding transaction is not configured correctly");
PreloadedVertex v = (PreloadedVertex)vertex;
v.setAccessCheck(PreloadedVertex.OPENSTAR_CHECK);
for (Map.Entry<SliceQuery,EntryList> entry : entries.entrySet()) {
SliceQuery sq = entry.getKey();
if (sq.equals(VERTEX_EXISTS_QUERY)) continue;
EntryList entryList = entry.getValue();
if (entryList.size()>=sq.getLimit()) metrics.incrementCustom(TRUNCATED_ENTRY_LISTS);
v.addToQueryCache(sq.updateLimit(Query.NO_LIMIT),entryList);
}
job.process(v, metrics);
}
protected boolean isGhostVertex(long vertexId, EntryList firstEntries) {
if (idManager.isPartitionedVertex(vertexId) && !idManager.isCanonicalVertexId(vertexId)) return false;
RelationCache relCache = tx.getEdgeSerializer().parseRelation(
firstEntries.get(0),true,tx);
return relCache.typeId != BaseKey.VertexExists.longId();
}
@Override
public List<SliceQuery> getQueries() {
try {
QueryContainer qc = new QueryContainer(tx);
job.getQueries(qc);
List<SliceQuery> slices = new ArrayList<>();
slices.add(VERTEX_EXISTS_QUERY);
slices.addAll(qc.getSliceQueries());
return slices;
} catch (Throwable e) {
close();
throw e;
}
}
@Override
public Predicate<StaticBuffer> getKeyFilter() {
return buffer -> {
long vertexId = getVertexId(buffer);
if (IDManager.VertexIDType.Invisible.is(vertexId)) return false;
else return true;
};
}
@Override
public VertexJobConverter clone() {
return new VertexJobConverter(this);
}
protected long getVertexId(StaticBuffer key) {
return idManager.getKeyID(key);
}
public static class GraphProvider {
private StandardJanusGraph graph=null;
private boolean provided=false;
public void setGraph(JanusGraph graph) {
Preconditions.checkArgument(graph!=null && graph.isOpen(),"Need to provide open graph");
this.graph = (StandardJanusGraph)graph;
provided = true;
}
public void initializeGraph(Configuration config) {
if (!provided) {
this.graph = (StandardJanusGraph) JanusGraphFactory.open((BasicConfiguration) config);
}
}
public void close() {
if (!provided && null != graph && graph.isOpen()) {
graph.close();
graph=null;
}
}
public boolean isProvided() {
return provided;
}
public final StandardJanusGraph get() {
Preconditions.checkState(graph!=null);
return graph;
}
}
}