// Copyright 2017 JanusGraph Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package org.janusgraph.graphdb.olap.job; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import org.janusgraph.core.JanusGraph; import org.janusgraph.core.JanusGraphRelation; import org.janusgraph.core.JanusGraphVertex; import org.janusgraph.diskstorage.EntryList; import org.janusgraph.diskstorage.StaticBuffer; import org.janusgraph.diskstorage.configuration.Configuration; import org.janusgraph.diskstorage.keycolumnvalue.SliceQuery; import org.janusgraph.diskstorage.keycolumnvalue.scan.ScanMetrics; import org.janusgraph.diskstorage.util.BufferUtil; import org.janusgraph.graphdb.configuration.GraphDatabaseConfiguration; import org.janusgraph.graphdb.olap.QueryContainer; import org.janusgraph.graphdb.olap.VertexJobConverter; import org.janusgraph.graphdb.olap.VertexScanJob; import org.janusgraph.graphdb.transaction.StandardJanusGraphTx; import org.janusgraph.graphdb.transaction.StandardTransactionBuilder; import org.janusgraph.graphdb.vertices.CacheVertex; import org.janusgraph.util.datastructures.Retriever; import java.time.Instant; import java.util.Iterator; import java.util.List; import java.util.Map; /** * @author Matthias Broecheler (me@matthiasb.com) */ public class GhostVertexRemover extends VertexJobConverter { private static final int RELATION_COUNT_LIMIT = 10000; private static final SliceQuery EVERYTHING_QUERY = new SliceQuery(BufferUtil.zeroBuffer(1),BufferUtil.oneBuffer(4)); public static final String REMOVED_RELATION_COUNT = "removed-relations"; public static final String REMOVED_VERTEX_COUNT = "removed-vertices"; public static final String SKIPPED_GHOST_LIMIT_COUNT = "skipped-ghosts"; private final SliceQuery everythingQueryLimit = EVERYTHING_QUERY.updateLimit(RELATION_COUNT_LIMIT); private Instant jobStartTime; public GhostVertexRemover(JanusGraph graph) { super(graph, new NoOpJob()); } public GhostVertexRemover() { this((JanusGraph)null); } protected GhostVertexRemover(GhostVertexRemover copy) { super(copy); } @Override public GhostVertexRemover clone() { return new GhostVertexRemover(this); } @Override public void workerIterationStart(Configuration jobConfig, Configuration graphConfig, ScanMetrics metrics) { super.workerIterationStart(jobConfig, graphConfig, metrics); Preconditions.checkArgument(jobConfig.has(GraphDatabaseConfiguration.JOB_START_TIME),"Invalid configuration for this job. Start time is required."); this.jobStartTime = Instant.ofEpochMilli(jobConfig.get(GraphDatabaseConfiguration.JOB_START_TIME)); assert tx!=null && tx.isOpen(); tx.rollback(); StandardTransactionBuilder txb = graph.get().buildTransaction(); txb.commitTime(jobStartTime); txb.checkExternalVertexExistence(false); txb.checkInternalVertexExistence(false); tx = (StandardJanusGraphTx)txb.start(); } @Override public void process(StaticBuffer key, Map<SliceQuery, EntryList> entries, ScanMetrics metrics) { long vertexId = getVertexId(key); assert entries.size()==1; assert entries.get(everythingQueryLimit)!=null; final EntryList everything = entries.get(everythingQueryLimit); if (!isGhostVertex(vertexId, everything)) { return; } if (everything.size()>=RELATION_COUNT_LIMIT) { metrics.incrementCustom(SKIPPED_GHOST_LIMIT_COUNT); return; } JanusGraphVertex vertex = tx.getInternalVertex(vertexId); Preconditions.checkArgument(vertex instanceof CacheVertex, "The bounding transaction is not configured correctly"); CacheVertex v = (CacheVertex)vertex; v.loadRelations(EVERYTHING_QUERY,new Retriever<SliceQuery, EntryList>() { @Override public EntryList get(SliceQuery input) { return everything; } }); int removedRelations = 0; Iterator<JanusGraphRelation> iter = v.query().noPartitionRestriction().relations().iterator(); while (iter.hasNext()) { iter.next(); iter.remove(); removedRelations++; } //There should be no more system relations to remove metrics.incrementCustom(REMOVED_VERTEX_COUNT); metrics.incrementCustom(REMOVED_RELATION_COUNT,removedRelations); } @Override public List<SliceQuery> getQueries() { return ImmutableList.of(everythingQueryLimit); } private static class NoOpJob implements VertexScanJob { @Override public void process(JanusGraphVertex vertex, ScanMetrics metrics) { throw new UnsupportedOperationException(); } @Override public void getQueries(QueryContainer queries) { throw new UnsupportedOperationException(); } @Override public NoOpJob clone() { return this; } } }