/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.atlas.repository.graph;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.BiMap;
import com.google.common.collect.HashBiMap;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasException;
import org.apache.atlas.RequestContext;
import org.apache.atlas.model.instance.AtlasEntity.Status;
import org.apache.atlas.repository.Constants;
import org.apache.atlas.repository.RepositoryException;
import org.apache.atlas.repository.graphdb.AtlasEdge;
import org.apache.atlas.repository.graphdb.AtlasEdgeDirection;
import org.apache.atlas.repository.graphdb.AtlasElement;
import org.apache.atlas.repository.graphdb.AtlasGraph;
import org.apache.atlas.repository.graphdb.AtlasGraphQuery;
import org.apache.atlas.repository.graphdb.AtlasVertex;
import org.apache.atlas.repository.store.graph.v1.AtlasGraphUtilsV1;
import org.apache.atlas.type.AtlasType;
import org.apache.atlas.typesystem.IReferenceableInstance;
import org.apache.atlas.typesystem.ITypedInstance;
import org.apache.atlas.typesystem.ITypedReferenceableInstance;
import org.apache.atlas.typesystem.Referenceable;
import org.apache.atlas.typesystem.exception.EntityNotFoundException;
import org.apache.atlas.typesystem.exception.TypeNotFoundException;
import org.apache.atlas.typesystem.json.InstanceSerialization;
import org.apache.atlas.typesystem.persistence.Id;
import org.apache.atlas.typesystem.persistence.ReferenceableInstance;
import org.apache.atlas.typesystem.types.AttributeInfo;
import org.apache.atlas.typesystem.types.ClassType;
import org.apache.atlas.typesystem.types.DataTypes;
import org.apache.atlas.typesystem.types.DataTypes.TypeCategory;
import org.apache.atlas.typesystem.types.HierarchicalType;
import org.apache.atlas.typesystem.types.IDataType;
import org.apache.atlas.typesystem.types.Multiplicity;
import org.apache.atlas.typesystem.types.TypeSystem;
import org.apache.atlas.typesystem.types.ValueConversionException;
import org.apache.atlas.typesystem.types.utils.TypesUtil;
import org.apache.atlas.util.AttributeValueMap;
import org.apache.atlas.util.IndexedInstance;
import org.apache.atlas.utils.ParamChecker;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.codehaus.jettison.json.JSONArray;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.Stack;
import java.util.UUID;
/**
* Utility class for graph operations.
*/
public final class GraphHelper {
private static final Logger LOG = LoggerFactory.getLogger(GraphHelper.class);
public static final String EDGE_LABEL_PREFIX = "__";
private static final TypeSystem typeSystem = TypeSystem.getInstance();
public static final String RETRY_COUNT = "atlas.graph.storage.num.retries";
public static final String RETRY_DELAY = "atlas.graph.storage.retry.sleeptime.ms";
private static volatile GraphHelper INSTANCE;
private AtlasGraph graph;
private static int maxRetries;
public static long retrySleepTimeMillis;
@VisibleForTesting
GraphHelper(AtlasGraph graph) {
this.graph = graph;
try {
maxRetries = ApplicationProperties.get().getInt(RETRY_COUNT, 3);
retrySleepTimeMillis = ApplicationProperties.get().getLong(RETRY_DELAY, 1000);
} catch (AtlasException e) {
LOG.error("Could not load configuration. Setting to default value for " + RETRY_COUNT, e);
}
}
public static GraphHelper getInstance() {
if ( INSTANCE == null) {
synchronized (GraphHelper.class) {
if (INSTANCE == null) {
INSTANCE = new GraphHelper(AtlasGraphProvider.getGraphInstance());
}
}
}
return INSTANCE;
}
@VisibleForTesting
static GraphHelper getInstance(AtlasGraph graph) {
if ( INSTANCE == null) {
synchronized (GraphHelper.class) {
if (INSTANCE == null) {
INSTANCE = new GraphHelper(graph);
}
}
}
return INSTANCE;
}
public AtlasVertex createVertexWithIdentity(ITypedReferenceableInstance typedInstance, Set<String> superTypeNames) {
final String guid = UUID.randomUUID().toString();
final AtlasVertex vertexWithIdentity = createVertexWithoutIdentity(typedInstance.getTypeName(),
new Id(guid, 0, typedInstance.getTypeName()), superTypeNames);
// add identity
setProperty(vertexWithIdentity, Constants.GUID_PROPERTY_KEY, guid);
// add version information
setProperty(vertexWithIdentity, Constants.VERSION_PROPERTY_KEY, typedInstance.getId().version);
return vertexWithIdentity;
}
public AtlasVertex createVertexWithoutIdentity(String typeName, Id typedInstanceId, Set<String> superTypeNames) {
if (LOG.isDebugEnabled()) {
LOG.debug("Creating AtlasVertex for type {} id {}", typeName,
typedInstanceId != null ? typedInstanceId._getId() : null);
}
final AtlasVertex vertexWithoutIdentity = graph.addVertex();
// add type information
setProperty(vertexWithoutIdentity, Constants.ENTITY_TYPE_PROPERTY_KEY, typeName);
// add super types
for (String superTypeName : superTypeNames) {
addProperty(vertexWithoutIdentity, Constants.SUPER_TYPES_PROPERTY_KEY, superTypeName);
}
// add state information
setProperty(vertexWithoutIdentity, Constants.STATE_PROPERTY_KEY, Id.EntityState.ACTIVE.name());
// add timestamp information
setProperty(vertexWithoutIdentity, Constants.TIMESTAMP_PROPERTY_KEY, RequestContext.get().getRequestTime());
setProperty(vertexWithoutIdentity, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY,
RequestContext.get().getRequestTime());
setProperty(vertexWithoutIdentity, Constants.CREATED_BY_KEY, RequestContext.get().getUser());
setProperty(vertexWithoutIdentity, Constants.MODIFIED_BY_KEY, RequestContext.get().getUser());
return vertexWithoutIdentity;
}
private AtlasEdge addEdge(AtlasVertex fromVertex, AtlasVertex toVertex, String edgeLabel) {
if (LOG.isDebugEnabled()) {
LOG.debug("Adding edge for {} -> label {} -> {}", string(fromVertex), edgeLabel, string(toVertex));
}
AtlasEdge edge = graph.addEdge(fromVertex, toVertex, edgeLabel);
setProperty(edge, Constants.STATE_PROPERTY_KEY, Id.EntityState.ACTIVE.name());
setProperty(edge, Constants.TIMESTAMP_PROPERTY_KEY, RequestContext.get().getRequestTime());
setProperty(edge, Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, RequestContext.get().getRequestTime());
setProperty(edge, Constants.CREATED_BY_KEY, RequestContext.get().getUser());
setProperty(edge, Constants.MODIFIED_BY_KEY, RequestContext.get().getUser());
if (LOG.isDebugEnabled()) {
LOG.debug("Added {}", string(edge));
}
return edge;
}
public AtlasEdge getOrCreateEdge(AtlasVertex outVertex, AtlasVertex inVertex, String edgeLabel) throws RepositoryException {
for (int numRetries = 0; numRetries < maxRetries; numRetries++) {
try {
if (LOG.isDebugEnabled()) {
LOG.debug("Running edge creation attempt {}", numRetries);
}
Iterator<AtlasEdge> edges = getAdjacentEdgesByLabel(inVertex, AtlasEdgeDirection.IN, edgeLabel);
while (edges.hasNext()) {
AtlasEdge edge = edges.next();
if (edge.getOutVertex().equals(outVertex)) {
Id.EntityState edgeState = getState(edge);
if (edgeState == null || edgeState == Id.EntityState.ACTIVE) {
return edge;
}
}
}
return addEdge(outVertex, inVertex, edgeLabel);
} catch (Exception e) {
LOG.warn(String.format("Exception while trying to create edge from %s to %s with label %s. Retrying",
vertexString(outVertex), vertexString(inVertex), edgeLabel), e);
if (numRetries == (maxRetries - 1)) {
LOG.error("Max retries exceeded for edge creation {} {} {} ", outVertex, inVertex, edgeLabel, e);
throw new RepositoryException("Edge creation failed after retries", e);
}
try {
LOG.info("Retrying with delay of {} ms ", retrySleepTimeMillis);
Thread.sleep(retrySleepTimeMillis);
} catch(InterruptedException ie) {
LOG.warn("Retry interrupted during edge creation ");
throw new RepositoryException("Retry interrupted during edge creation", ie);
}
}
}
return null;
}
public AtlasEdge getEdgeByEdgeId(AtlasVertex outVertex, String edgeLabel, String edgeId) {
if (edgeId == null) {
return null;
}
return graph.getEdge(edgeId);
//TODO get edge id is expensive. Use this logic. But doesn't work for now
/**
Iterable<AtlasEdge> edges = outVertex.getEdges(Direction.OUT, edgeLabel);
for (AtlasEdge edge : edges) {
if (edge.getObjectId().toString().equals(edgeId)) {
return edge;
}
}
return null;
**/
}
/**
* Args of the format prop1, key1, prop2, key2...
* Searches for a AtlasVertex with prop1=key1 && prop2=key2
* @param args
* @return AtlasVertex with the given property keys
* @throws EntityNotFoundException
*/
public AtlasVertex findVertex(Object... args) throws EntityNotFoundException {
AtlasGraphQuery query = graph.query();
for (int i = 0 ; i < args.length; i+=2) {
query = query.has((String) args[i], args[i+1]);
}
Iterator<AtlasVertex> results = query.vertices().iterator();
// returning one since entityType, qualifiedName should be unique
AtlasVertex vertex = results.hasNext() ? results.next() : null;
if (vertex == null) {
String conditionStr = getConditionString(args);
if (LOG.isDebugEnabled()) {
LOG.debug("Could not find a vertex with {}", conditionStr);
}
throw new EntityNotFoundException("Could not find an entity in the repository with " + conditionStr);
} else {
if (LOG.isDebugEnabled()) {
LOG.debug("Found a vertex {} with {}", string(vertex), getConditionString(args));
}
}
return vertex;
}
//In some cases of parallel APIs, the edge is added, but get edge by label doesn't return the edge. ATLAS-1104
//So traversing all the edges
public Iterator<AtlasEdge> getAdjacentEdgesByLabel(AtlasVertex instanceVertex, AtlasEdgeDirection direction, final String edgeLabel) {
if (LOG.isDebugEnabled()) {
LOG.debug("Finding edges for {} with label {}", string(instanceVertex), edgeLabel);
}
if(instanceVertex != null && edgeLabel != null) {
final Iterator<AtlasEdge> iterator = instanceVertex.getEdges(direction).iterator();
return new Iterator<AtlasEdge>() {
private AtlasEdge edge = null;
@Override
public boolean hasNext() {
while (edge == null && iterator.hasNext()) {
AtlasEdge localEdge = iterator.next();
if (localEdge.getLabel().equals(edgeLabel)) {
edge = localEdge;
}
}
return edge != null;
}
@Override
public AtlasEdge next() {
if (hasNext()) {
AtlasEdge localEdge = edge;
edge = null;
return localEdge;
}
return null;
}
@Override
public void remove() {
throw new IllegalStateException("Not handled");
}
};
}
return null;
}
public Iterator<AtlasEdge> getOutGoingEdgesByLabel(AtlasVertex instanceVertex, String edgeLabel) {
return getAdjacentEdgesByLabel(instanceVertex, AtlasEdgeDirection.OUT, edgeLabel);
}
/**
* Returns the active edge for the given edge label.
* If the vertex is deleted and there is no active edge, it returns the latest deleted edge
* @param vertex
* @param edgeLabel
* @return
*/
public AtlasEdge getEdgeForLabel(AtlasVertex vertex, String edgeLabel) {
Iterator<AtlasEdge> iterator = getAdjacentEdgesByLabel(vertex, AtlasEdgeDirection.OUT, edgeLabel);
AtlasEdge latestDeletedEdge = null;
long latestDeletedEdgeTime = Long.MIN_VALUE;
while (iterator != null && iterator.hasNext()) {
AtlasEdge edge = iterator.next();
Id.EntityState edgeState = getState(edge);
if (edgeState == null || edgeState == Id.EntityState.ACTIVE) {
if (LOG.isDebugEnabled()) {
LOG.debug("Found {}", string(edge));
}
return edge;
} else {
Long modificationTime = edge.getProperty(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, Long.class);
if (modificationTime != null && modificationTime >= latestDeletedEdgeTime) {
latestDeletedEdgeTime = modificationTime;
latestDeletedEdge = edge;
}
}
}
if (LOG.isDebugEnabled()) {
LOG.debug("Found {}", latestDeletedEdge == null ? "null" : string(latestDeletedEdge));
}
//If the vertex is deleted, return latest deleted edge
return latestDeletedEdge;
}
public static String vertexString(final AtlasVertex vertex) {
StringBuilder properties = new StringBuilder();
for (String propertyKey : vertex.getPropertyKeys()) {
Collection<?> propertyValues = vertex.getPropertyValues(propertyKey, Object.class);
properties.append(propertyKey).append("=").append(propertyValues.toString()).append(", ");
}
return "v[" + vertex.getIdForDisplay() + "], Properties[" + properties + "]";
}
public static String edgeString(final AtlasEdge edge) {
return "e[" + edge.getLabel() + "], [" + edge.getOutVertex() + " -> " + edge.getLabel() + " -> "
+ edge.getInVertex() + "]";
}
public static <T extends AtlasElement> void setProperty(T element, String propertyName, Object value) {
String actualPropertyName = GraphHelper.encodePropertyKey(propertyName);
String elementStr = null;
if (LOG.isDebugEnabled()) {
elementStr = string(element);
LOG.debug("Setting property {} = \"{}\" to {}", actualPropertyName, value, elementStr);
}
Object existValue = element.getProperty(actualPropertyName, Object.class);
if(value == null || (value instanceof Collection && ((Collection) value).isEmpty())) {
if(existValue != null) {
if (LOG.isDebugEnabled()) {
LOG.debug("Removing property - {} value from {}", actualPropertyName, elementStr);
}
element.removeProperty(actualPropertyName);
}
} else {
if (!value.equals(existValue)) {
element.setProperty(actualPropertyName, value);
if (LOG.isDebugEnabled()) {
LOG.debug("Set property {} = \"{}\" to {}", actualPropertyName, value, elementStr);
}
}
}
}
/**
* Gets the value of a property that is stored in the graph as a single property value. If
* a multi-property such as {@link Constants#TRAIT_NAMES_PROPERTY_KEY} or {@link Constants#SUPER_TYPES_PROPERTY_KEY}
* is used, an exception will be thrown.
*
* @param element
* @param propertyName
* @param clazz
* @return
*/
public static <T> T getSingleValuedProperty(AtlasElement element, String propertyName, Class<T> clazz) {
String actualPropertyName = GraphHelper.encodePropertyKey(propertyName);
if (LOG.isDebugEnabled()) {
LOG.debug("Reading property {} from {}", actualPropertyName, string(element));
}
return element.getProperty(actualPropertyName, clazz);
}
public static Object getProperty(AtlasVertex<?,?> vertex, String propertyName) {
String actualPropertyName = GraphHelper.encodePropertyKey(propertyName);
if (LOG.isDebugEnabled()) {
LOG.debug("Reading property {} from {}", actualPropertyName, string(vertex));
}
if(AtlasGraphProvider.getGraphInstance().isMultiProperty(actualPropertyName)) {
return vertex.getPropertyValues(actualPropertyName, String.class);
}
else {
return vertex.getProperty(actualPropertyName, Object.class);
}
}
public static Object getProperty(AtlasEdge<?,?> edge, String propertyName) {
String actualPropertyName = GraphHelper.encodePropertyKey(propertyName);
if (LOG.isDebugEnabled()) {
LOG.debug("Reading property {} from {}", actualPropertyName, string(edge));
}
return edge.getProperty(actualPropertyName, Object.class);
}
private static <T extends AtlasElement> String string(T element) {
if (element instanceof AtlasVertex) {
return string((AtlasVertex) element);
} else if (element instanceof AtlasEdge) {
return string((AtlasEdge)element);
}
return element.toString();
}
/**
* Adds an additional value to a multi-property.
*
* @param vertex
* @param propertyName
* @param value
*/
public static void addProperty(AtlasVertex vertex, String propertyName, Object value) {
String actualPropertyName = GraphHelper.encodePropertyKey(propertyName);
if (LOG.isDebugEnabled()) {
LOG.debug("Adding property {} = \"{}\" to vertex {}", actualPropertyName, value, string(vertex));
}
vertex.addProperty(actualPropertyName, value);
}
/**
* Remove the specified edge from the graph.
*
* @param edge
*/
public void removeEdge(AtlasEdge edge) {
String edgeString = null;
if (LOG.isDebugEnabled()) {
edgeString = string(edge);
LOG.debug("Removing {}", edgeString);
}
graph.removeEdge(edge);
if (LOG.isDebugEnabled()) {
LOG.info("Removed {}", edgeString);
}
}
/**
* Remove the specified AtlasVertex from the graph.
*
* @param vertex
*/
public void removeVertex(AtlasVertex vertex) {
String vertexString = null;
if (LOG.isDebugEnabled()) {
vertexString = string(vertex);
LOG.debug("Removing {}", vertexString);
}
graph.removeVertex(vertex);
if (LOG.isDebugEnabled()) {
LOG.info("Removed {}", vertexString);
}
}
public AtlasVertex getVertexForGUID(String guid) throws EntityNotFoundException {
return findVertex(Constants.GUID_PROPERTY_KEY, guid);
}
/**
* Finds the Vertices that correspond to the given property values. Property
* values that are not found in the graph will not be in the map.
*
* @return propertyValue to AtlasVertex map with the result.
*/
public Map<String, AtlasVertex> getVerticesForPropertyValues(String property, List<String> values) {
if(values.isEmpty()) {
return Collections.emptyMap();
}
Collection<String> nonNullValues = new HashSet<>(values.size());
for(String value : values) {
if(value != null) {
nonNullValues.add(value);
}
}
//create graph query that finds vertices with the guids
AtlasGraphQuery query = graph.query();
query.in(property, nonNullValues);
Iterable<AtlasVertex> results = query.vertices();
Map<String, AtlasVertex> result = new HashMap<>(values.size());
//Process the result, using the guidToIndexMap to figure out where
//each vertex should go in the result list.
for(AtlasVertex vertex : results) {
if(vertex.exists()) {
String propertyValue = vertex.getProperty(property, String.class);
if(LOG.isDebugEnabled()) {
LOG.debug("Found a vertex {} with {} = {}", string(vertex), property, propertyValue);
}
result.put(propertyValue, vertex);
}
}
return result;
}
/**
* Finds the Vertices that correspond to the given GUIDs. GUIDs
* that are not found in the graph will not be in the map.
*
* @return GUID to AtlasVertex map with the result.
*/
public Map<String, AtlasVertex> getVerticesForGUIDs(List<String> guids) {
return getVerticesForPropertyValues(Constants.GUID_PROPERTY_KEY, guids);
}
public static String getQualifiedNameForMapKey(String prefix, String key) {
return prefix + "." + key;
}
public static String getQualifiedFieldName(ITypedInstance typedInstance, AttributeInfo attributeInfo) throws AtlasException {
IDataType dataType = typeSystem.getDataType(IDataType.class, typedInstance.getTypeName());
return getQualifiedFieldName(dataType, attributeInfo.name);
}
public static String getQualifiedFieldName(IDataType dataType, String attributeName) throws AtlasException {
return dataType.getTypeCategory() == DataTypes.TypeCategory.STRUCT ? dataType.getName() + "." + attributeName
// else class or trait
: ((HierarchicalType) dataType).getQualifiedName(attributeName);
}
public static String getTraitLabel(String typeName, String attrName) {
return attrName;
}
public static List<String> getTraitNames(AtlasVertex<?,?> entityVertex) {
ArrayList<String> traits = new ArrayList<>();
Collection<String> propertyValues = entityVertex.getPropertyValues(Constants.TRAIT_NAMES_PROPERTY_KEY, String.class);
for(String value : propertyValues) {
traits.add(value);
}
return traits;
}
public static List<String> getSuperTypeNames(AtlasVertex<?,?> entityVertex) {
ArrayList<String> superTypes = new ArrayList<>();
Collection<String> propertyValues = entityVertex.getPropertyValues(Constants.SUPER_TYPES_PROPERTY_KEY, String.class);
if (CollectionUtils.isNotEmpty(propertyValues)) {
for(String value : propertyValues) {
superTypes.add(value);
}
}
return superTypes;
}
public static String getEdgeLabel(ITypedInstance typedInstance, AttributeInfo aInfo) throws AtlasException {
IDataType dataType = typeSystem.getDataType(IDataType.class, typedInstance.getTypeName());
return getEdgeLabel(dataType, aInfo);
}
public static String getEdgeLabel(IDataType dataType, AttributeInfo aInfo) throws AtlasException {
return GraphHelper.EDGE_LABEL_PREFIX + getQualifiedFieldName(dataType, aInfo.name);
}
public static Id getIdFromVertex(String dataTypeName, AtlasVertex vertex) {
return new Id(getGuid(vertex),
getVersion(vertex), dataTypeName,
getStateAsString(vertex));
}
public static Id getIdFromVertex(AtlasVertex vertex) {
return getIdFromVertex(getTypeName(vertex), vertex);
}
public static String getGuid(AtlasVertex vertex) {
return vertex.<String>getProperty(Constants.GUID_PROPERTY_KEY, String.class);
}
public static String getTypeName(AtlasVertex instanceVertex) {
return instanceVertex.getProperty(Constants.ENTITY_TYPE_PROPERTY_KEY, String.class);
}
public static Id.EntityState getState(AtlasElement element) {
String state = getStateAsString(element);
return state == null ? null : Id.EntityState.valueOf(state);
}
public static Integer getVersion(AtlasElement element) {
return element.getProperty(Constants.VERSION_PROPERTY_KEY, Integer.class);
}
public static String getStateAsString(AtlasElement element) {
return element.getProperty(Constants.STATE_PROPERTY_KEY, String.class);
}
public static Status getStatus(AtlasElement element) {
return (getState(element) == Id.EntityState.DELETED) ? Status.DELETED : Status.ACTIVE;
}
//Added conditions in fetching system attributes to handle test failures in GremlinTest where these properties are not set
public static String getCreatedByAsString(AtlasElement element){
return element.getProperty(Constants.CREATED_BY_KEY, String.class);
}
public static String getModifiedByAsString(AtlasElement element){
return element.getProperty(Constants.MODIFIED_BY_KEY, String.class);
}
public static long getCreatedTime(AtlasElement element){
return element.getProperty(Constants.TIMESTAMP_PROPERTY_KEY, Long.class);
}
public static long getModifiedTime(AtlasElement element){
return element.getProperty(Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY, Long.class);
}
/**
* For the given type, finds an unique attribute and checks if there is an existing instance with the same
* unique value
*
* @param classType
* @param instance
* @return
* @throws AtlasException
*/
public AtlasVertex getVertexForInstanceByUniqueAttribute(ClassType classType, IReferenceableInstance instance)
throws AtlasException {
if (LOG.isDebugEnabled()) {
LOG.debug("Checking if there is an instance with the same unique attributes for instance {}", instance.toShortString());
}
AtlasVertex result = null;
for (AttributeInfo attributeInfo : classType.fieldMapping().fields.values()) {
if (attributeInfo.isUnique) {
String propertyKey = getQualifiedFieldName(classType, attributeInfo.name);
try {
result = findVertex(propertyKey, instance.get(attributeInfo.name),
Constants.ENTITY_TYPE_PROPERTY_KEY, classType.getName(),
Constants.STATE_PROPERTY_KEY, Id.EntityState.ACTIVE.name());
if (LOG.isDebugEnabled()) {
LOG.debug("Found vertex by unique attribute : {}={}", propertyKey, instance.get(attributeInfo.name));
}
} catch (EntityNotFoundException e) {
//Its ok if there is no entity with the same unique value
}
}
}
return result;
}
/**
* Finds vertices that match at least one unique attribute of the instances specified. The AtlasVertex at a given index in the result corresponds
* to the IReferencableInstance at that same index that was passed in. The number of elements in the resultant list is guaranteed to match the
* number of instances that were passed in. If no vertex is found for a given instance, that entry will be null in the resultant list.
*
*
* @param classType
* @param instancesForClass
* @return
* @throws AtlasException
*/
public List<AtlasVertex> getVerticesForInstancesByUniqueAttribute(ClassType classType, List<? extends IReferenceableInstance> instancesForClass) throws AtlasException {
//For each attribute, need to figure out what values to search for and which instance(s)
//those values correspond to.
Map<String, AttributeValueMap> map = new HashMap<String, AttributeValueMap>();
for (AttributeInfo attributeInfo : classType.fieldMapping().fields.values()) {
if (attributeInfo.isUnique) {
String propertyKey = getQualifiedFieldName(classType, attributeInfo.name);
AttributeValueMap mapForAttribute = new AttributeValueMap();
for(int idx = 0; idx < instancesForClass.size(); idx++) {
IReferenceableInstance instance = instancesForClass.get(idx);
Object value = instance.get(attributeInfo.name);
mapForAttribute.put(value, instance, idx);
}
map.put(propertyKey, mapForAttribute);
}
}
AtlasVertex[] result = new AtlasVertex[instancesForClass.size()];
if(map.isEmpty()) {
//no unique attributes
return Arrays.asList(result);
}
//construct gremlin query
AtlasGraphQuery query = graph.query();
query.has(Constants.ENTITY_TYPE_PROPERTY_KEY, classType.getName());
query.has(Constants.STATE_PROPERTY_KEY,Id.EntityState.ACTIVE.name());
List<AtlasGraphQuery> orChildren = new ArrayList<AtlasGraphQuery>();
//build up an or expression to find vertices which match at least one of the unique attribute constraints
//For each unique attributes, we add a within clause to match vertices that have a value of that attribute
//that matches the value in some instance.
for(Map.Entry<String, AttributeValueMap> entry : map.entrySet()) {
AtlasGraphQuery orChild = query.createChildQuery();
String propertyName = entry.getKey();
AttributeValueMap valueMap = entry.getValue();
Set<Object> values = valueMap.getAttributeValues();
if(values.size() == 1) {
orChild.has(propertyName, values.iterator().next());
}
else if(values.size() > 1) {
orChild.in(propertyName, values);
}
orChildren.add(orChild);
}
if(orChildren.size() == 1) {
AtlasGraphQuery child = orChildren.get(0);
query.addConditionsFrom(child);
}
else if(orChildren.size() > 1) {
query.or(orChildren);
}
Iterable<AtlasVertex> queryResult = query.vertices();
for(AtlasVertex matchingVertex : queryResult) {
Collection<IndexedInstance> matches = getInstancesForVertex(map, matchingVertex);
for(IndexedInstance wrapper : matches) {
result[wrapper.getIndex()]= matchingVertex;
}
}
return Arrays.asList(result);
}
//finds the instance(s) that correspond to the given vertex
private Collection<IndexedInstance> getInstancesForVertex(Map<String, AttributeValueMap> map, AtlasVertex foundVertex) {
//loop through the unique attributes. For each attribute, check to see if the vertex property that
//corresponds to that attribute has a value from one or more of the instances that were passed in.
for(Map.Entry<String, AttributeValueMap> entry : map.entrySet()) {
String propertyName = entry.getKey();
AttributeValueMap valueMap = entry.getValue();
Object vertexValue = foundVertex.getProperty(propertyName, Object.class);
Collection<IndexedInstance> instances = valueMap.get(vertexValue);
if(instances != null && instances.size() > 0) {
//return first match. Let the underling graph determine if this is a problem
//(i.e. if the other unique attributes change be changed safely to match what
//the user requested).
return instances;
}
//try another attribute
}
return Collections.emptyList();
}
/**
* Guid and AtlasVertex combo
*/
public static class VertexInfo {
private String guid;
private AtlasVertex vertex;
private String typeName;
public VertexInfo(String guid, AtlasVertex vertex, String typeName) {
this.guid = guid;
this.vertex = vertex;
this.typeName = typeName;
}
public String getGuid() {
return guid;
}
public AtlasVertex getVertex() {
return vertex;
}
public String getTypeName() {
return typeName;
}
@Override
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
VertexInfo that = (VertexInfo) o;
return Objects.equals(guid, that.guid) &&
Objects.equals(vertex, that.vertex) &&
Objects.equals(typeName, that.typeName);
}
@Override
public int hashCode() {
return Objects.hash(guid, vertex, typeName);
}
}
/**
* Get the GUIDs and vertices for all composite entities owned/contained by the specified root entity AtlasVertex.
* The graph is traversed from the root entity through to the leaf nodes of the containment graph.
*
* @param entityVertex the root entity vertex
* @return set of VertexInfo for all composite entities
* @throws AtlasException
*/
public Set<VertexInfo> getCompositeVertices(AtlasVertex entityVertex) throws AtlasException {
Set<VertexInfo> result = new HashSet<>();
Stack<AtlasVertex> vertices = new Stack<>();
vertices.push(entityVertex);
while (vertices.size() > 0) {
AtlasVertex vertex = vertices.pop();
String typeName = GraphHelper.getTypeName(vertex);
String guid = GraphHelper.getGuid(vertex);
Id.EntityState state = GraphHelper.getState(vertex);
if (state == Id.EntityState.DELETED) {
//If the reference vertex is marked for deletion, skip it
continue;
}
result.add(new VertexInfo(guid, vertex, typeName));
ClassType classType = typeSystem.getDataType(ClassType.class, typeName);
for (AttributeInfo attributeInfo : classType.fieldMapping().fields.values()) {
if (!attributeInfo.isComposite) {
continue;
}
String edgeLabel = GraphHelper.getEdgeLabel(classType, attributeInfo);
switch (attributeInfo.dataType().getTypeCategory()) {
case CLASS:
AtlasEdge edge = getEdgeForLabel(vertex, edgeLabel);
if (edge != null && GraphHelper.getState(edge) == Id.EntityState.ACTIVE) {
AtlasVertex compositeVertex = edge.getInVertex();
vertices.push(compositeVertex);
}
break;
case ARRAY:
IDataType elementType = ((DataTypes.ArrayType) attributeInfo.dataType()).getElemType();
DataTypes.TypeCategory elementTypeCategory = elementType.getTypeCategory();
if (elementTypeCategory != TypeCategory.CLASS) {
continue;
}
Iterator<AtlasEdge> edges = getOutGoingEdgesByLabel(vertex, edgeLabel);
if (edges != null) {
while (edges.hasNext()) {
edge = edges.next();
if (edge != null && GraphHelper.getState(edge) == Id.EntityState.ACTIVE) {
AtlasVertex compositeVertex = edge.getInVertex();
vertices.push(compositeVertex);
}
}
}
break;
case MAP:
DataTypes.MapType mapType = (DataTypes.MapType) attributeInfo.dataType();
DataTypes.TypeCategory valueTypeCategory = mapType.getValueType().getTypeCategory();
if (valueTypeCategory != TypeCategory.CLASS) {
continue;
}
String propertyName = GraphHelper.getQualifiedFieldName(classType, attributeInfo.name);
List<String> keys = vertex.getProperty(propertyName, List.class);
if (keys != null) {
for (String key : keys) {
String mapEdgeLabel = GraphHelper.getQualifiedNameForMapKey(edgeLabel, key);
edge = getEdgeForLabel(vertex, mapEdgeLabel);
if (edge != null && GraphHelper.getState(edge) == Id.EntityState.ACTIVE) {
AtlasVertex compositeVertex = edge.getInVertex();
vertices.push(compositeVertex);
}
}
}
break;
default:
}
}
}
return result;
}
public static ITypedReferenceableInstance[] deserializeClassInstances(TypeSystem typeSystem, String entityInstanceDefinition)
throws AtlasException {
try {
JSONArray referableInstances = new JSONArray(entityInstanceDefinition);
ITypedReferenceableInstance[] instances = new ITypedReferenceableInstance[referableInstances.length()];
for (int index = 0; index < referableInstances.length(); index++) {
Referenceable entityInstance =
InstanceSerialization.fromJsonReferenceable(referableInstances.getString(index), true);
ITypedReferenceableInstance typedInstrance = getTypedReferenceableInstance(typeSystem, entityInstance);
instances[index] = typedInstrance;
}
return instances;
} catch(ValueConversionException | TypeNotFoundException e) {
throw e;
} catch (Exception e) { // exception from deserializer
LOG.error("Unable to deserialize json={}", entityInstanceDefinition, e);
throw new IllegalArgumentException("Unable to deserialize json", e);
}
}
public static ITypedReferenceableInstance getTypedReferenceableInstance(TypeSystem typeSystem, Referenceable entityInstance)
throws AtlasException {
final String entityTypeName = ParamChecker.notEmpty(entityInstance.getTypeName(), "Entity type cannot be null");
ClassType entityType = typeSystem.getDataType(ClassType.class, entityTypeName);
//Both assigned id and values are required for full update
//classtype.convert() will remove values if id is assigned. So, set temp id, convert and
// then replace with original id
Id origId = entityInstance.getId();
entityInstance.replaceWithNewId(new Id(entityInstance.getTypeName()));
ITypedReferenceableInstance typedInstrance = entityType.convert(entityInstance, Multiplicity.REQUIRED);
((ReferenceableInstance)typedInstrance).replaceWithNewId(origId);
return typedInstrance;
}
public static boolean isReference(IDataType type) {
return type.getTypeCategory() == DataTypes.TypeCategory.STRUCT ||
type.getTypeCategory() == DataTypes.TypeCategory.CLASS;
}
public static void setArrayElementsProperty(IDataType elementType, AtlasVertex instanceVertex, String propertyName, List<Object> values) {
String actualPropertyName = GraphHelper.encodePropertyKey(propertyName);
if(GraphHelper.isReference(elementType)) {
setListPropertyFromElementIds(instanceVertex, actualPropertyName, (List)values);
}
else {
setProperty(instanceVertex, actualPropertyName, values);
}
}
public static void setMapValueProperty(IDataType elementType, AtlasVertex instanceVertex, String propertyName, Object value) {
String actualPropertyName = GraphHelper.encodePropertyKey(propertyName);
if(GraphHelper.isReference(elementType)) {
instanceVertex.setPropertyFromElementId(actualPropertyName, (AtlasEdge)value);
}
else {
instanceVertex.setProperty(actualPropertyName, value);
}
}
public static Object getMapValueProperty(IDataType elementType, AtlasVertex instanceVertex, String propertyName) {
String actualPropertyName = GraphHelper.encodePropertyKey(propertyName);
if(GraphHelper.isReference(elementType)) {
return instanceVertex.getProperty(actualPropertyName, AtlasEdge.class);
}
else {
return instanceVertex.getProperty(actualPropertyName, Object.class).toString();
}
}
public static Object getMapValueProperty(AtlasType elementType, AtlasVertex instanceVertex, String propertyName) {
String vertexPropertyName = GraphHelper.encodePropertyKey(propertyName);
if (AtlasGraphUtilsV1.isReference(elementType)) {
return instanceVertex.getProperty(vertexPropertyName, AtlasEdge.class);
} else {
return instanceVertex.getProperty(vertexPropertyName, Object.class).toString();
}
}
// newly added
public static List<Object> getArrayElementsProperty(AtlasType elementType, AtlasVertex instanceVertex, String propertyName) {
String encodedPropertyName = GraphHelper.encodePropertyKey(propertyName);
if(AtlasGraphUtilsV1.isReference(elementType)) {
return (List)instanceVertex.getListProperty(encodedPropertyName, AtlasEdge.class);
}
else {
return (List)instanceVertex.getListProperty(encodedPropertyName);
}
}
public static List<Object> getArrayElementsProperty(IDataType elementType, AtlasVertex instanceVertex, String propertyName) {
String actualPropertyName = GraphHelper.encodePropertyKey(propertyName);
if(GraphHelper.isReference(elementType)) {
return (List)instanceVertex.getListProperty(actualPropertyName, AtlasEdge.class);
}
else {
return (List)instanceVertex.getListProperty(actualPropertyName);
}
}
public static void dumpToLog(final AtlasGraph<?,?> graph) {
LOG.debug("*******************Graph Dump****************************");
LOG.debug("Vertices of {}", graph);
for (AtlasVertex vertex : graph.getVertices()) {
LOG.debug(vertexString(vertex));
}
LOG.debug("Edges of {}", graph);
for (AtlasEdge edge : graph.getEdges()) {
LOG.debug(edgeString(edge));
}
LOG.debug("*******************Graph Dump****************************");
}
public static String string(ITypedReferenceableInstance instance) {
return String.format("entity[type=%s guid=%s]", instance.getTypeName(), instance.getId()._getId());
}
public static String string(AtlasVertex<?,?> vertex) {
if(vertex == null) {
return "vertex[null]";
} else {
if (LOG.isDebugEnabled()) {
return getVertexDetails(vertex);
} else {
return String.format("vertex[id=%s]", vertex.getIdForDisplay());
}
}
}
public static String getVertexDetails(AtlasVertex<?,?> vertex) {
return String.format("vertex[id=%s type=%s guid=%s]", vertex.getIdForDisplay(), getTypeName(vertex),
getGuid(vertex));
}
public static String string(AtlasEdge<?,?> edge) {
if(edge == null) {
return "edge[null]";
} else {
if (LOG.isDebugEnabled()) {
return getEdgeDetails(edge);
} else {
return String.format("edge[id=%s]", edge.getIdForDisplay());
}
}
}
public static String getEdgeDetails(AtlasEdge<?,?> edge) {
return String.format("edge[id=%s label=%s from %s -> to %s]", edge.getIdForDisplay(), edge.getLabel(),
string(edge.getOutVertex()), string(edge.getInVertex()));
}
@VisibleForTesting
//Keys copied from com.thinkaurelius.titan.graphdb.types.StandardRelationTypeMaker
//Titan checks that these chars are not part of any keys. So, encoding...
public static BiMap<String, String> RESERVED_CHARS_ENCODE_MAP =
HashBiMap.create(new HashMap<String, String>() {{
put("{", "_o");
put("}", "_c");
put("\"", "_q");
put("$", "_d");
put("%", "_p");
}});
public static String encodePropertyKey(String key) {
if (StringUtils.isBlank(key)) {
return key;
}
for (String str : RESERVED_CHARS_ENCODE_MAP.keySet()) {
key = key.replace(str, RESERVED_CHARS_ENCODE_MAP.get(str));
}
return key;
}
public static String decodePropertyKey(String key) {
if (StringUtils.isBlank(key)) {
return key;
}
for (String encodedStr : RESERVED_CHARS_ENCODE_MAP.values()) {
key = key.replace(encodedStr, RESERVED_CHARS_ENCODE_MAP.inverse().get(encodedStr));
}
return key;
}
public Object getVertexId(String guid) throws EntityNotFoundException {
AtlasVertex instanceVertex = getVertexForGUID(guid);
Object instanceVertexId = instanceVertex.getId();
return instanceVertexId;
}
public static AttributeInfo getAttributeInfoForSystemAttributes(String field) {
switch (field) {
case Constants.STATE_PROPERTY_KEY:
case Constants.GUID_PROPERTY_KEY:
case Constants.CREATED_BY_KEY:
case Constants.MODIFIED_BY_KEY:
return TypesUtil.newAttributeInfo(field, DataTypes.STRING_TYPE);
case Constants.TIMESTAMP_PROPERTY_KEY:
case Constants.MODIFICATION_TIMESTAMP_PROPERTY_KEY:
return TypesUtil.newAttributeInfo(field, DataTypes.DATE_TYPE);
}
return null;
}
public static boolean elementExists(AtlasElement v) {
return v != null && v.exists();
}
public static void setListPropertyFromElementIds(AtlasVertex<?, ?> instanceVertex, String propertyName,
List<AtlasElement> elements) {
String actualPropertyName = GraphHelper.encodePropertyKey(propertyName);
instanceVertex.setPropertyFromElementsIds(actualPropertyName, elements);
}
public static void setPropertyFromElementId(AtlasVertex<?, ?> instanceVertex, String propertyName,
AtlasElement value) {
String actualPropertyName = GraphHelper.encodePropertyKey(propertyName);
instanceVertex.setPropertyFromElementId(actualPropertyName, value);
}
public static void setListProperty(AtlasVertex instanceVertex, String propertyName, ArrayList<String> value) throws AtlasException {
String actualPropertyName = GraphHelper.encodePropertyKey(propertyName);
instanceVertex.setListProperty(actualPropertyName, value);
}
public static List<String> getListProperty(AtlasVertex instanceVertex, String propertyName) {
String actualPropertyName = GraphHelper.encodePropertyKey(propertyName);
return instanceVertex.getListProperty(actualPropertyName);
}
private String getConditionString(Object[] args) {
StringBuilder condition = new StringBuilder();
for (int i = 0; i < args.length; i+=2) {
condition.append(args[i]).append(" = ").append(args[i+1]).append(", ");
}
return condition.toString();
}
}